diff options
author | reshilkin <[email protected]> | 2023-08-11 12:01:29 +0300 |
---|---|---|
committer | reshilkin <[email protected]> | 2023-08-11 13:00:03 +0300 |
commit | 5894c3ce50a49a9f9bc1c0316e2ef0708e3a7ef8 (patch) | |
tree | 624e56690fe689e569d05612fa0e92f2c5bdb75b /contrib/libs/flatbuffers/src | |
parent | 1eb895279c52b0d2505a31b79ad326b56d0b2681 (diff) |
Update contrib/libs/flatbuffers to 23.5.9
Diffstat (limited to 'contrib/libs/flatbuffers/src')
58 files changed, 16814 insertions, 4978 deletions
diff --git a/contrib/libs/flatbuffers/src/annotated_binary_text_gen.cpp b/contrib/libs/flatbuffers/src/annotated_binary_text_gen.cpp new file mode 100644 index 00000000000..3967066b168 --- /dev/null +++ b/contrib/libs/flatbuffers/src/annotated_binary_text_gen.cpp @@ -0,0 +1,461 @@ +#include "annotated_binary_text_gen.h" + +#include <algorithm> +#include <cstdint> +#include <fstream> +#include <ostream> +#include <sstream> +#include <string> + +#include "binary_annotator.h" +#include "flatbuffers/base.h" +#include "flatbuffers/util.h" + +namespace flatbuffers { +namespace { + +struct OutputConfig { + size_t largest_type_string = 10; + + size_t largest_value_string = 20; + + size_t max_bytes_per_line = 8; + + size_t offset_max_char = 4; + + char delimiter = '|'; + + bool include_vector_contents = true; +}; + +static std::string ToString(const BinarySectionType type) { + switch (type) { + case BinarySectionType::Header: return "header"; + case BinarySectionType::Table: return "table"; + case BinarySectionType::RootTable: return "root_table"; + case BinarySectionType::VTable: return "vtable"; + case BinarySectionType::Struct: return "struct"; + case BinarySectionType::String: return "string"; + case BinarySectionType::Vector: return "vector"; + case BinarySectionType::Vector64: return "vector64"; + case BinarySectionType::Unknown: return "unknown"; + case BinarySectionType::Union: return "union"; + case BinarySectionType::Padding: return "padding"; + default: return "todo"; + } +} + +static bool IsOffset(const BinaryRegionType type) { + return type == BinaryRegionType::UOffset || + type == BinaryRegionType::SOffset || + type == BinaryRegionType::UOffset64; +} + +template<typename T> std::string ToString(T value) { + if (std::is_floating_point<T>::value) { + std::stringstream ss; + ss << value; + return ss.str(); + } else { + return std::to_string(value); + } +} + +template<typename T> +std::string ToValueString(const BinaryRegion ®ion, const uint8_t *binary) { + std::string s; + s += "0x"; + const T val = ReadScalar<T>(binary + region.offset); + const uint64_t start_index = region.offset + region.length - 1; + for (uint64_t i = 0; i < region.length; ++i) { + s += ToHex(binary[start_index - i]); + } + s += " ("; + s += ToString(val); + s += ")"; + return s; +} + +template<> +std::string ToValueString<std::string>(const BinaryRegion ®ion, + const uint8_t *binary) { + return std::string(reinterpret_cast<const char *>(binary + region.offset), + static_cast<size_t>(region.array_length)); +} + +static std::string ToValueString(const BinaryRegion ®ion, + const uint8_t *binary, + const OutputConfig &output_config) { + std::string s; + + if (region.array_length) { + if (region.type == BinaryRegionType::Uint8 || + region.type == BinaryRegionType::Unknown) { + // Interpret each value as a ASCII to aid debugging + for (uint64_t i = 0; i < region.array_length; ++i) { + const uint8_t c = *(binary + region.offset + i); + s += isprint(c) ? static_cast<char>(c & 0x7F) : '.'; + } + return s; + } else if (region.type == BinaryRegionType::Char) { + // string value + return ToValueString<std::string>(region, binary); + } + } + + switch (region.type) { + case BinaryRegionType::Uint32: + return ToValueString<uint32_t>(region, binary); + case BinaryRegionType::Int32: return ToValueString<int32_t>(region, binary); + case BinaryRegionType::Uint16: + return ToValueString<uint16_t>(region, binary); + case BinaryRegionType::Int16: return ToValueString<int16_t>(region, binary); + case BinaryRegionType::Bool: return ToValueString<bool>(region, binary); + case BinaryRegionType::Uint8: return ToValueString<uint8_t>(region, binary); + case BinaryRegionType::Char: return ToValueString<char>(region, binary); + case BinaryRegionType::Byte: + case BinaryRegionType::Int8: return ToValueString<int8_t>(region, binary); + case BinaryRegionType::Int64: return ToValueString<int64_t>(region, binary); + case BinaryRegionType::Uint64: + return ToValueString<uint64_t>(region, binary); + case BinaryRegionType::Double: return ToValueString<double>(region, binary); + case BinaryRegionType::Float: return ToValueString<float>(region, binary); + case BinaryRegionType::UType: return ToValueString<uint8_t>(region, binary); + + // Handle Offsets separately, incase they add additional details. + case BinaryRegionType::UOffset64: + s += ToValueString<uint64_t>(region, binary); + break; + case BinaryRegionType::UOffset: + s += ToValueString<uint32_t>(region, binary); + break; + case BinaryRegionType::SOffset: + s += ToValueString<int32_t>(region, binary); + break; + case BinaryRegionType::VOffset: + s += ToValueString<uint16_t>(region, binary); + break; + + default: break; + } + // If this is an offset type, include the calculated offset location in the + // value. + // TODO(dbaileychess): It might be nicer to put this in the comment field. + if (IsOffset(region.type)) { + s += " Loc: 0x"; + s += ToHex(region.points_to_offset, output_config.offset_max_char); + } + return s; +} + +struct DocContinuation { + // The start column where the value text first starts + size_t value_start_column = 0; + + // The remaining part of the doc to print. + std::string value; +}; + +static std::string GenerateTypeString(const BinaryRegion ®ion) { + return ToString(region.type) + + ((region.array_length) + ? "[" + std::to_string(region.array_length) + "]" + : ""); +} + +static std::string GenerateComment(const BinaryRegionComment &comment, + const BinarySection &) { + std::string s; + switch (comment.type) { + case BinaryRegionCommentType::Unknown: s = "unknown"; break; + case BinaryRegionCommentType::SizePrefix: s = "size prefix"; break; + case BinaryRegionCommentType::RootTableOffset: + s = "offset to root table `" + comment.name + "`"; + break; + // TODO(dbaileychess): make this lowercase to follow the convention. + case BinaryRegionCommentType::FileIdentifier: s = "File Identifier"; break; + case BinaryRegionCommentType::Padding: s = "padding"; break; + case BinaryRegionCommentType::VTableSize: s = "size of this vtable"; break; + case BinaryRegionCommentType::VTableRefferingTableLength: + s = "size of referring table"; + break; + case BinaryRegionCommentType::VTableFieldOffset: + s = "offset to field `" + comment.name; + break; + case BinaryRegionCommentType::VTableUnknownFieldOffset: + s = "offset to unknown field (id: " + std::to_string(comment.index) + ")"; + break; + + case BinaryRegionCommentType::TableVTableOffset: + s = "offset to vtable"; + break; + case BinaryRegionCommentType::TableField: + s = "table field `" + comment.name; + break; + case BinaryRegionCommentType::TableUnknownField: s = "unknown field"; break; + case BinaryRegionCommentType::TableOffsetField: + s = "offset to field `" + comment.name + "`"; + break; + case BinaryRegionCommentType::StructField: + s = "struct field `" + comment.name + "`"; + break; + case BinaryRegionCommentType::ArrayField: + s = "array field `" + comment.name + "`[" + + std::to_string(comment.index) + "]"; + break; + case BinaryRegionCommentType::StringLength: s = "length of string"; break; + case BinaryRegionCommentType::StringValue: s = "string literal"; break; + case BinaryRegionCommentType::StringTerminator: + s = "string terminator"; + break; + case BinaryRegionCommentType::VectorLength: + s = "length of vector (# items)"; + break; + case BinaryRegionCommentType::VectorValue: + s = "value[" + std::to_string(comment.index) + "]"; + break; + case BinaryRegionCommentType::VectorTableValue: + s = "offset to table[" + std::to_string(comment.index) + "]"; + break; + case BinaryRegionCommentType::VectorStringValue: + s = "offset to string[" + std::to_string(comment.index) + "]"; + break; + case BinaryRegionCommentType::VectorUnionValue: + s = "offset to union[" + std::to_string(comment.index) + "]"; + break; + + default: break; + } + if (!comment.default_value.empty()) { s += " " + comment.default_value; } + + switch (comment.status) { + case BinaryRegionStatus::OK: break; // no-op + case BinaryRegionStatus::WARN: s = "WARN: " + s; break; + case BinaryRegionStatus::WARN_NO_REFERENCES: + s = "WARN: nothing refers to this section."; + break; + case BinaryRegionStatus::WARN_CORRUPTED_PADDING: + s = "WARN: could be corrupted padding region."; + break; + case BinaryRegionStatus::WARN_PADDING_LENGTH: + s = "WARN: padding is longer than expected."; + break; + case BinaryRegionStatus::ERROR: s = "ERROR: " + s; break; + case BinaryRegionStatus::ERROR_OFFSET_OUT_OF_BINARY: + s = "ERROR: " + s + ". Invalid offset, points outside the binary."; + break; + case BinaryRegionStatus::ERROR_INCOMPLETE_BINARY: + s = "ERROR: " + s + ". Incomplete binary, expected to read " + + comment.status_message + " bytes."; + break; + case BinaryRegionStatus::ERROR_LENGTH_TOO_LONG: + s = "ERROR: " + s + ". Longer than the binary."; + break; + case BinaryRegionStatus::ERROR_LENGTH_TOO_SHORT: + s = "ERROR: " + s + ". Shorter than the minimum length: "; + break; + case BinaryRegionStatus::ERROR_REQUIRED_FIELD_NOT_PRESENT: + s = "ERROR: " + s + ". Required field is not present."; + break; + case BinaryRegionStatus::ERROR_INVALID_UNION_TYPE: + s = "ERROR: " + s + ". Invalid union type value."; + break; + case BinaryRegionStatus::ERROR_CYCLE_DETECTED: + s = "ERROR: " + s + ". Invalid offset, cycle detected."; + break; + } + + return s; +} + +static void GenerateDocumentation(std::ostream &os, const BinaryRegion ®ion, + const BinarySection §ion, + const uint8_t *binary, + DocContinuation &continuation, + const OutputConfig &output_config) { + // Check if there is a doc continuation that should be prioritized. + if (continuation.value_start_column) { + os << std::string(continuation.value_start_column - 2, ' '); + os << output_config.delimiter << " "; + + os << continuation.value.substr(0, output_config.max_bytes_per_line); + continuation.value = continuation.value.substr( + std::min(output_config.max_bytes_per_line, continuation.value.size())); + return; + } + + size_t size_of = 0; + { + std::stringstream ss; + ss << std::setw(static_cast<int>(output_config.largest_type_string)) + << std::left; + ss << GenerateTypeString(region); + os << ss.str(); + size_of = ss.str().size(); + } + os << " " << output_config.delimiter << " "; + if (region.array_length) { + // Record where the value is first being outputted. + continuation.value_start_column = 3 + size_of; + + // Get the full-length value, which we will chunk below. + const std::string value = ToValueString(region, binary, output_config); + + std::stringstream ss; + ss << std::setw(static_cast<int>(output_config.largest_value_string)) + << std::left; + ss << value.substr(0, output_config.max_bytes_per_line); + os << ss.str(); + + continuation.value = + value.substr(std::min(output_config.max_bytes_per_line, value.size())); + } else { + std::stringstream ss; + ss << std::setw(static_cast<int>(output_config.largest_value_string)) + << std::left; + ss << ToValueString(region, binary, output_config); + os << ss.str(); + } + + os << " " << output_config.delimiter << " "; + os << GenerateComment(region.comment, section); +} + +static void GenerateRegion(std::ostream &os, const BinaryRegion ®ion, + const BinarySection §ion, const uint8_t *binary, + const OutputConfig &output_config) { + bool doc_generated = false; + DocContinuation doc_continuation; + for (uint64_t i = 0; i < region.length; ++i) { + if ((i % output_config.max_bytes_per_line) == 0) { + // Start a new line of output + os << std::endl; + os << " +0x" << ToHex(region.offset + i, output_config.offset_max_char); + os << " " << output_config.delimiter; + } + + // Add each byte + os << " " << ToHex(binary[region.offset + i]); + + // Check for end of line or end of region conditions. + if (((i + 1) % output_config.max_bytes_per_line == 0) || + i + 1 == region.length) { + if (i + 1 == region.length) { + // We are out of bytes but haven't the kMaxBytesPerLine, so we need to + // zero those out to align everything globally. + for (uint64_t j = i + 1; (j % output_config.max_bytes_per_line) != 0; + ++j) { + os << " "; + } + } + os << " " << output_config.delimiter; + // This is the end of the first line or its the last byte of the region, + // generate the end-of-line documentation. + if (!doc_generated) { + os << " "; + GenerateDocumentation(os, region, section, binary, doc_continuation, + output_config); + + // If we have a value in the doc continuation, that means the doc is + // being printed on multiple lines. + doc_generated = doc_continuation.value.empty(); + } + } + } +} + +static void GenerateSection(std::ostream &os, const BinarySection §ion, + const uint8_t *binary, + const OutputConfig &output_config) { + os << std::endl; + os << ToString(section.type); + if (!section.name.empty()) { os << " (" + section.name + ")"; } + os << ":"; + + // As a space saving measure, skip generating every vector element, just put + // the first and last elements in the output. Skip the whole thing if there + // are only three or fewer elements, as it doesn't save space. + if ((section.type == BinarySectionType::Vector || + section.type == BinarySectionType::Vector64) && + !output_config.include_vector_contents && section.regions.size() > 4) { + // Generate the length region which should be first. + GenerateRegion(os, section.regions[0], section, binary, output_config); + + // Generate the first element. + GenerateRegion(os, section.regions[1], section, binary, output_config); + + // Indicate that we omitted elements. + os << std::endl + << " <" << section.regions.size() - 3 << " regions omitted>"; + + // Generate the last element. + GenerateRegion(os, section.regions.back(), section, binary, output_config); + os << std::endl; + return; + } + + for (const BinaryRegion ®ion : section.regions) { + GenerateRegion(os, region, section, binary, output_config); + } + os << std::endl; +} +} // namespace + +bool AnnotatedBinaryTextGenerator::Generate( + const std::string &filename, const std::string &schema_filename) { + OutputConfig output_config; + output_config.max_bytes_per_line = options_.max_bytes_per_line; + output_config.include_vector_contents = options_.include_vector_contents; + + // Given the length of the binary, we can calculate the maximum number of + // characters to display in the offset hex: (i.e. 2 would lead to 0XFF being + // the max output). + output_config.offset_max_char = + binary_length_ > 0xFFFFFF + ? 8 + : (binary_length_ > 0xFFFF ? 6 : (binary_length_ > 0xFF ? 4 : 2)); + + // Find the largest type string of all the regions in this file, so we can + // align the output nicely. + output_config.largest_type_string = 0; + for (const auto §ion : annotations_) { + for (const auto ®ion : section.second.regions) { + std::string s = GenerateTypeString(region); + if (s.size() > output_config.largest_type_string) { + output_config.largest_type_string = s.size(); + } + + // Don't consider array regions, as they will be split to multiple lines. + if (!region.array_length) { + s = ToValueString(region, binary_, output_config); + if (s.size() > output_config.largest_value_string) { + output_config.largest_value_string = s.size(); + } + } + } + } + + // Modify the output filename. + std::string output_filename = StripExtension(filename); + output_filename += options_.output_postfix; + output_filename += + "." + (options_.output_extension.empty() ? GetExtension(filename) + : options_.output_extension); + + std::ofstream ofs(output_filename.c_str()); + + ofs << "// Annotated Flatbuffer Binary" << std::endl; + ofs << "//" << std::endl; + ofs << "// Schema file: " << schema_filename << std::endl; + ofs << "// Binary file: " << filename << std::endl; + + // Generate each of the binary sections + for (const auto §ion : annotations_) { + GenerateSection(ofs, section.second, binary_, output_config); + } + + ofs.close(); + return true; +} + +} // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/annotated_binary_text_gen.h b/contrib/libs/flatbuffers/src/annotated_binary_text_gen.h new file mode 100644 index 00000000000..a0806d88a06 --- /dev/null +++ b/contrib/libs/flatbuffers/src/annotated_binary_text_gen.h @@ -0,0 +1,75 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_ANNOTATED_BINARY_TEXT_GEN_H_ +#define FLATBUFFERS_ANNOTATED_BINARY_TEXT_GEN_H_ + +#include <map> +#include <memory> +#include <string> + +#include "binary_annotator.h" + +namespace flatbuffers { + +class AnnotatedBinaryTextGenerator { + public: + struct Options { + // The maximum number of raw bytes to print per line in the output. 8 is a + // good default due to the largest type (double) being 8 bytes long. + size_t max_bytes_per_line = 8; + + // The output file postfix, appended between the filename and the extension. + // Example binary1.bin -> binary1_annotated.bin + std::string output_postfix = ""; + + // The output file extension, replacing any extension given. If empty, don't + // change the provided extension. AFB = Annotated Flatbuffer Binary + // + // Example: binary1.bin -> binary1.afb + std::string output_extension = "afb"; + + // Controls. + bool include_vector_contents = true; + }; + + explicit AnnotatedBinaryTextGenerator( + const Options &options, std::map<uint64_t, BinarySection> annotations, + const uint8_t *const binary, const int64_t binary_length) + : annotations_(std::move(annotations)), + binary_(binary), + binary_length_(binary_length), + options_(options) {} + + // Generate the annotated binary for the given `filename`. Returns true if the + // annotated binary was successfully saved. + bool Generate(const std::string &filename, + const std::string &schema_filename); + + private: + const std::map<uint64_t, BinarySection> annotations_; + + // The binary data itself. + const uint8_t *binary_; + const int64_t binary_length_; + + // Output configuration + const Options options_; +}; + +} // namespace flatbuffers + +#endif // FLATBUFFERS_ANNOTATED_BINARY_TEXT_GEN_H_ diff --git a/contrib/libs/flatbuffers/src/bfbs_gen.h b/contrib/libs/flatbuffers/src/bfbs_gen.h new file mode 100644 index 00000000000..ed20dc8965e --- /dev/null +++ b/contrib/libs/flatbuffers/src/bfbs_gen.h @@ -0,0 +1,206 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_BFBS_GEN_H_ +#define FLATBUFFERS_BFBS_GEN_H_ + +#include <cstdint> + +#include "flatbuffers/code_generator.h" +#include "flatbuffers/reflection_generated.h" + +namespace flatbuffers { + +namespace { + +static void ForAllEnums( + const flatbuffers::Vector<flatbuffers::Offset<reflection::Enum>> *enums, + std::function<void(const reflection::Enum *)> func) { + for (auto it = enums->cbegin(); it != enums->cend(); ++it) { func(*it); } +} + +static void ForAllObjects( + const flatbuffers::Vector<flatbuffers::Offset<reflection::Object>> *objects, + std::function<void(const reflection::Object *)> func) { + for (auto it = objects->cbegin(); it != objects->cend(); ++it) { func(*it); } +} + +static void ForAllEnumValues( + const reflection::Enum *enum_def, + std::function<void(const reflection::EnumVal *)> func) { + for (auto it = enum_def->values()->cbegin(); it != enum_def->values()->cend(); + ++it) { + func(*it); + } +} + +static void ForAllDocumentation( + const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>> + *documentation, + std::function<void(const flatbuffers::String *)> func) { + if (!documentation) { return; } + for (auto it = documentation->cbegin(); it != documentation->cend(); ++it) { + func(*it); + } +} + +// Maps the field index into object->fields() to the field's ID (the ith element +// in the return vector). +static std::vector<uint32_t> FieldIdToIndex(const reflection::Object *object) { + std::vector<uint32_t> field_index_by_id; + field_index_by_id.resize(object->fields()->size()); + + // Create the mapping of field ID to the index into the vector. + for (uint32_t i = 0; i < object->fields()->size(); ++i) { + auto field = object->fields()->Get(i); + field_index_by_id[field->id()] = i; + } + + return field_index_by_id; +} + +static bool IsStructOrTable(const reflection::BaseType base_type) { + return base_type == reflection::Obj; +} + +static bool IsFloatingPoint(const reflection::BaseType base_type) { + return base_type == reflection::Float || base_type == reflection::Double; +} + +static bool IsBool(const reflection::BaseType base_type) { + return base_type == reflection::Bool; +} + +static bool IsSingleByte(const reflection::BaseType base_type) { + return base_type >= reflection::UType && base_type <= reflection::UByte; +} + +static bool IsVector(const reflection::BaseType base_type) { + return base_type == reflection::Vector; +} + +} // namespace + +// A concrete base Flatbuffer Generator that specific language generators can +// derive from. +class BaseBfbsGenerator : public CodeGenerator { + public: + virtual ~BaseBfbsGenerator() {} + BaseBfbsGenerator() : schema_(nullptr) {} + + virtual Status GenerateFromSchema( + const reflection::Schema *schema) = 0; + + virtual uint64_t SupportedAdvancedFeatures() const = 0; + + // Override of the Generator::GenerateCode method that does the initial + // deserialization and verification steps. + Status GenerateCode(const uint8_t *buffer, + int64_t length) FLATBUFFERS_OVERRIDE { + flatbuffers::Verifier verifier(buffer, static_cast<size_t>(length)); + if (!reflection::VerifySchemaBuffer(verifier)) { + return FAILED_VERIFICATION; + } + + // Store the root schema since there are cases where leaf nodes refer to + // things in the root schema (e.g., indexing the objects). + schema_ = reflection::GetSchema(buffer); + + const uint64_t advance_features = schema_->advanced_features(); + if (advance_features > SupportedAdvancedFeatures()) { + return FAILED_VERIFICATION; + } + + Status status = GenerateFromSchema(schema_); + schema_ = nullptr; + return status; + } + + protected: + // GetObject returns the underlying object struct of the given type + // if element_type is true and GetObject is a list of objects then + // GetObject will correctly return the object struct of the vector's elements + const reflection::Object *GetObject(const reflection::Type *type, + bool element_type = false) const { + const reflection::BaseType base_type = + element_type ? type->element() : type->base_type(); + if (type->index() >= 0 && IsStructOrTable(base_type)) { + return GetObjectByIndex(type->index()); + } + return nullptr; + } + + // GetEnum returns the underlying enum struct of the given type + // if element_type is true and GetEnum is a list of enums then + // GetEnum will correctly return the enum struct of the vector's elements + const reflection::Enum *GetEnum(const reflection::Type *type, + bool element_type = false) const { + const reflection::BaseType base_type = + element_type ? type->element() : type->base_type(); + // TODO(derekbailey): it would be better to have a explicit list of allowed + // base types, instead of negating Obj types. + if (type->index() >= 0 && !IsStructOrTable(base_type)) { + return GetEnumByIndex(type->index()); + } + return nullptr; + } + + // Used to get a object that is reference by index. (e.g. + // reflection::Type::index). Returns nullptr if no object is available. + const reflection::Object *GetObjectByIndex(int32_t index) const { + if (!schema_ || index < 0 || + index >= static_cast<int32_t>(schema_->objects()->size())) { + return nullptr; + } + return schema_->objects()->Get(index); + } + + // Used to get a enum that is reference by index. (e.g. + // reflection::Type::index). Returns nullptr if no enum is available. + const reflection::Enum *GetEnumByIndex(int32_t index) const { + if (!schema_ || index < 0 || + index >= static_cast<int32_t>(schema_->enums()->size())) { + return nullptr; + } + return schema_->enums()->Get(index); + } + + void ForAllFields(const reflection::Object *object, bool reverse, + std::function<void(const reflection::Field *)> func) const { + const std::vector<uint32_t> field_to_id_map = FieldIdToIndex(object); + for (size_t i = 0; i < field_to_id_map.size(); ++i) { + func(object->fields()->Get( + field_to_id_map[reverse ? field_to_id_map.size() - (i + 1) : i])); + } + } + + bool IsTable(const reflection::Type *type, bool use_element = false) const { + return !IsStruct(type, use_element); + } + + bool IsStruct(const reflection::Type *type, bool use_element = false) const { + const reflection::BaseType base_type = + use_element ? type->element() : type->base_type(); + return IsStructOrTable(base_type) && + GetObjectByIndex(type->index())->is_struct(); + } + + const reflection::Schema *schema_; +}; + +} // namespace flatbuffers + +#endif // FLATBUFFERS_BFBS_GEN_H_ diff --git a/contrib/libs/flatbuffers/src/bfbs_gen_lua.cpp b/contrib/libs/flatbuffers/src/bfbs_gen_lua.cpp new file mode 100644 index 00000000000..8823d912b17 --- /dev/null +++ b/contrib/libs/flatbuffers/src/bfbs_gen_lua.cpp @@ -0,0 +1,674 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "bfbs_gen_lua.h" + +#include <cstdint> +#include <map> +#include <memory> +#include <string> +#include <unordered_set> +#include <vector> + +// Ensure no includes to flatc internals. bfbs_gen.h and generator.h are OK. +#include "bfbs_gen.h" +#include "bfbs_namer.h" + +// The intermediate representation schema. +#include "flatbuffers/reflection.h" +#include "flatbuffers/reflection_generated.h" + +namespace flatbuffers { +namespace { + +// To reduce typing +namespace r = ::reflection; + +std::set<std::string> LuaKeywords() { + return { "and", "break", "do", "else", "elseif", "end", + "false", "for", "function", "goto", "if", "in", + "local", "nil", "not", "or", "repeat", "return", + "then", "true", "until", "while" }; +} + +Namer::Config LuaDefaultConfig() { + return { /*types=*/Case::kUpperCamel, + /*constants=*/Case::kUnknown, + /*methods=*/Case::kUpperCamel, + /*functions=*/Case::kUpperCamel, + /*fields=*/Case::kUpperCamel, + /*variables=*/Case::kLowerCamel, + /*variants=*/Case::kKeep, + /*enum_variant_seperator=*/"", + /*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase, + /*namespaces=*/Case::kKeep, + /*namespace_seperator=*/"__", + /*object_prefix=*/"", + /*object_suffix=*/"", + /*keyword_prefix=*/"", + /*keyword_suffix=*/"_", + /*filenames=*/Case::kKeep, + /*directories=*/Case::kKeep, + /*output_path=*/"", + /*filename_suffix=*/"", + /*filename_extension=*/".lua" }; +} + +class LuaBfbsGenerator : public BaseBfbsGenerator { + public: + explicit LuaBfbsGenerator(const std::string &flatc_version) + : BaseBfbsGenerator(), + keywords_(), + requires_(), + current_obj_(nullptr), + current_enum_(nullptr), + flatc_version_(flatc_version), + namer_(LuaDefaultConfig(), LuaKeywords()) {} + + Status GenerateFromSchema(const r::Schema *schema) FLATBUFFERS_OVERRIDE { + if (!GenerateEnums(schema->enums())) { return ERROR; } + if (!GenerateObjects(schema->objects(), schema->root_table())) { + return ERROR; + } + return OK; + } + + using BaseBfbsGenerator::GenerateCode; + + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) FLATBUFFERS_OVERRIDE { + if (!GenerateLua(parser, path, filename)) { return ERROR; } + return OK; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + (void)parser; + (void)path; + (void)filename; + (void)output; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return true; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kLua; } + + std::string LanguageName() const override { return "Lua"; } + + uint64_t SupportedAdvancedFeatures() const FLATBUFFERS_OVERRIDE { + return 0xF; + } + + protected: + bool GenerateEnums( + const flatbuffers::Vector<flatbuffers::Offset<r::Enum>> *enums) { + ForAllEnums(enums, [&](const r::Enum *enum_def) { + std::string code; + + StartCodeBlock(enum_def); + + std::string ns; + const std::string enum_name = + namer_.Type(namer_.Denamespace(enum_def, ns)); + + GenerateDocumentation(enum_def->documentation(), "", code); + code += "local " + enum_name + " = {\n"; + + ForAllEnumValues(enum_def, [&](const reflection::EnumVal *enum_val) { + GenerateDocumentation(enum_val->documentation(), " ", code); + code += " " + namer_.Variant(enum_val->name()->str()) + " = " + + NumToString(enum_val->value()) + ",\n"; + }); + code += "}\n"; + code += "\n"; + + EmitCodeBlock(code, enum_name, ns, enum_def->declaration_file()->str()); + }); + return true; + } + + bool GenerateObjects( + const flatbuffers::Vector<flatbuffers::Offset<r::Object>> *objects, + const r::Object *root_object) { + ForAllObjects(objects, [&](const r::Object *object) { + std::string code; + + StartCodeBlock(object); + + // Register the main flatbuffers module. + RegisterRequires("flatbuffers", "flatbuffers"); + + std::string ns; + const std::string object_name = + namer_.Type(namer_.Denamespace(object, ns)); + + GenerateDocumentation(object->documentation(), "", code); + + code += "local " + object_name + " = {}\n"; + code += "local mt = {}\n"; + code += "\n"; + code += "function " + object_name + ".New()\n"; + code += " local o = {}\n"; + code += " setmetatable(o, {__index = mt})\n"; + code += " return o\n"; + code += "end\n"; + code += "\n"; + + if (object == root_object) { + code += "function " + object_name + ".GetRootAs" + object_name + + "(buf, offset)\n"; + code += " if type(buf) == \"string\" then\n"; + code += " buf = flatbuffers.binaryArray.New(buf)\n"; + code += " end\n"; + code += "\n"; + code += " local n = flatbuffers.N.UOffsetT:Unpack(buf, offset)\n"; + code += " local o = " + object_name + ".New()\n"; + code += " o:Init(buf, n + offset)\n"; + code += " return o\n"; + code += "end\n"; + code += "\n"; + } + + // Generates a init method that receives a pre-existing accessor object, + // so that objects can be reused. + + code += "function mt:Init(buf, pos)\n"; + code += " self.view = flatbuffers.view.New(buf, pos)\n"; + code += "end\n"; + code += "\n"; + + // Create all the field accessors. + ForAllFields(object, /*reverse=*/false, [&](const r::Field *field) { + // Skip writing deprecated fields altogether. + if (field->deprecated()) { return; } + + const std::string field_name = namer_.Field(*field); + const r::BaseType base_type = field->type()->base_type(); + + // Generate some fixed strings so we don't repeat outselves later. + const std::string getter_signature = + "function mt:" + field_name + "()\n"; + const std::string offset_prefix = "local o = self.view:Offset(" + + NumToString(field->offset()) + ")\n"; + const std::string offset_prefix_2 = "if o ~= 0 then\n"; + + GenerateDocumentation(field->documentation(), "", code); + + if (IsScalar(base_type)) { + code += getter_signature; + + if (object->is_struct()) { + // TODO(derekbailey): it would be nice to modify the view:Get to + // just pass in the offset and not have to add it its own + // self.view.pos. + code += " return " + GenerateGetter(field->type()) + + "self.view.pos + " + NumToString(field->offset()) + ")\n"; + } else { + // Table accessors + code += " " + offset_prefix; + code += " " + offset_prefix_2; + + std::string getter = + GenerateGetter(field->type()) + "self.view.pos + o)"; + if (IsBool(base_type)) { getter = "(" + getter + " ~=0)"; } + code += " return " + getter + "\n"; + code += " end\n"; + code += " return " + DefaultValue(field) + "\n"; + } + code += "end\n"; + code += "\n"; + } else { + switch (base_type) { + case r::String: { + code += getter_signature; + code += " " + offset_prefix; + code += " " + offset_prefix_2; + code += " return " + GenerateGetter(field->type()) + + "self.view.pos + o)\n"; + code += " end\n"; + code += "end\n"; + code += "\n"; + break; + } + case r::Obj: { + if (object->is_struct()) { + code += "function mt:" + field_name + "(obj)\n"; + code += " obj:Init(self.view.bytes, self.view.pos + " + + NumToString(field->offset()) + ")\n"; + code += " return obj\n"; + code += "end\n"; + code += "\n"; + } else { + code += getter_signature; + code += " " + offset_prefix; + code += " " + offset_prefix_2; + + const r::Object *field_object = GetObject(field->type()); + if (!field_object) { + // TODO(derekbailey): this is an error condition. we + // should report it better. + return; + } + code += " local x = " + + std::string( + field_object->is_struct() + ? "self.view.pos + o\n" + : "self.view:Indirect(self.view.pos + o)\n"); + const std::string require_name = RegisterRequires(field); + code += " local obj = " + require_name + ".New()\n"; + code += " obj:Init(self.view.bytes, x)\n"; + code += " return obj\n"; + code += " end\n"; + code += "end\n"; + code += "\n"; + } + break; + } + case r::Union: { + code += getter_signature; + code += " " + offset_prefix; + code += " " + offset_prefix_2; + code += + " local obj = " + "flatbuffers.view.New(flatbuffers.binaryArray.New(" + "0), 0)\n"; + code += " " + GenerateGetter(field->type()) + "obj, o)\n"; + code += " return obj\n"; + code += " end\n"; + code += "end\n"; + code += "\n"; + break; + } + case r::Array: + case r::Vector: { + const r::BaseType vector_base_type = field->type()->element(); + int32_t element_size = field->type()->element_size(); + code += "function mt:" + field_name + "(j)\n"; + code += " " + offset_prefix; + code += " " + offset_prefix_2; + + if (IsStructOrTable(vector_base_type)) { + code += " local x = self.view:Vector(o)\n"; + code += + " x = x + ((j-1) * " + NumToString(element_size) + ")\n"; + if (IsTable(field->type(), /*use_element=*/true)) { + code += " x = self.view:Indirect(x)\n"; + } else { + // Vector of structs are inline, so we need to query the + // size of the struct. + const reflection::Object *obj = + GetObjectByIndex(field->type()->index()); + element_size = obj->bytesize(); + } + + // Include the referenced type, thus we need to make sure + // we set `use_element` to true. + const std::string require_name = + RegisterRequires(field, /*use_element=*/true); + code += " local obj = " + require_name + ".New()\n"; + code += " obj:Init(self.view.bytes, x)\n"; + code += " return obj\n"; + } else { + code += " local a = self.view:Vector(o)\n"; + code += " return " + GenerateGetter(field->type()) + + "a + ((j-1) * " + NumToString(element_size) + "))\n"; + } + code += " end\n"; + // Only generate a default value for those types that are + // supported. + if (!IsStructOrTable(vector_base_type)) { + code += + " return " + + std::string(vector_base_type == r::String ? "''\n" : "0\n"); + } + code += "end\n"; + code += "\n"; + + // If the vector is composed of single byte values, we + // generate a helper function to get it as a byte string in + // Lua. + if (IsSingleByte(vector_base_type)) { + code += "function mt:" + field_name + "AsString(start, stop)\n"; + code += " return self.view:VectorAsString(" + + NumToString(field->offset()) + ", start, stop)\n"; + code += "end\n"; + code += "\n"; + } + + // We also make a new accessor to query just the length of the + // vector. + code += "function mt:" + field_name + "Length()\n"; + code += " " + offset_prefix; + code += " " + offset_prefix_2; + code += " return self.view:VectorLen(o)\n"; + code += " end\n"; + code += " return 0\n"; + code += "end\n"; + code += "\n"; + break; + } + default: { + return; + } + } + } + return; + }); + + // Create all the builders + if (object->is_struct()) { + code += "function " + object_name + ".Create" + object_name + + "(builder" + GenerateStructBuilderArgs(object) + ")\n"; + code += AppendStructBuilderBody(object); + code += " return builder:Offset()\n"; + code += "end\n"; + code += "\n"; + } else { + // Table builders + code += "function " + object_name + ".Start(builder)\n"; + code += " builder:StartObject(" + + NumToString(object->fields()->size()) + ")\n"; + code += "end\n"; + code += "\n"; + + ForAllFields(object, /*reverse=*/false, [&](const r::Field *field) { + if (field->deprecated()) { return; } + + const std::string field_name = namer_.Field(*field); + const std::string variable_name = namer_.Variable(*field); + + code += "function " + object_name + ".Add" + field_name + + "(builder, " + variable_name + ")\n"; + code += " builder:Prepend" + GenerateMethod(field) + "Slot(" + + NumToString(field->id()) + ", " + variable_name + ", " + + DefaultValue(field) + ")\n"; + code += "end\n"; + code += "\n"; + + if (IsVector(field->type()->base_type())) { + code += "function " + object_name + ".Start" + field_name + + "Vector(builder, numElems)\n"; + + const int32_t element_size = field->type()->element_size(); + int32_t alignment = 0; + if (IsStruct(field->type(), /*use_element=*/true)) { + alignment = GetObjectByIndex(field->type()->index())->minalign(); + } else { + alignment = element_size; + } + + code += " return builder:StartVector(" + + NumToString(element_size) + ", numElems, " + + NumToString(alignment) + ")\n"; + code += "end\n"; + code += "\n"; + } + }); + + code += "function " + object_name + ".End(builder)\n"; + code += " return builder:EndObject()\n"; + code += "end\n"; + code += "\n"; + } + + EmitCodeBlock(code, object_name, ns, object->declaration_file()->str()); + }); + return true; + } + + private: + void GenerateDocumentation( + const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>> + *documentation, + std::string indent, std::string &code) const { + flatbuffers::ForAllDocumentation( + documentation, [&](const flatbuffers::String *str) { + code += indent + "--" + str->str() + "\n"; + }); + } + + std::string GenerateStructBuilderArgs(const r::Object *object, + std::string prefix = "") const { + std::string signature; + ForAllFields(object, /*reverse=*/false, [&](const r::Field *field) { + if (IsStructOrTable(field->type()->base_type())) { + const r::Object *field_object = GetObject(field->type()); + signature += GenerateStructBuilderArgs( + field_object, prefix + namer_.Variable(*field) + "_"); + } else { + signature += ", " + prefix + namer_.Variable(*field); + } + }); + return signature; + } + + std::string AppendStructBuilderBody(const r::Object *object, + std::string prefix = "") const { + std::string code; + code += " builder:Prep(" + NumToString(object->minalign()) + ", " + + NumToString(object->bytesize()) + ")\n"; + + // We need to reverse the order we iterate over, since we build the + // buffer backwards. + ForAllFields(object, /*reverse=*/true, [&](const r::Field *field) { + const int32_t num_padding_bytes = field->padding(); + if (num_padding_bytes) { + code += " builder:Pad(" + NumToString(num_padding_bytes) + ")\n"; + } + if (IsStructOrTable(field->type()->base_type())) { + const r::Object *field_object = GetObject(field->type()); + code += AppendStructBuilderBody(field_object, + prefix + namer_.Variable(*field) + "_"); + } else { + code += " builder:Prepend" + GenerateMethod(field) + "(" + prefix + + namer_.Variable(*field) + ")\n"; + } + }); + + return code; + } + + std::string GenerateMethod(const r::Field *field) const { + const r::BaseType base_type = field->type()->base_type(); + if (IsScalar(base_type)) { return namer_.Type(GenerateType(base_type)); } + if (IsStructOrTable(base_type)) { return "Struct"; } + return "UOffsetTRelative"; + } + + std::string GenerateGetter(const r::Type *type, + bool element_type = false) const { + switch (element_type ? type->element() : type->base_type()) { + case r::String: return "self.view:String("; + case r::Union: return "self.view:Union("; + case r::Vector: return GenerateGetter(type, true); + default: + return "self.view:Get(flatbuffers.N." + + namer_.Type(GenerateType(type, element_type)) + ", "; + } + } + + std::string GenerateType(const r::Type *type, + bool element_type = false) const { + const r::BaseType base_type = + element_type ? type->element() : type->base_type(); + if (IsScalar(base_type)) { return GenerateType(base_type); } + switch (base_type) { + case r::String: return "string"; + case r::Vector: return GenerateGetter(type, true); + case r::Obj: return namer_.Type(namer_.Denamespace(GetObject(type))); + + default: return "*flatbuffers.Table"; + } + } + + std::string GenerateType(const r::BaseType base_type) const { + // Need to override the default naming to match the Lua runtime libraries. + // TODO(derekbailey): make overloads in the runtime libraries to avoid this. + switch (base_type) { + case r::None: return "uint8"; + case r::UType: return "uint8"; + case r::Byte: return "int8"; + case r::UByte: return "uint8"; + case r::Short: return "int16"; + case r::UShort: return "uint16"; + case r::Int: return "int32"; + case r::UInt: return "uint32"; + case r::Long: return "int64"; + case r::ULong: return "uint64"; + case r::Float: return "Float32"; + case r::Double: return "Float64"; + default: return r::EnumNameBaseType(base_type); + } + } + + std::string DefaultValue(const r::Field *field) const { + const r::BaseType base_type = field->type()->base_type(); + if (IsFloatingPoint(base_type)) { + return NumToString(field->default_real()); + } + if (IsBool(base_type)) { + return field->default_integer() ? "true" : "false"; + } + if (IsScalar(base_type)) { return NumToString((field->default_integer())); } + // represents offsets + return "0"; + } + + void StartCodeBlock(const reflection::Enum *enum_def) { + current_enum_ = enum_def; + current_obj_ = nullptr; + requires_.clear(); + } + + void StartCodeBlock(const reflection::Object *object) { + current_obj_ = object; + current_enum_ = nullptr; + requires_.clear(); + } + + std::string RegisterRequires(const r::Field *field, + bool use_element = false) { + std::string type_name; + + const r::BaseType type = + use_element ? field->type()->element() : field->type()->base_type(); + + if (IsStructOrTable(type)) { + const r::Object *object = GetObjectByIndex(field->type()->index()); + if (object == current_obj_) { return namer_.Denamespace(object); } + type_name = object->name()->str(); + } else { + const r::Enum *enum_def = GetEnumByIndex(field->type()->index()); + if (enum_def == current_enum_) { return namer_.Denamespace(enum_def); } + type_name = enum_def->name()->str(); + } + + // Prefix with double __ to avoid name clashing, since these are defined + // at the top of the file and have lexical scoping. Replace '.' with '_' + // so it can be a legal identifier. + std::string name = "__" + type_name; + std::replace(name.begin(), name.end(), '.', '_'); + + return RegisterRequires(name, type_name); + } + + std::string RegisterRequires(const std::string &local_name, + const std::string &requires_name) { + requires_[local_name] = requires_name; + return local_name; + } + + void EmitCodeBlock(const std::string &code_block, const std::string &name, + const std::string &ns, + const std::string &declaring_file) const { + const std::string root_type = schema_->root_table()->name()->str(); + const std::string root_file = + schema_->root_table()->declaration_file()->str(); + const std::string full_qualified_name = ns.empty() ? name : ns + "." + name; + + std::string code = "--[[ " + full_qualified_name + "\n\n"; + code += + " Automatically generated by the FlatBuffers compiler, do not " + "modify.\n"; + code += " Or modify. I'm a message, not a cop.\n"; + code += "\n"; + code += " flatc version: " + flatc_version_ + "\n"; + code += "\n"; + code += " Declared by : " + declaring_file + "\n"; + code += " Rooting type : " + root_type + " (" + root_file + ")\n"; + code += "\n--]]\n\n"; + + if (!requires_.empty()) { + for (auto it = requires_.cbegin(); it != requires_.cend(); ++it) { + code += "local " + it->first + " = require('" + it->second + "')\n"; + } + code += "\n"; + } + + code += code_block; + code += "return " + name; + + // Namespaces are '.' deliminted, so replace it with the path separator. + std::string path = ns; + + if (ns.empty()) { + path = "."; + } else { + std::replace(path.begin(), path.end(), '.', '/'); + } + + // TODO(derekbailey): figure out a save file without depending on util.h + EnsureDirExists(path); + const std::string file_name = path + "/" + namer_.File(name); + SaveFile(file_name.c_str(), code, false); + } + + std::unordered_set<std::string> keywords_; + std::map<std::string, std::string> requires_; + const r::Object *current_obj_; + const r::Enum *current_enum_; + const std::string flatc_version_; + const BfbsNamer namer_; +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewLuaBfbsGenerator( + const std::string &flatc_version) { + return std::unique_ptr<LuaBfbsGenerator>(new LuaBfbsGenerator(flatc_version)); +} + +} // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/bfbs_gen_lua.h b/contrib/libs/flatbuffers/src/bfbs_gen_lua.h new file mode 100644 index 00000000000..86d97621ebf --- /dev/null +++ b/contrib/libs/flatbuffers/src/bfbs_gen_lua.h @@ -0,0 +1,33 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_BFBS_GEN_LUA_H_ +#define FLATBUFFERS_BFBS_GEN_LUA_H_ + +#include <memory> +#include <string> + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Lua Code generator. +std::unique_ptr<CodeGenerator> NewLuaBfbsGenerator( + const std::string &flatc_version); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_BFBS_GEN_LUA_H_ diff --git a/contrib/libs/flatbuffers/src/bfbs_gen_nim.cpp b/contrib/libs/flatbuffers/src/bfbs_gen_nim.cpp new file mode 100644 index 00000000000..45bd3c33d41 --- /dev/null +++ b/contrib/libs/flatbuffers/src/bfbs_gen_nim.cpp @@ -0,0 +1,692 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "bfbs_gen_nim.h" + +#include <cstdint> +#include <map> +#include <memory> +#include <string> +#include <unordered_set> +#include <vector> + +// Ensure no includes to flatc internals. bfbs_gen.h and generator.h are OK. +#include "bfbs_gen.h" +#include "bfbs_namer.h" + +// The intermediate representation schema. +#include "flatbuffers/reflection.h" +#include "flatbuffers/reflection_generated.h" + +namespace flatbuffers { +namespace { + +// To reduce typing +namespace r = ::reflection; + +std::set<std::string> NimKeywords() { + return { + "addr", "and", "as", "asm", "bind", "block", + "break", "case", "cast", "concept", "const", "continue", + "converter", "defer", "discard", "distinct", "div", "do", + "elif", "else", "end", "enum", "except", "export", + "finally", "for", "from", "func", "if", "import", + "in", "include", "interface", "is", "isnot", "iterator", + "let", "macro", "method", "mixin", "mod", "nil", + "not", "notin", "object", "of", "or", "out", + "proc", "ptr", "raise", "ref", "return", "shl", + "shr", "static", "template", "try", "tuple", "type", + "using", "var", "when", "while", "xor", "yield", + }; +} + +Namer::Config NimDefaultConfig() { + return { /*types=*/Case::kUpperCamel, + /*constants=*/Case::kUpperCamel, + /*methods=*/Case::kLowerCamel, + /*functions=*/Case::kUpperCamel, + /*fields=*/Case::kLowerCamel, + /*variable=*/Case::kLowerCamel, + /*variants=*/Case::kUpperCamel, + /*enum_variant_seperator=*/".", + /*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase, + /*namespaces=*/Case::kKeep, + /*namespace_seperator=*/"/", + /*object_prefix=*/"", + /*object_suffix=*/"T", + /*keyword_prefix=*/"", + /*keyword_suffix=*/"_", + /*filenames=*/Case::kKeep, + /*directories=*/Case::kKeep, + /*output_path=*/"", + /*filename_suffix=*/"", + /*filename_extension=*/".nim" }; +} + +const std::string Indent = " "; +const std::string Export = "*"; +const std::set<std::string> builtin_types = { + "uint8", "uint8", "bool", "int8", "uint8", "int16", + "uint16", "int32", "uint32", "int64", "uint64", "float32", + "float64", "string", "int", "uint", "uoffset", "Builder" +}; + +class NimBfbsGenerator : public BaseBfbsGenerator { + public: + explicit NimBfbsGenerator(const std::string &flatc_version) + : BaseBfbsGenerator(), + keywords_(), + imports_(), + current_obj_(nullptr), + current_enum_(nullptr), + flatc_version_(flatc_version), + namer_(NimDefaultConfig(), NimKeywords()) {} + + Status GenerateFromSchema(const r::Schema *schema) FLATBUFFERS_OVERRIDE { + ForAllEnums(schema->enums(), [&](const r::Enum *enum_def) { + StartCodeBlock(enum_def); + GenerateEnum(enum_def); + }); + ForAllObjects(schema->objects(), [&](const r::Object *object) { + StartCodeBlock(object); + GenerateObject(object); + }); + return OK; + } + + using BaseBfbsGenerator::GenerateCode; + + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + (void)parser; + (void)path; + (void)filename; + (void)output; + return NOT_IMPLEMENTED; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return true; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kNim; } + + std::string LanguageName() const override { return "Nim"; } + + uint64_t SupportedAdvancedFeatures() const FLATBUFFERS_OVERRIDE { + return r::AdvancedArrayFeatures | r::AdvancedUnionFeatures | + r::OptionalScalars | r::DefaultVectorsAndStrings; + } + + protected: + void GenerateEnum(const r::Enum *enum_def) { + std::string code; + + std::string ns; + const std::string enum_name = namer_.Type(namer_.Denamespace(enum_def, ns)); + const std::string enum_type = + GenerateTypeBasic(enum_def->underlying_type()); + + GenerateDocumentation(enum_def->documentation(), "", code); + code += "type " + enum_name + Export + "{.pure.} = enum\n"; + + ForAllEnumValues(enum_def, [&](const reflection::EnumVal *enum_val) { + GenerateDocumentation(enum_val->documentation(), " ", code); + code += " " + namer_.Variant(enum_val->name()->str()) + " = " + + NumToString(enum_val->value()) + "." + enum_type + ",\n"; + }); + + EmitCodeBlock(code, enum_name, ns, enum_def->declaration_file()->str()); + } + + void GenerateObject(const r::Object *object) { + // Register the main flatbuffers module. + RegisterImports("flatbuffers", ""); + std::string code; + + std::string ns; + const std::string object_name = namer_.Type(namer_.Denamespace(object, ns)); + + GenerateDocumentation(object->documentation(), "", code); + code += "type " + object_name + "* = object of FlatObj\n"; + + // Create all the field accessors. + ForAllFields(object, /*reverse=*/false, [&](const r::Field *field) { + // Skip writing deprecated fields altogether. + if (field->deprecated()) { return; } + + const std::string field_name = namer_.Field(*field); + const r::BaseType base_type = field->type()->base_type(); + std::string field_type = GenerateType(field->type()); + + if (field->optional() && !object->is_struct()) { + RegisterImports("std/options", ""); + field_type = "Option[" + field_type + "]"; + } + + const std::string offset_prefix = + "let o = self.tab.Offset(" + NumToString(field->offset()) + ")\n"; + const std::string offset_prefix_2 = "if o != 0:\n"; + + if (IsScalar(base_type) || base_type == r::String || + base_type == r::Obj || base_type == r::Union) { + GenerateDocumentation(field->documentation(), "", code); + + std::string getter_signature = "func " + namer_.Method(field_name) + + "*(self: " + object_name + + "): " + field_type + " =\n"; + std::string getter_code; + std::string setter_signature = + "func `" + namer_.Method(field_name + "=") + "`*(self: var " + + object_name + ", n: " + field_type + "): bool =\n"; + std::string setter_code; + + if (base_type == r::Obj || base_type == r::Union || + field->type()->index() >= 0) { + RegisterImports(object, field); + } + + if (object->is_struct()) { + std::string field_getter = + GenerateGetter(field->type(), NumToString(field->offset())); + getter_code += " return " + field_getter + "\n"; + + if (IsScalar(base_type)) { + setter_code += " return self.tab.Mutate(self.tab.Pos + " + + NumToString(field->offset()) + ", n)\n"; + } + } else { + // Table accessors + getter_code += " " + offset_prefix; + getter_code += " " + offset_prefix_2; + std::string field_getter = GenerateGetter(field->type(), "o"); + if (field->optional()) { + field_getter = "some(" + field_getter + ")"; + } + getter_code += " return " + field_getter + "\n"; + if (!field->optional()) { + getter_code += " return " + DefaultValue(field) + "\n"; + } + + if (IsScalar(base_type)) { + setter_code += " return self.tab.MutateSlot(" + + NumToString(field->offset()) + ", n)\n"; + } + } + code += getter_signature + getter_code; + if (IsScalar(base_type)) { code += setter_signature + setter_code; } + } else if (base_type == r::Array || base_type == r::Vector) { + const r::BaseType vector_base_type = field->type()->element(); + uint32_t element_size = field->type()->element_size(); + + if (vector_base_type == r::Obj || vector_base_type == r::Union || + field->type()->index() >= 0) { + RegisterImports(object, field, true); + } + + // Get vector length: + code += "func " + namer_.Method(field_name + "Length") + + "*(self: " + object_name + "): int = \n"; + code += " " + offset_prefix; + code += " " + offset_prefix_2; + code += " return self.tab.VectorLen(o)\n"; + + // Get single vector field: + code += "func " + namer_.Method(field_name) + "*(self: " + object_name + + ", j: int): " + GenerateType(field->type(), true) + " = \n"; + code += " " + offset_prefix; + code += " " + offset_prefix_2; + code += " var x = self.tab.Vector(o)\n"; + code += + " x += j.uoffset * " + NumToString(element_size) + ".uoffset\n"; + code += " return " + GenerateGetter(field->type(), "x", true) + "\n"; + + // Get entire vector: + code += "func " + namer_.Method(field_name) + "*(self: " + object_name + + "): " + GenerateType(field->type()) + " = \n"; + code += " let len = self." + field_name + "Length\n"; + code += " for i in countup(0, len - 1):\n"; + code += " result.add(self." + field_name + "(i))\n"; + + (void)IsSingleByte(vector_base_type); // unnused function warning + } + }); + + // Create all the builders + if (object->is_struct()) { + code += "proc " + namer_.Function(object_name + "Create") + + "*(self: var Builder"; + code += GenerateStructBuilderArgs(object); + code += "): uoffset =\n"; + code += AppendStructBuilderBody(object); + code += " return self.Offset()\n"; + } else { + // Table builders + code += "proc " + namer_.Function(object_name + "Start") + + "*(builder: var Builder) =\n"; + code += " builder.StartObject(" + NumToString(object->fields()->size()) + + ")\n"; + + ForAllFields(object, /*reverse=*/false, [&](const r::Field *field) { + if (field->deprecated()) { return; } + + const std::string field_name = namer_.Field(*field); + const std::string variable_name = namer_.Variable(*field); + const std::string variable_type = GenerateTypeBasic(field->type()); + + code += "proc " + namer_.Function(object_name + "Add" + field_name) + + "*(builder: var Builder, " + variable_name + ": " + + variable_type + ") =\n"; + code += " builder.Prepend" + GenerateMethod(field) + "Slot(" + + NumToString(field->id()) + ", " + variable_name + ", default(" + + variable_type + "))\n"; + + if (IsVector(field->type()->base_type())) { + code += "proc " + + namer_.Function(object_name + "Start" + field_name) + + "Vector*(builder: var Builder, numElems: uoffset) =\n"; + + const int32_t element_size = field->type()->element_size(); + int32_t alignment = element_size; + if (IsStruct(field->type(), /*use_element=*/true)) { + alignment = GetObjectByIndex(field->type()->index())->minalign(); + } + + code += " builder.StartVector(" + NumToString(element_size) + + ", numElems, " + NumToString(alignment) + ")\n"; + } + }); + + code += "proc " + namer_.Function(object_name + "End") + + "*(builder: var Builder): uoffset =\n"; + code += " return builder.EndObject()\n"; + } + EmitCodeBlock(code, object_name, ns, object->declaration_file()->str()); + } + + private: + void GenerateDocumentation( + const flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>> + *documentation, + std::string indent, std::string &code) const { + flatbuffers::ForAllDocumentation( + documentation, [&](const flatbuffers::String *str) { + code += indent + "# " + str->str() + "\n"; + }); + } + + std::string GenerateStructBuilderArgs(const r::Object *object, + std::string prefix = "") const { + std::string signature; + ForAllFields(object, /*reverse=*/false, [&](const r::Field *field) { + if (IsStructOrTable(field->type()->base_type())) { + const r::Object *field_object = GetObject(field->type()); + signature += GenerateStructBuilderArgs( + field_object, prefix + namer_.Variable(*field) + "_"); + } else { + signature += ", " + prefix + namer_.Variable(*field) + ": " + + GenerateType(field->type()); + } + }); + return signature; + } + + std::string AppendStructBuilderBody(const r::Object *object, + std::string prefix = "") const { + std::string code; + code += " self.Prep(" + NumToString(object->minalign()) + ", " + + NumToString(object->bytesize()) + ")\n"; + + // We need to reverse the order we iterate over, since we build the + // buffer backwards. + ForAllFields(object, /*reverse=*/true, [&](const r::Field *field) { + const int32_t num_padding_bytes = field->padding(); + if (num_padding_bytes) { + code += " self.Pad(" + NumToString(num_padding_bytes) + ")\n"; + } + if (IsStructOrTable(field->type()->base_type())) { + const r::Object *field_object = GetObject(field->type()); + code += AppendStructBuilderBody(field_object, + prefix + namer_.Variable(*field) + "_"); + } else { + code += " self.Prepend(" + prefix + namer_.Variable(*field) + ")\n"; + } + }); + + return code; + } + + std::string GenerateMethod(const r::Field *field) const { + const r::BaseType base_type = field->type()->base_type(); + if (IsStructOrTable(base_type)) { return "Struct"; } + return ""; + } + + std::string GenerateGetter(const r::Type *type, const std::string &offsetval, + bool element_type = false) const { + const r::BaseType base_type = + element_type ? type->element() : type->base_type(); + std::string offset = offsetval; + if (!element_type) { offset = "self.tab.Pos + " + offset; } + switch (base_type) { + case r::String: return "self.tab.String(" + offset + ")"; + case r::Union: return "self.tab.Union(" + offsetval + ")"; + case r::Obj: { + return GenerateType(type, element_type) + + "(tab: Vtable(Bytes: self.tab.Bytes, Pos: " + offset + "))"; + } + case r::Vector: return GenerateGetter(type, offsetval, true); + default: + const r::Enum *type_enum = GetEnum(type, element_type); + if (type_enum != nullptr) { + return GenerateType(type, element_type) + "(" + "Get[" + + GenerateType(base_type) + "](self.tab, " + offset + ")" + ")"; + } else { + return "Get[" + GenerateType(base_type) + "](self.tab, " + offset + + ")"; + } + } + } + + std::string Denamespace(const std::string &s, std::string &importns, + std::string &ns) const { + if (builtin_types.find(s) != builtin_types.end()) { return s; } + std::string type = namer_.Type(namer_.Denamespace(s, ns)); + importns = ns.empty() ? type : ns + "." + type; + std::replace(importns.begin(), importns.end(), '.', '_'); + return type; + } + + std::string Denamespace(const std::string &s, std::string &importns) const { + std::string ns; + return Denamespace(s, importns, ns); + } + + std::string Denamespace(const std::string &s) const { + std::string importns; + return Denamespace(s, importns); + } + + std::string GenerateType(const r::Type *type, bool element_type = false, + bool enum_inner = false) const { + const r::BaseType base_type = + element_type ? type->element() : type->base_type(); + if (IsScalar(base_type) && !enum_inner) { + const r::Enum *type_enum = GetEnum(type, element_type); + if (type_enum != nullptr) { + std::string importns; + std::string type_name = Denamespace(type_enum->name()->str(), importns); + return importns + "." + type_name; + } + } + if (IsScalar(base_type)) { return Denamespace(GenerateType(base_type)); } + switch (base_type) { + case r::String: return "string"; + case r::Vector: { + return "seq[" + GenerateType(type, true) + "]"; + } + case r::Union: return "Vtable"; + case r::Obj: { + const r::Object *type_obj = GetObject(type, element_type); + std::string importns; + std::string type_name = Denamespace(type_obj->name()->str(), importns); + if (type_obj == current_obj_) { + return type_name; + } else { + return importns + "." + type_name; + } + } + default: return "uoffset"; + } + } + + std::string GenerateTypeBasic(const r::Type *type, + bool element_type = false) const { + const r::BaseType base_type = + element_type ? type->element() : type->base_type(); + if (IsScalar(base_type)) { + return GenerateType(base_type); + } else { + return "uoffset"; + } + } + + std::string GenerateType(const r::BaseType base_type) const { + switch (base_type) { + case r::None: return "uint8"; + case r::UType: return "uint8"; + case r::Bool: return "bool"; + case r::Byte: return "int8"; + case r::UByte: return "uint8"; + case r::Short: return "int16"; + case r::UShort: return "uint16"; + case r::Int: return "int32"; + case r::UInt: return "uint32"; + case r::Long: return "int64"; + case r::ULong: return "uint64"; + case r::Float: return "float32"; + case r::Double: return "float64"; + case r::String: return "string"; + default: return r::EnumNameBaseType(base_type); + } + } + + std::string DefaultValue(const r::Field *field) const { + const r::BaseType base_type = field->type()->base_type(); + if (IsFloatingPoint(base_type)) { + if (field->default_real() != field->default_real()) { + return "NaN"; + } else if (field->default_real() == + std::numeric_limits<double>::infinity()) { + return "Inf"; + } else if (field->default_real() == + -std::numeric_limits<double>::infinity()) { + return "-Inf"; + } + return NumToString(field->default_real()); + } + if (IsBool(base_type)) { + return field->default_integer() ? "true" : "false"; + } + if (IsScalar(base_type)) { + const r::Enum *type_enum = GetEnum(field->type()); + if (type_enum != nullptr) { + return "type(result)(" + NumToString((field->default_integer())) + ")"; + } + return NumToString((field->default_integer())); + } + if (base_type == r::String) { return "\"\""; } + // represents offsets + return "0"; + } + + void StartCodeBlock(const reflection::Enum *enum_def) { + current_enum_ = enum_def; + current_obj_ = nullptr; + imports_.clear(); + } + + void StartCodeBlock(const reflection::Object *object) { + current_enum_ = nullptr; + current_obj_ = object; + imports_.clear(); + } + + std::vector<std::string> StringSplit(const std::string orig_str, + const std::string token) { + std::vector<std::string> result; + std::string str = orig_str; + while (str.size()) { + size_t index = str.find(token); + if (index != std::string::npos) { + result.push_back(str.substr(0, index)); + str = str.substr(index + token.size()); + if (str.size() == 0) result.push_back(str); + } else { + result.push_back(str); + str = ""; + } + } + return result; + } + + std::string GetRelativePathFromNamespace(const std::string &relative_to, + const std::string &str2) { + std::vector<std::string> relative_to_vec = StringSplit(relative_to, "."); + std::vector<std::string> str2_vec = StringSplit(str2, "."); + while (relative_to_vec.size() > 0 && str2_vec.size() > 0) { + if (relative_to_vec[0] == str2_vec[0]) { + relative_to_vec.erase(relative_to_vec.begin()); + str2_vec.erase(str2_vec.begin()); + } else { + break; + } + } + relative_to_vec.pop_back(); + for (size_t i = 0; i < relative_to_vec.size(); ++i) { + str2_vec.insert(str2_vec.begin(), std::string("..")); + } + + std::string new_path; + for (size_t i = 0; i < str2_vec.size(); ++i) { + new_path += str2_vec[i]; + if (i != str2_vec.size() - 1) { new_path += "/"; } + } + return new_path; + } + + void RegisterImports(const r::Object *object, const r::Field *field, + bool use_element = false) { + std::string importns; + std::string type_name; + + const r::BaseType type = + use_element ? field->type()->element() : field->type()->base_type(); + + if (IsStructOrTable(type)) { + const r::Object *object_def = GetObjectByIndex(field->type()->index()); + if (object_def == current_obj_) { return; } + std::string ns; + type_name = Denamespace(object_def->name()->str(), importns, ns); + type_name = ns.empty() ? type_name : ns + "." + type_name; + } else { + const r::Enum *enum_def = GetEnumByIndex(field->type()->index()); + if (enum_def == current_enum_) { return; } + std::string ns; + type_name = Denamespace(enum_def->name()->str(), importns, ns); + type_name = ns.empty() ? type_name : ns + "." + type_name; + } + + std::string import_path = + GetRelativePathFromNamespace(object->name()->str(), type_name); + std::replace(type_name.begin(), type_name.end(), '.', '_'); + RegisterImports(import_path, importns); + } + + void RegisterImports(const std::string &local_name, + const std::string &imports_name) { + imports_[local_name] = imports_name; + } + + void EmitCodeBlock(const std::string &code_block, const std::string &name, + const std::string &ns, const std::string &declaring_file) { + const std::string full_qualified_name = ns.empty() ? name : ns + "." + name; + + std::string code = "#[ " + full_qualified_name + "\n"; + code += + " Automatically generated by the FlatBuffers compiler, do not " + "modify.\n"; + code += " Or modify. I'm a message, not a cop.\n"; + code += "\n"; + code += " flatc version: " + flatc_version_ + "\n"; + code += "\n"; + code += " Declared by : " + declaring_file + "\n"; + if (schema_->root_table() != nullptr) { + const std::string root_type = schema_->root_table()->name()->str(); + const std::string root_file = + schema_->root_table()->declaration_file()->str(); + code += " Rooting type : " + root_type + " (" + root_file + ")\n"; + } + code += "]#\n\n"; + + if (!imports_.empty()) { + for (auto it = imports_.cbegin(); it != imports_.cend(); ++it) { + if (it->second.empty()) { + code += "import " + it->first + "\n"; + } else { + code += "import " + it->first + " as " + it->second + "\n"; + } + } + code += "\n"; + } + code += code_block; + + // Namespaces are '.' deliminted, so replace it with the path separator. + std::string path = ns; + + if (ns.empty()) { + path = "."; + } else { + std::replace(path.begin(), path.end(), '.', '/'); + } + + // TODO(derekbailey): figure out a save file without depending on util.h + EnsureDirExists(path); + const std::string file_name = path + "/" + namer_.File(name); + SaveFile(file_name.c_str(), code, false); + } + + std::unordered_set<std::string> keywords_; + std::map<std::string, std::string> imports_; + const r::Object *current_obj_; + const r::Enum *current_enum_; + const std::string flatc_version_; + const BfbsNamer namer_; +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewNimBfbsGenerator( + const std::string &flatc_version) { + return std::unique_ptr<NimBfbsGenerator>(new NimBfbsGenerator(flatc_version)); +} + +} // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/bfbs_gen_nim.h b/contrib/libs/flatbuffers/src/bfbs_gen_nim.h new file mode 100644 index 00000000000..39e8b218081 --- /dev/null +++ b/contrib/libs/flatbuffers/src/bfbs_gen_nim.h @@ -0,0 +1,33 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_BFBS_GEN_NIM_H_ +#define FLATBUFFERS_BFBS_GEN_NIM_H_ + +#include <memory> +#include <string> + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Nim Code generator. +std::unique_ptr<CodeGenerator> NewNimBfbsGenerator( + const std::string &flatc_version); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_BFBS_GEN_NIM_H_ diff --git a/contrib/libs/flatbuffers/src/bfbs_namer.h b/contrib/libs/flatbuffers/src/bfbs_namer.h new file mode 100644 index 00000000000..d197574cf71 --- /dev/null +++ b/contrib/libs/flatbuffers/src/bfbs_namer.h @@ -0,0 +1,51 @@ +#ifndef FLATBUFFERS_BFBS_NAMER +#define FLATBUFFERS_BFBS_NAMER + +#include "flatbuffers/reflection.h" +#include "namer.h" + +namespace flatbuffers { + +// Provides Namer capabilities to types defined in the flatbuffers reflection. +class BfbsNamer : public Namer { + public: + explicit BfbsNamer(Config config, std::set<std::string> keywords) + : Namer(config, std::move(keywords)) {} + + using Namer::Constant; + using Namer::Denamespace; + using Namer::Directories; + using Namer::Field; + using Namer::File; + using Namer::Function; + using Namer::Method; + using Namer::Namespace; + using Namer::NamespacedType; + using Namer::ObjectType; + using Namer::Type; + using Namer::Variable; + using Namer::Variant; + + template<typename T> + std::string Denamespace(T t, std::string &namespace_prefix, + const char delimiter = '.') const { + return Namer::Denamespace(t->name()->c_str(), namespace_prefix, delimiter); + } + + template<typename T> + std::string Denamespace(T t, const char delimiter = '.') const { + return Namer::Denamespace(t->name()->c_str(), delimiter); + } + + virtual std::string Field(const ::reflection::Field &f) const { + return Field(f.name()->str()); + } + + virtual std::string Variable(const ::reflection::Field &f) const { + return Variable(f.name()->str()); + } +}; + +} // namespace flatbuffers + +#endif // FLATBUFFERS_BFBS_NAMER diff --git a/contrib/libs/flatbuffers/src/binary_annotator.cpp b/contrib/libs/flatbuffers/src/binary_annotator.cpp new file mode 100644 index 00000000000..498a7bfc6a3 --- /dev/null +++ b/contrib/libs/flatbuffers/src/binary_annotator.cpp @@ -0,0 +1,1519 @@ +#include "binary_annotator.h" + +#include <algorithm> +#include <cstdint> +#include <iostream> +#include <limits> +#include <string> +#include <vector> + +#include "flatbuffers/base.h" +#include "flatbuffers/reflection.h" +#include "flatbuffers/util.h" +#include "flatbuffers/verifier.h" + +namespace flatbuffers { +namespace { + +static bool BinaryRegionSort(const BinaryRegion &a, const BinaryRegion &b) { + return a.offset < b.offset; +} + +static void SetError(BinaryRegionComment &comment, BinaryRegionStatus status, + std::string message = "") { + comment.status = status; + comment.status_message = message; +} + +static BinaryRegion MakeBinaryRegion( + const uint64_t offset = 0, const uint64_t length = 0, + const BinaryRegionType type = BinaryRegionType::Unknown, + const uint64_t array_length = 0, const uint64_t points_to_offset = 0, + BinaryRegionComment comment = {}) { + BinaryRegion region; + region.offset = offset; + region.length = length; + region.type = type; + region.array_length = array_length; + region.points_to_offset = points_to_offset; + region.comment = std::move(comment); + return region; +} + +static BinarySection MakeBinarySection(const std::string &name, + const BinarySectionType type, + std::vector<BinaryRegion> regions) { + BinarySection section; + section.name = name; + section.type = type; + section.regions = std::move(regions); + return section; +} + +static BinarySection MakeSingleRegionBinarySection(const std::string &name, + const BinarySectionType type, + const BinaryRegion ®ion) { + std::vector<BinaryRegion> regions; + regions.push_back(region); + return MakeBinarySection(name, type, std::move(regions)); +} + +static bool IsNonZeroRegion(const uint64_t offset, const uint64_t length, + const uint8_t *const binary) { + for (uint64_t i = offset; i < offset + length; ++i) { + if (binary[i] != 0) { return true; } + } + return false; +} + +static bool IsPrintableRegion(const uint64_t offset, const uint64_t length, + const uint8_t *const binary) { + for (uint64_t i = offset; i < offset + length; ++i) { + if (!isprint(binary[i])) { return false; } + } + return true; +} + +static BinarySection GenerateMissingSection(const uint64_t offset, + const uint64_t length, + const uint8_t *const binary) { + std::vector<BinaryRegion> regions; + + // Check if the region is all zeros or not, as that can tell us if it is + // padding or not. + if (IsNonZeroRegion(offset, length, binary)) { + // Some of the padding bytes are non-zero, so this might be an unknown + // section of the binary. + // TODO(dbaileychess): We could be a bit smarter with different sized + // alignments. For now, the 8 byte check encompasses all the smaller + // alignments. + BinaryRegionComment comment; + comment.type = BinaryRegionCommentType::Unknown; + if (length >= 8) { + SetError(comment, BinaryRegionStatus::WARN_NO_REFERENCES); + } else { + SetError(comment, BinaryRegionStatus::WARN_CORRUPTED_PADDING); + } + + regions.push_back(MakeBinaryRegion(offset, length * sizeof(uint8_t), + BinaryRegionType::Unknown, length, 0, + comment)); + + return MakeBinarySection("no known references", BinarySectionType::Unknown, + std::move(regions)); + } + + BinaryRegionComment comment; + comment.type = BinaryRegionCommentType::Padding; + if (length >= 8) { + SetError(comment, BinaryRegionStatus::WARN_PADDING_LENGTH); + } + + // This region is most likely padding. + regions.push_back(MakeBinaryRegion(offset, length * sizeof(uint8_t), + BinaryRegionType::Uint8, length, 0, + comment)); + + return MakeBinarySection("", BinarySectionType::Padding, std::move(regions)); +} + +} // namespace + +std::map<uint64_t, BinarySection> BinaryAnnotator::Annotate() { + flatbuffers::Verifier verifier(bfbs_, static_cast<size_t>(bfbs_length_)); + + if ((is_size_prefixed_ && + !reflection::VerifySizePrefixedSchemaBuffer(verifier)) || + !reflection::VerifySchemaBuffer(verifier)) { + return {}; + } + + // The binary is too short to read as a flatbuffers. + if (binary_length_ < FLATBUFFERS_MIN_BUFFER_SIZE) { return {}; } + + // Make sure we start with a clean slate. + vtables_.clear(); + sections_.clear(); + + // First parse the header region which always start at offset 0. + // The returned offset will point to the root_table location. + const uint64_t root_table_offset = BuildHeader(0); + + if (IsValidOffset(root_table_offset)) { + // Build the root table, and all else will be referenced from it. + BuildTable(root_table_offset, BinarySectionType::RootTable, + schema_->root_table()); + } + + // Now that all the sections are built, make sure the binary sections are + // contiguous. + FixMissingRegions(); + + // Then scan the area between BinarySections insert padding sections that are + // implied. + FixMissingSections(); + + return sections_; +} + +uint64_t BinaryAnnotator::BuildHeader(const uint64_t header_offset) { + uint64_t offset = header_offset; + std::vector<BinaryRegion> regions; + + // If this binary is a size prefixed one, attempt to parse the size. + if (is_size_prefixed_) { + BinaryRegionComment prefix_length_comment; + prefix_length_comment.type = BinaryRegionCommentType::SizePrefix; + + bool has_prefix_value = false; + const auto prefix_length = ReadScalar<uoffset64_t>(offset); + if (*prefix_length <= binary_length_) { + regions.push_back(MakeBinaryRegion(offset, sizeof(uoffset64_t), + BinaryRegionType::Uint64, 0, 0, + prefix_length_comment)); + offset += sizeof(uoffset64_t); + has_prefix_value = true; + } + + if (!has_prefix_value) { + const auto prefix_length = ReadScalar<uoffset_t>(offset); + if (*prefix_length <= binary_length_) { + regions.push_back(MakeBinaryRegion(offset, sizeof(uoffset_t), + BinaryRegionType::Uint32, 0, 0, + prefix_length_comment)); + offset += sizeof(uoffset_t); + has_prefix_value = true; + } + } + + if (!has_prefix_value) { + SetError(prefix_length_comment, BinaryRegionStatus::ERROR); + } + } + + const auto root_table_offset = ReadScalar<uint32_t>(offset); + + if (!root_table_offset.has_value()) { + // This shouldn't occur, since we validate the min size of the buffer + // before. But for completion sake, we shouldn't read passed the binary end. + return std::numeric_limits<uint64_t>::max(); + } + + const auto root_table_loc = offset + *root_table_offset; + + BinaryRegionComment root_offset_comment; + root_offset_comment.type = BinaryRegionCommentType::RootTableOffset; + root_offset_comment.name = schema_->root_table()->name()->str(); + + if (!IsValidOffset(root_table_loc)) { + SetError(root_offset_comment, + BinaryRegionStatus::ERROR_OFFSET_OUT_OF_BINARY); + } + + regions.push_back(MakeBinaryRegion(offset, sizeof(uint32_t), + BinaryRegionType::UOffset, 0, + root_table_loc, root_offset_comment)); + offset += sizeof(uint32_t); + + if (IsValidRead(offset, flatbuffers::kFileIdentifierLength) && + IsPrintableRegion(offset, flatbuffers::kFileIdentifierLength, binary_)) { + BinaryRegionComment comment; + comment.type = BinaryRegionCommentType::FileIdentifier; + // Check if the file identifier region has non-zero data, and assume its + // the file identifier. Otherwise, it will get filled in with padding + // later. + regions.push_back(MakeBinaryRegion( + offset, flatbuffers::kFileIdentifierLength * sizeof(uint8_t), + BinaryRegionType::Char, flatbuffers::kFileIdentifierLength, 0, + comment)); + } + + AddSection(header_offset, MakeBinarySection("", BinarySectionType::Header, + std::move(regions))); + + return root_table_loc; +} + +BinaryAnnotator::VTable *BinaryAnnotator::GetOrBuildVTable( + const uint64_t vtable_offset, const reflection::Object *const table, + const uint64_t offset_of_referring_table) { + // Get a list of vtables (if any) already defined at this offset. + std::list<VTable> &vtables = vtables_[vtable_offset]; + + // See if this vtable for the table type has been generated before. + for (VTable &vtable : vtables) { + if (vtable.referring_table == table) { return &vtable; } + } + + // If we are trying to make a new vtable and it is already encompassed by + // another binary section, something is corrupted. + if (vtables.empty() && ContainsSection(vtable_offset)) { return nullptr; } + + const std::string referring_table_name = table->name()->str(); + + BinaryRegionComment vtable_size_comment; + vtable_size_comment.type = BinaryRegionCommentType::VTableSize; + + const auto vtable_length = ReadScalar<uint16_t>(vtable_offset); + if (!vtable_length.has_value()) { + const uint64_t remaining = RemainingBytes(vtable_offset); + + SetError(vtable_size_comment, BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, + "2"); + + AddSection(vtable_offset, + MakeSingleRegionBinarySection( + referring_table_name, BinarySectionType::VTable, + MakeBinaryRegion(vtable_offset, remaining, + BinaryRegionType::Unknown, remaining, 0, + vtable_size_comment))); + return nullptr; + } + + // Vtables start with the size of the vtable + const uint16_t vtable_size = vtable_length.value(); + + if (!IsValidOffset(vtable_offset + vtable_size - 1)) { + SetError(vtable_size_comment, BinaryRegionStatus::ERROR_LENGTH_TOO_LONG); + // The vtable_size points to off the end of the binary. + AddSection(vtable_offset, + MakeSingleRegionBinarySection( + referring_table_name, BinarySectionType::VTable, + MakeBinaryRegion(vtable_offset, sizeof(uint16_t), + BinaryRegionType::Uint16, 0, 0, + vtable_size_comment))); + + return nullptr; + } else if (vtable_size < 2 * sizeof(uint16_t)) { + SetError(vtable_size_comment, BinaryRegionStatus::ERROR_LENGTH_TOO_SHORT, + "4"); + // The size includes itself and the table size which are both uint16_t. + AddSection(vtable_offset, + MakeSingleRegionBinarySection( + referring_table_name, BinarySectionType::VTable, + MakeBinaryRegion(vtable_offset, sizeof(uint16_t), + BinaryRegionType::Uint16, 0, 0, + vtable_size_comment))); + return nullptr; + } + + std::vector<BinaryRegion> regions; + + regions.push_back(MakeBinaryRegion(vtable_offset, sizeof(uint16_t), + BinaryRegionType::Uint16, 0, 0, + vtable_size_comment)); + uint64_t offset = vtable_offset + sizeof(uint16_t); + + BinaryRegionComment ref_table_len_comment; + ref_table_len_comment.type = + BinaryRegionCommentType::VTableRefferingTableLength; + + // Ensure we can read the next uint16_t field, which is the size of the + // referring table. + const auto table_length = ReadScalar<uint16_t>(offset); + + if (!table_length.has_value()) { + const uint64_t remaining = RemainingBytes(offset); + SetError(ref_table_len_comment, BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, + "2"); + + AddSection(offset, MakeSingleRegionBinarySection( + referring_table_name, BinarySectionType::VTable, + MakeBinaryRegion( + offset, remaining, BinaryRegionType::Unknown, + remaining, 0, ref_table_len_comment))); + return nullptr; + } + + // Then they have the size of the table they reference. + const uint16_t table_size = table_length.value(); + + if (!IsValidOffset(offset_of_referring_table + table_size - 1)) { + SetError(ref_table_len_comment, BinaryRegionStatus::ERROR_LENGTH_TOO_LONG); + } else if (table_size < 4) { + SetError(ref_table_len_comment, BinaryRegionStatus::ERROR_LENGTH_TOO_SHORT, + "4"); + } + + regions.push_back(MakeBinaryRegion(offset, sizeof(uint16_t), + BinaryRegionType::Uint16, 0, 0, + ref_table_len_comment)); + offset += sizeof(uint16_t); + + const uint64_t offset_start = offset; + + // A mapping between field (and its id) to the relative offset (uin16_t) from + // the start of the table. + std::map<uint16_t, VTable::Entry> fields; + + // Counter for determining if the binary has more vtable entries than the + // schema provided. This can occur if the binary was created at a newer schema + // version and is being processed with an older one. + uint16_t fields_processed = 0; + + // Loop over all the fields. + ForAllFields(table, /*reverse=*/false, [&](const reflection::Field *field) { + const uint64_t field_offset = offset_start + field->id() * sizeof(uint16_t); + + if (field_offset >= vtable_offset + vtable_size) { + // This field_offset is too large for this vtable, so it must come from a + // newer schema than the binary was create with or the binary writer did + // not write it. For either case, it is safe to ignore. + + // TODO(dbaileychess): We could show which fields are not set an their + // default values if we want. We just need a way to make it obvious that + // it isn't part of the buffer. + return; + } + + BinaryRegionComment field_comment; + field_comment.type = BinaryRegionCommentType::VTableFieldOffset; + field_comment.name = std::string(field->name()->c_str()) + + "` (id: " + std::to_string(field->id()) + ")"; + + const auto offset_from_table = ReadScalar<uint16_t>(field_offset); + + if (!offset_from_table.has_value()) { + const uint64_t remaining = RemainingBytes(field_offset); + + SetError(field_comment, BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, "2"); + regions.push_back(MakeBinaryRegion(field_offset, remaining, + BinaryRegionType::Unknown, remaining, + 0, field_comment)); + + return; + } + + if (!IsValidOffset(offset_of_referring_table + offset_from_table.value() - + 1)) { + SetError(field_comment, BinaryRegionStatus::ERROR_OFFSET_OUT_OF_BINARY); + regions.push_back(MakeBinaryRegion(field_offset, sizeof(uint16_t), + BinaryRegionType::VOffset, 0, 0, + field_comment)); + return; + } + + VTable::Entry entry; + entry.field = field; + entry.offset_from_table = offset_from_table.value(); + fields.insert(std::make_pair(field->id(), entry)); + + std::string default_label; + if (offset_from_table.value() == 0) { + // Not present, so could be default or be optional. + if (field->required()) { + SetError(field_comment, + BinaryRegionStatus::ERROR_REQUIRED_FIELD_NOT_PRESENT); + // If this is a required field, make it known this is an error. + regions.push_back(MakeBinaryRegion(field_offset, sizeof(uint16_t), + BinaryRegionType::VOffset, 0, 0, + field_comment)); + return; + } else { + // Its an optional field, so get the default value and interpret and + // provided an annotation for it. + if (IsScalar(field->type()->base_type())) { + default_label += "<defaults to "; + default_label += IsFloat(field->type()->base_type()) + ? std::to_string(field->default_real()) + : std::to_string(field->default_integer()); + default_label += "> ("; + } else { + default_label += "<null> ("; + } + default_label += + reflection::EnumNameBaseType(field->type()->base_type()); + default_label += ")"; + } + } + field_comment.default_value = default_label; + + regions.push_back(MakeBinaryRegion(field_offset, sizeof(uint16_t), + BinaryRegionType::VOffset, 0, 0, + field_comment)); + + fields_processed++; + }); + + // Check if we covered all the expectant fields. If not, we need to add them + // as unknown fields. + uint16_t expectant_vtable_fields = + (vtable_size - sizeof(uint16_t) - sizeof(uint16_t)) / sizeof(uint16_t); + + // Prevent a bad binary from declaring a really large vtable_size, that we can + // not independently verify. + expectant_vtable_fields = std::min( + static_cast<uint16_t>(fields_processed * 3), expectant_vtable_fields); + + for (uint16_t id = fields_processed; id < expectant_vtable_fields; ++id) { + const uint64_t field_offset = offset_start + id * sizeof(uint16_t); + + const auto offset_from_table = ReadScalar<uint16_t>(field_offset); + + BinaryRegionComment field_comment; + field_comment.type = BinaryRegionCommentType::VTableUnknownFieldOffset; + field_comment.index = id; + + if (!offset_from_table.has_value()) { + const uint64_t remaining = RemainingBytes(field_offset); + SetError(field_comment, BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, "2"); + regions.push_back(MakeBinaryRegion(field_offset, remaining, + BinaryRegionType::Unknown, remaining, + 0, field_comment)); + continue; + } + + VTable::Entry entry; + entry.field = nullptr; // No field to reference. + entry.offset_from_table = offset_from_table.value(); + fields.insert(std::make_pair(id, entry)); + + regions.push_back(MakeBinaryRegion(field_offset, sizeof(uint16_t), + BinaryRegionType::VOffset, 0, 0, + field_comment)); + } + + // If we have never added this vtable before record the Binary section. + if (vtables.empty()) { + sections_[vtable_offset] = MakeBinarySection( + referring_table_name, BinarySectionType::VTable, std::move(regions)); + } else { + // Add the current table name to the name of the section. + sections_[vtable_offset].name += ", " + referring_table_name; + } + + VTable vtable; + vtable.referring_table = table; + vtable.fields = std::move(fields); + vtable.table_size = table_size; + vtable.vtable_size = vtable_size; + + // Add this vtable to the collection of vtables at this offset. + vtables.push_back(std::move(vtable)); + + // Return the vtable we just added. + return &vtables.back(); +} + +void BinaryAnnotator::BuildTable(const uint64_t table_offset, + const BinarySectionType type, + const reflection::Object *const table) { + if (ContainsSection(table_offset)) { return; } + + BinaryRegionComment vtable_offset_comment; + vtable_offset_comment.type = BinaryRegionCommentType::TableVTableOffset; + + const auto vtable_soffset = ReadScalar<int32_t>(table_offset); + + if (!vtable_soffset.has_value()) { + const uint64_t remaining = RemainingBytes(table_offset); + SetError(vtable_offset_comment, BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, + "4"); + + AddSection( + table_offset, + MakeSingleRegionBinarySection( + table->name()->str(), type, + MakeBinaryRegion(table_offset, remaining, BinaryRegionType::Unknown, + remaining, 0, vtable_offset_comment))); + + // If there aren't enough bytes left to read the vtable offset, there is + // nothing we can do. + return; + } + + // Tables start with the vtable + const uint64_t vtable_offset = table_offset - vtable_soffset.value(); + + if (!IsValidOffset(vtable_offset)) { + SetError(vtable_offset_comment, + BinaryRegionStatus::ERROR_OFFSET_OUT_OF_BINARY); + + AddSection(table_offset, + MakeSingleRegionBinarySection( + table->name()->str(), type, + MakeBinaryRegion(table_offset, sizeof(int32_t), + BinaryRegionType::SOffset, 0, vtable_offset, + vtable_offset_comment))); + + // There isn't much to do with an invalid vtable offset, as we won't be able + // to intepret the rest of the table fields. + return; + } + + std::vector<BinaryRegion> regions; + regions.push_back(MakeBinaryRegion(table_offset, sizeof(int32_t), + BinaryRegionType::SOffset, 0, + vtable_offset, vtable_offset_comment)); + + // Parse the vtable first so we know what the rest of the fields in the table + // are. + const VTable *const vtable = + GetOrBuildVTable(vtable_offset, table, table_offset); + + if (vtable == nullptr) { + // There is no valid vtable for this table, so we cannot process the rest of + // the table entries. + return; + } + + // This is the size and length of this table. + const uint16_t table_size = vtable->table_size; + uint64_t table_end_offset = table_offset + table_size; + + if (!IsValidOffset(table_end_offset - 1)) { + // We already validated the table size in BuildVTable, but we have to make + // sure we don't use a bad value here. + table_end_offset = binary_length_; + } + + // We need to iterate over the vtable fields by their offset in the binary, + // not by their IDs. So copy them over to another vector that we can sort on + // the offset_from_table property. + std::vector<VTable::Entry> fields; + for (const auto &vtable_field : vtable->fields) { + fields.push_back(vtable_field.second); + } + + std::stable_sort(fields.begin(), fields.end(), + [](const VTable::Entry &a, const VTable::Entry &b) { + return a.offset_from_table < b.offset_from_table; + }); + + // Iterate over all the fields by order of their offset. + for (size_t i = 0; i < fields.size(); ++i) { + const reflection::Field *field = fields[i].field; + const uint16_t offset_from_table = fields[i].offset_from_table; + + if (offset_from_table == 0) { + // Skip non-present fields. + continue; + } + + // The field offsets are relative to the start of the table. + const uint64_t field_offset = table_offset + offset_from_table; + + if (!IsValidOffset(field_offset)) { + // The field offset is larger than the binary, nothing we can do. + continue; + } + + // We have a vtable entry for a non-existant field, that means its a binary + // generated by a newer schema than we are currently processing. + if (field == nullptr) { + // Calculate the length of this unknown field. + const uint64_t unknown_field_length = + // Check if there is another unknown field after this one. + ((i + 1 < fields.size()) + ? table_offset + fields[i + 1].offset_from_table + // Otherwise use the known end of the table. + : table_end_offset) - + field_offset; + + if (unknown_field_length == 0) { continue; } + + std::string hint; + + if (unknown_field_length == 4) { + const auto relative_offset = ReadScalar<uint32_t>(field_offset); + if (relative_offset.has_value()) { + // The field is 4 in length, so it could be an offset? Provide a hint. + hint += "<possibly an offset? Check Loc: +0x"; + hint += ToHex(field_offset + relative_offset.value()); + hint += ">"; + } + } + + BinaryRegionComment unknown_field_comment; + unknown_field_comment.type = BinaryRegionCommentType::TableUnknownField; + + if (!IsValidRead(field_offset, unknown_field_length)) { + const uint64_t remaining = RemainingBytes(field_offset); + + SetError(unknown_field_comment, + BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, + std::to_string(unknown_field_length)); + + regions.push_back(MakeBinaryRegion(field_offset, remaining, + BinaryRegionType::Unknown, remaining, + 0, unknown_field_comment)); + continue; + } + + unknown_field_comment.default_value = hint; + + regions.push_back(MakeBinaryRegion( + field_offset, unknown_field_length, BinaryRegionType::Unknown, + unknown_field_length, 0, unknown_field_comment)); + continue; + } + + if (IsScalar(field->type()->base_type())) { + // These are the raw values store in the table. + const uint64_t type_size = GetTypeSize(field->type()->base_type()); + const BinaryRegionType region_type = + GetRegionType(field->type()->base_type()); + + BinaryRegionComment scalar_field_comment; + scalar_field_comment.type = BinaryRegionCommentType::TableField; + scalar_field_comment.name = + std::string(field->name()->c_str()) + "` (" + + reflection::EnumNameBaseType(field->type()->base_type()) + ")"; + + if (!IsValidRead(field_offset, type_size)) { + const uint64_t remaining = RemainingBytes(field_offset); + SetError(scalar_field_comment, + BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, + std::to_string(type_size)); + + regions.push_back(MakeBinaryRegion(field_offset, remaining, + BinaryRegionType::Unknown, remaining, + 0, scalar_field_comment)); + continue; + } + + if (IsUnionType(field)) { + // This is a type for a union. Validate the value + const auto enum_value = ReadScalar<uint8_t>(field_offset); + + // This should always have a value, due to the IsValidRead check above. + if (!IsValidUnionValue(field, enum_value.value())) { + SetError(scalar_field_comment, + BinaryRegionStatus::ERROR_INVALID_UNION_TYPE); + + regions.push_back(MakeBinaryRegion(field_offset, type_size, + region_type, 0, 0, + scalar_field_comment)); + continue; + } + } + + regions.push_back(MakeBinaryRegion(field_offset, type_size, region_type, + 0, 0, scalar_field_comment)); + continue; + } + + // Read the offset + uint64_t offset = 0; + uint64_t length = sizeof(uint32_t); + BinaryRegionType region_type = BinaryRegionType::UOffset; + + if (field->offset64()) { + length = sizeof(uint64_t); + region_type = BinaryRegionType::UOffset64; + offset = ReadScalar<uint64_t>(field_offset).value_or(0); + } else { + offset = ReadScalar<uint32_t>(field_offset).value_or(0); + } + // const auto offset_from_field = ReadScalar<uint32_t>(field_offset); + uint64_t offset_of_next_item = 0; + BinaryRegionComment offset_field_comment; + offset_field_comment.type = BinaryRegionCommentType::TableOffsetField; + offset_field_comment.name = field->name()->c_str(); + const std::string offset_prefix = + "offset to field `" + std::string(field->name()->c_str()) + "`"; + + // Validate any field that isn't inline (i.e., non-structs). + if (!IsInlineField(field)) { + if (offset == 0) { + const uint64_t remaining = RemainingBytes(field_offset); + + SetError(offset_field_comment, + BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, "4"); + + regions.push_back(MakeBinaryRegion(field_offset, remaining, + BinaryRegionType::Unknown, remaining, + 0, offset_field_comment)); + continue; + } + + offset_of_next_item = field_offset + offset; + + if (!IsValidOffset(offset_of_next_item)) { + SetError(offset_field_comment, + BinaryRegionStatus::ERROR_OFFSET_OUT_OF_BINARY); + regions.push_back(MakeBinaryRegion(field_offset, length, region_type, 0, + offset_of_next_item, + offset_field_comment)); + continue; + } + } + + switch (field->type()->base_type()) { + case reflection::BaseType::Obj: { + const reflection::Object *next_object = + schema_->objects()->Get(field->type()->index()); + + if (next_object->is_struct()) { + // Structs are stored inline. + BuildStruct(field_offset, regions, field->name()->c_str(), + next_object); + } else { + offset_field_comment.default_value = "(table)"; + + regions.push_back(MakeBinaryRegion(field_offset, length, region_type, + 0, offset_of_next_item, + offset_field_comment)); + + BuildTable(offset_of_next_item, BinarySectionType::Table, + next_object); + } + } break; + + case reflection::BaseType::String: { + offset_field_comment.default_value = "(string)"; + regions.push_back(MakeBinaryRegion(field_offset, length, region_type, 0, + offset_of_next_item, + offset_field_comment)); + BuildString(offset_of_next_item, table, field); + } break; + + case reflection::BaseType::Vector: { + offset_field_comment.default_value = "(vector)"; + regions.push_back(MakeBinaryRegion(field_offset, length, region_type, 0, + offset_of_next_item, + offset_field_comment)); + BuildVector(offset_of_next_item, table, field, table_offset, + vtable->fields); + } break; + case reflection::BaseType::Vector64: { + offset_field_comment.default_value = "(vector64)"; + regions.push_back(MakeBinaryRegion(field_offset, length, region_type, 0, + offset_of_next_item, + offset_field_comment)); + BuildVector(offset_of_next_item, table, field, table_offset, + vtable->fields); + } break; + + case reflection::BaseType::Union: { + const uint64_t union_offset = offset_of_next_item; + + // The union type field is always one less than the union itself. + const uint16_t union_type_id = field->id() - 1; + + auto vtable_field = vtable->fields.find(union_type_id); + if (vtable_field == vtable->fields.end()) { + // TODO(dbaileychess): need to capture this error condition. + break; + } + offset_field_comment.default_value = "(union)"; + + const uint64_t type_offset = + table_offset + vtable_field->second.offset_from_table; + + const auto realized_type = ReadScalar<uint8_t>(type_offset); + if (!realized_type.has_value()) { + const uint64_t remaining = RemainingBytes(type_offset); + SetError(offset_field_comment, + BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, "1"); + regions.push_back(MakeBinaryRegion( + type_offset, remaining, BinaryRegionType::Unknown, remaining, 0, + offset_field_comment)); + continue; + } + + if (!IsValidUnionValue(field, realized_type.value())) { + // We already export an error in the union type field, so just skip + // building the union itself and it will default to an unreference + // Binary section. + continue; + } + + const std::string enum_type = + BuildUnion(union_offset, realized_type.value(), field); + + offset_field_comment.default_value = + "(union of type `" + enum_type + "`)"; + + regions.push_back(MakeBinaryRegion(field_offset, length, region_type, 0, + union_offset, offset_field_comment)); + + } break; + + default: break; + } + } + + // Handle the case where there is padding after the last known binary + // region. Calculate where we left off towards the expected end of the + // table. + const uint64_t i = regions.back().offset + regions.back().length + 1; + + if (i < table_end_offset) { + const uint64_t pad_bytes = table_end_offset - i + 1; + + BinaryRegionComment padding_comment; + padding_comment.type = BinaryRegionCommentType::Padding; + + regions.push_back(MakeBinaryRegion(i - 1, pad_bytes * sizeof(uint8_t), + BinaryRegionType::Uint8, pad_bytes, 0, + padding_comment)); + } + + AddSection(table_offset, + MakeBinarySection(table->name()->str(), type, std::move(regions))); +} + +uint64_t BinaryAnnotator::BuildStruct(const uint64_t struct_offset, + std::vector<BinaryRegion> ®ions, + const std::string referring_field_name, + const reflection::Object *const object) { + if (!object->is_struct()) { return struct_offset; } + uint64_t offset = struct_offset; + + // Loop over all the fields in increasing order + ForAllFields(object, /*reverse=*/false, [&](const reflection::Field *field) { + if (IsScalar(field->type()->base_type())) { + // Structure Field value + const uint64_t type_size = GetTypeSize(field->type()->base_type()); + const BinaryRegionType region_type = + GetRegionType(field->type()->base_type()); + + BinaryRegionComment comment; + comment.type = BinaryRegionCommentType::StructField; + comment.name = referring_field_name + "." + field->name()->str(); + comment.default_value = "of '" + object->name()->str() + "' (" + + std::string(reflection::EnumNameBaseType( + field->type()->base_type())) + + ")"; + + if (!IsValidRead(offset, type_size)) { + const uint64_t remaining = RemainingBytes(offset); + SetError(comment, BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, + std::to_string(type_size)); + regions.push_back(MakeBinaryRegion(offset, remaining, + BinaryRegionType::Unknown, remaining, + 0, comment)); + + // TODO(dbaileychess): Should I bail out here? This sets offset to the + // end of the binary. So all other reads in the loop should fail. + offset += remaining; + return; + } + + regions.push_back( + MakeBinaryRegion(offset, type_size, region_type, 0, 0, comment)); + offset += type_size; + } else if (field->type()->base_type() == reflection::BaseType::Obj) { + // Structs are stored inline, even when nested. + offset = BuildStruct(offset, regions, + referring_field_name + "." + field->name()->str(), + schema_->objects()->Get(field->type()->index())); + } else if (field->type()->base_type() == reflection::BaseType::Array) { + const bool is_scalar = IsScalar(field->type()->element()); + const uint64_t type_size = GetTypeSize(field->type()->element()); + const BinaryRegionType region_type = + GetRegionType(field->type()->element()); + + // Arrays are just repeated structures. + for (uint16_t i = 0; i < field->type()->fixed_length(); ++i) { + if (is_scalar) { + BinaryRegionComment array_comment; + array_comment.type = BinaryRegionCommentType::ArrayField; + array_comment.name = + referring_field_name + "." + field->name()->str(); + array_comment.index = i; + array_comment.default_value = + "of '" + object->name()->str() + "' (" + + std::string( + reflection::EnumNameBaseType(field->type()->element())) + + ")"; + + if (!IsValidRead(offset, type_size)) { + const uint64_t remaining = RemainingBytes(offset); + + SetError(array_comment, BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, + std::to_string(type_size)); + + regions.push_back(MakeBinaryRegion(offset, remaining, + BinaryRegionType::Unknown, + remaining, 0, array_comment)); + + // TODO(dbaileychess): Should I bail out here? This sets offset to + // the end of the binary. So all other reads in the loop should + // fail. + offset += remaining; + break; + } + + regions.push_back(MakeBinaryRegion(offset, type_size, region_type, 0, + 0, array_comment)); + + offset += type_size; + } else { + // Array of Structs. + // + // TODO(dbaileychess): This works, but the comments on the fields lose + // some context. Need to figure a way how to plumb the nested arrays + // comments together that isn't too confusing. + offset = + BuildStruct(offset, regions, + referring_field_name + "." + field->name()->str(), + schema_->objects()->Get(field->type()->index())); + } + } + } + + // Insert any padding after this field. + const uint16_t padding = field->padding(); + if (padding > 0 && IsValidOffset(offset + padding)) { + BinaryRegionComment padding_comment; + padding_comment.type = BinaryRegionCommentType::Padding; + + regions.push_back(MakeBinaryRegion(offset, padding, + BinaryRegionType::Uint8, padding, 0, + padding_comment)); + offset += padding; + } + }); + + return offset; +} + +void BinaryAnnotator::BuildString(const uint64_t string_offset, + const reflection::Object *const table, + const reflection::Field *const field) { + // Check if we have already generated this string section, and this is a + // shared string instance. + if (ContainsSection(string_offset)) { return; } + + std::vector<BinaryRegion> regions; + const auto string_length = ReadScalar<uint32_t>(string_offset); + + BinaryRegionComment string_length_comment; + string_length_comment.type = BinaryRegionCommentType::StringLength; + + if (!string_length.has_value()) { + const uint64_t remaining = RemainingBytes(string_offset); + + SetError(string_length_comment, BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, + "4"); + + regions.push_back(MakeBinaryRegion(string_offset, remaining, + BinaryRegionType::Unknown, remaining, 0, + string_length_comment)); + + } else { + const uint32_t string_size = string_length.value(); + const uint64_t string_end = + string_offset + sizeof(uint32_t) + string_size + sizeof(char); + + if (!IsValidOffset(string_end - 1)) { + SetError(string_length_comment, + BinaryRegionStatus::ERROR_LENGTH_TOO_LONG); + + regions.push_back(MakeBinaryRegion(string_offset, sizeof(uint32_t), + BinaryRegionType::Uint32, 0, 0, + string_length_comment)); + } else { + regions.push_back(MakeBinaryRegion(string_offset, sizeof(uint32_t), + BinaryRegionType::Uint32, 0, 0, + string_length_comment)); + + BinaryRegionComment string_comment; + string_comment.type = BinaryRegionCommentType::StringValue; + + regions.push_back(MakeBinaryRegion(string_offset + sizeof(uint32_t), + string_size, BinaryRegionType::Char, + string_size, 0, string_comment)); + + BinaryRegionComment string_terminator_comment; + string_terminator_comment.type = + BinaryRegionCommentType::StringTerminator; + + regions.push_back(MakeBinaryRegion( + string_offset + sizeof(uint32_t) + string_size, sizeof(char), + BinaryRegionType::Char, 0, 0, string_terminator_comment)); + } + } + + AddSection(string_offset, + MakeBinarySection(std::string(table->name()->c_str()) + "." + + field->name()->c_str(), + BinarySectionType::String, std::move(regions))); +} + +void BinaryAnnotator::BuildVector( + const uint64_t vector_offset, const reflection::Object *const table, + const reflection::Field *const field, const uint64_t parent_table_offset, + const std::map<uint16_t, VTable::Entry> vtable_fields) { + if (ContainsSection(vector_offset)) { return; } + + BinaryRegionComment vector_length_comment; + vector_length_comment.type = BinaryRegionCommentType::VectorLength; + + const bool is_64_bit_vector = + field->type()->base_type() == reflection::BaseType::Vector64; + + flatbuffers::Optional<uint64_t> vector_length; + uint32_t vector_length_size_type = 0; + BinaryRegionType region_type = BinaryRegionType::Uint32; + BinarySectionType section_type = BinarySectionType::Vector; + + if (is_64_bit_vector) { + auto v = ReadScalar<uint64_t>(vector_offset); + if (v.has_value()) { vector_length = v.value(); } + vector_length_size_type = sizeof(uint64_t); + region_type = BinaryRegionType::Uint64; + section_type = BinarySectionType::Vector64; + } else { + auto v = ReadScalar<uint32_t>(vector_offset); + if (v.has_value()) { vector_length = v.value(); } + vector_length_size_type = sizeof(uint32_t); + region_type = BinaryRegionType::Uint32; + section_type = BinarySectionType::Vector; + } + + if (!vector_length.has_value()) { + const uint64_t remaining = RemainingBytes(vector_offset); + SetError(vector_length_comment, BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, + "4"); + + AddSection( + vector_offset, + MakeSingleRegionBinarySection( + std::string(table->name()->c_str()) + "." + field->name()->c_str(), + BinarySectionType::Vector, + MakeBinaryRegion(vector_offset, remaining, + BinaryRegionType::Unknown, remaining, 0, + vector_length_comment))); + return; + } + + // Validate there are enough bytes left in the binary to process all the + // items. + const uint64_t last_item_offset = + vector_offset + vector_length_size_type + + vector_length.value() * GetElementSize(field); + + if (!IsValidOffset(last_item_offset - 1)) { + SetError(vector_length_comment, BinaryRegionStatus::ERROR_LENGTH_TOO_LONG); + AddSection( + vector_offset, + MakeSingleRegionBinarySection( + std::string(table->name()->c_str()) + "." + field->name()->c_str(), + BinarySectionType::Vector, + MakeBinaryRegion(vector_offset, vector_length_size_type, + region_type, 0, 0, vector_length_comment))); + + return; + } + + std::vector<BinaryRegion> regions; + + regions.push_back(MakeBinaryRegion(vector_offset, vector_length_size_type, + region_type, 0, 0, vector_length_comment)); + // Consume the vector length offset. + uint64_t offset = vector_offset + vector_length_size_type; + + switch (field->type()->element()) { + case reflection::BaseType::Obj: { + const reflection::Object *object = + schema_->objects()->Get(field->type()->index()); + + if (object->is_struct()) { + // Vector of structs + for (size_t i = 0; i < vector_length.value(); ++i) { + // Structs are inline to the vector. + const uint64_t next_offset = + BuildStruct(offset, regions, "[" + NumToString(i) + "]", object); + if (next_offset == offset) { break; } + offset = next_offset; + } + } else { + // Vector of objects + for (size_t i = 0; i < vector_length.value(); ++i) { + BinaryRegionComment vector_object_comment; + vector_object_comment.type = + BinaryRegionCommentType::VectorTableValue; + vector_object_comment.index = i; + + const auto table_relative_offset = ReadScalar<uint32_t>(offset); + if (!table_relative_offset.has_value()) { + const uint64_t remaining = RemainingBytes(offset); + SetError(vector_object_comment, + BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, "4"); + + regions.push_back( + MakeBinaryRegion(offset, remaining, BinaryRegionType::Unknown, + remaining, 0, vector_object_comment)); + break; + } + + // The table offset is relative from the offset location itself. + const uint64_t table_offset = offset + table_relative_offset.value(); + + if (!IsValidOffset(table_offset)) { + SetError(vector_object_comment, + BinaryRegionStatus::ERROR_OFFSET_OUT_OF_BINARY); + regions.push_back(MakeBinaryRegion( + offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + table_offset, vector_object_comment)); + + offset += sizeof(uint32_t); + continue; + } + + if (table_offset == parent_table_offset) { + SetError(vector_object_comment, + BinaryRegionStatus::ERROR_CYCLE_DETECTED); + // A cycle detected where a table vector field is pointing to + // itself. This should only happen in corrupted files. + regions.push_back(MakeBinaryRegion( + offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + table_offset, vector_object_comment)); + + offset += sizeof(uint32_t); + continue; + } + + regions.push_back(MakeBinaryRegion( + offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + table_offset, vector_object_comment)); + + // Consume the offset to the table. + offset += sizeof(uint32_t); + + BuildTable(table_offset, BinarySectionType::Table, object); + } + } + } break; + case reflection::BaseType::String: { + // Vector of strings + for (size_t i = 0; i < vector_length.value(); ++i) { + BinaryRegionComment vector_object_comment; + vector_object_comment.type = BinaryRegionCommentType::VectorStringValue; + vector_object_comment.index = i; + + const auto string_relative_offset = ReadScalar<uint32_t>(offset); + if (!string_relative_offset.has_value()) { + const uint64_t remaining = RemainingBytes(offset); + + SetError(vector_object_comment, + BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, "4"); + + regions.push_back( + MakeBinaryRegion(offset, remaining, BinaryRegionType::Unknown, + remaining, 0, vector_object_comment)); + break; + } + + // The string offset is relative from the offset location itself. + const uint64_t string_offset = offset + string_relative_offset.value(); + + if (!IsValidOffset(string_offset)) { + SetError(vector_object_comment, + BinaryRegionStatus::ERROR_OFFSET_OUT_OF_BINARY); + regions.push_back(MakeBinaryRegion( + offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + string_offset, vector_object_comment)); + + offset += sizeof(uint32_t); + continue; + } + + regions.push_back(MakeBinaryRegion( + offset, sizeof(uint32_t), BinaryRegionType::UOffset, 0, + string_offset, vector_object_comment)); + + BuildString(string_offset, table, field); + + offset += sizeof(uint32_t); + } + } break; + case reflection::BaseType::Union: { + // Vector of unions + // Unions have both their realized type (uint8_t for now) that are + // stored separately. These are stored in the field->index() - 1 + // location. + const uint16_t union_type_vector_id = field->id() - 1; + + auto vtable_entry = vtable_fields.find(union_type_vector_id); + if (vtable_entry == vtable_fields.end()) { + // TODO(dbaileychess): need to capture this error condition. + break; + } + + const uint64_t union_type_vector_field_offset = + parent_table_offset + vtable_entry->second.offset_from_table; + + const auto union_type_vector_field_relative_offset = + ReadScalar<uint16_t>(union_type_vector_field_offset); + + if (!union_type_vector_field_relative_offset.has_value()) { + const uint64_t remaining = RemainingBytes(offset); + BinaryRegionComment vector_union_comment; + vector_union_comment.type = BinaryRegionCommentType::VectorUnionValue; + SetError(vector_union_comment, + BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, "2"); + + regions.push_back(MakeBinaryRegion(offset, remaining, + BinaryRegionType::Unknown, remaining, + 0, vector_union_comment)); + + break; + } + + // Get the offset to the first type (the + sizeof(uint32_t) is to skip + // over the vector length which we already know). Validation happens + // within the loop below. + const uint64_t union_type_vector_data_offset = + union_type_vector_field_offset + + union_type_vector_field_relative_offset.value() + sizeof(uint32_t); + + for (size_t i = 0; i < vector_length.value(); ++i) { + BinaryRegionComment comment; + comment.type = BinaryRegionCommentType::VectorUnionValue; + comment.index = i; + + const auto union_relative_offset = ReadScalar<uint32_t>(offset); + if (!union_relative_offset.has_value()) { + const uint64_t remaining = RemainingBytes(offset); + + SetError(comment, BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, "4"); + + regions.push_back(MakeBinaryRegion(offset, remaining, + BinaryRegionType::Unknown, + remaining, 0, comment)); + + break; + } + + // The union offset is relative from the offset location itself. + const uint64_t union_offset = offset + union_relative_offset.value(); + + if (!IsValidOffset(union_offset)) { + SetError(comment, BinaryRegionStatus::ERROR_OFFSET_OUT_OF_BINARY); + + regions.push_back(MakeBinaryRegion(offset, sizeof(uint32_t), + BinaryRegionType::UOffset, 0, + union_offset, comment)); + continue; + } + + const auto realized_type = + ReadScalar<uint8_t>(union_type_vector_data_offset + i); + + if (!realized_type.has_value()) { + SetError(comment, BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, "1"); + regions.push_back(MakeBinaryRegion( + offset, 0, BinaryRegionType::Unknown, 0, 0, comment)); + continue; + } + + if (!IsValidUnionValue(vtable_entry->second.field->type()->index(), + realized_type.value())) { + // We already export an error in the union type field, so just skip + // building the union itself and it will default to an unreference + // Binary section. + offset += sizeof(uint32_t); + continue; + } + + const std::string enum_type = + BuildUnion(union_offset, realized_type.value(), field); + + comment.default_value = "(`" + enum_type + "`)"; + regions.push_back(MakeBinaryRegion(offset, sizeof(uint32_t), + BinaryRegionType::UOffset, 0, + union_offset, comment)); + + offset += sizeof(uint32_t); + } + } break; + default: { + if (IsScalar(field->type()->element())) { + const BinaryRegionType binary_region_type = + GetRegionType(field->type()->element()); + + const uint64_t type_size = GetTypeSize(field->type()->element()); + + // TODO(dbaileychess): It might be nicer to user the + // BinaryRegion.array_length field to indicate this. + for (size_t i = 0; i < vector_length.value(); ++i) { + BinaryRegionComment vector_scalar_comment; + vector_scalar_comment.type = BinaryRegionCommentType::VectorValue; + vector_scalar_comment.index = i; + + if (!IsValidRead(offset, type_size)) { + const uint64_t remaining = RemainingBytes(offset); + + SetError(vector_scalar_comment, + BinaryRegionStatus::ERROR_INCOMPLETE_BINARY, + std::to_string(type_size)); + + regions.push_back( + MakeBinaryRegion(offset, remaining, BinaryRegionType::Unknown, + remaining, 0, vector_scalar_comment)); + break; + } + + if (IsUnionType(field->type()->element())) { + // This is a type for a union. Validate the value + const auto enum_value = ReadScalar<uint8_t>(offset); + + // This should always have a value, due to the IsValidRead check + // above. + if (!IsValidUnionValue(field->type()->index(), + enum_value.value())) { + SetError(vector_scalar_comment, + BinaryRegionStatus::ERROR_INVALID_UNION_TYPE); + regions.push_back(MakeBinaryRegion(offset, type_size, + binary_region_type, 0, 0, + vector_scalar_comment)); + offset += type_size; + continue; + } + } + + regions.push_back(MakeBinaryRegion(offset, type_size, + binary_region_type, 0, 0, + vector_scalar_comment)); + offset += type_size; + } + } + } break; + } + AddSection(vector_offset, + MakeBinarySection(std::string(table->name()->c_str()) + "." + + field->name()->c_str(), + section_type, std::move(regions))); +} + +std::string BinaryAnnotator::BuildUnion(const uint64_t union_offset, + const uint8_t realized_type, + const reflection::Field *const field) { + const reflection::Enum *next_enum = + schema_->enums()->Get(field->type()->index()); + + const reflection::EnumVal *enum_val = next_enum->values()->Get(realized_type); + + if (ContainsSection(union_offset)) { return enum_val->name()->c_str(); } + + const reflection::Type *union_type = enum_val->union_type(); + + if (union_type->base_type() == reflection::BaseType::Obj) { + const reflection::Object *object = + schema_->objects()->Get(union_type->index()); + + if (object->is_struct()) { + // Union of vectors point to a new Binary section + std::vector<BinaryRegion> regions; + + BuildStruct(union_offset, regions, field->name()->c_str(), object); + + AddSection( + union_offset, + MakeBinarySection(std::string(object->name()->c_str()) + "." + + field->name()->c_str(), + BinarySectionType::Union, std::move(regions))); + } else { + BuildTable(union_offset, BinarySectionType::Table, object); + } + } + // TODO(dbaileychess): handle the other union types. + + return enum_val->name()->c_str(); +} + +void BinaryAnnotator::FixMissingRegions() { + std::vector<BinaryRegion> regions_to_insert; + for (auto ¤t_section : sections_) { + BinarySection §ion = current_section.second; + if (section.regions.empty()) { + // TODO(dbaileychess): is this possible? + continue; + } + + uint64_t offset = section.regions[0].offset + section.regions[0].length; + for (size_t i = 1; i < section.regions.size(); ++i) { + BinaryRegion ®ion = section.regions[i]; + + const uint64_t next_offset = region.offset; + if (!IsValidOffset(next_offset)) { + // TODO(dbaileychess): figure out how we get into this situation. + continue; + } + + if (offset < next_offset) { + const uint64_t padding_bytes = next_offset - offset; + + BinaryRegionComment comment; + comment.type = BinaryRegionCommentType::Padding; + + if (IsNonZeroRegion(offset, padding_bytes, binary_)) { + SetError(comment, BinaryRegionStatus::WARN_NO_REFERENCES); + regions_to_insert.push_back( + MakeBinaryRegion(offset, padding_bytes, BinaryRegionType::Unknown, + padding_bytes, 0, comment)); + } else { + regions_to_insert.push_back( + MakeBinaryRegion(offset, padding_bytes, BinaryRegionType::Uint8, + padding_bytes, 0, comment)); + } + } + offset = next_offset + region.length; + } + + if (!regions_to_insert.empty()) { + section.regions.insert(section.regions.end(), regions_to_insert.begin(), + regions_to_insert.end()); + std::stable_sort(section.regions.begin(), section.regions.end(), + BinaryRegionSort); + regions_to_insert.clear(); + } + } +} + +void BinaryAnnotator::FixMissingSections() { + uint64_t offset = 0; + + std::vector<BinarySection> sections_to_insert; + + for (auto ¤t_section : sections_) { + BinarySection §ion = current_section.second; + const uint64_t section_start_offset = current_section.first; + const uint64_t section_end_offset = + section.regions.back().offset + section.regions.back().length; + + if (offset < section_start_offset) { + // We are at an offset that is less then the current section. + const uint64_t pad_bytes = section_start_offset - offset + 1; + + sections_to_insert.push_back( + GenerateMissingSection(offset - 1, pad_bytes, binary_)); + } + offset = section_end_offset + 1; + } + + // Handle the case where there are still bytes left in the binary that are + // unaccounted for. + if (offset < binary_length_) { + const uint64_t pad_bytes = binary_length_ - offset + 1; + sections_to_insert.push_back( + GenerateMissingSection(offset - 1, pad_bytes, binary_)); + } + + for (const BinarySection §ion_to_insert : sections_to_insert) { + AddSection(section_to_insert.regions[0].offset, section_to_insert); + } +} + +bool BinaryAnnotator::ContainsSection(const uint64_t offset) { + auto it = sections_.lower_bound(offset); + // If the section is found, check that it is exactly equal its offset. + if (it != sections_.end() && it->first == offset) { return true; } + + // If this was the first section, there are no other previous sections to + // check. + if (it == sections_.begin()) { return false; } + + // Go back one section. + --it; + + // And check that if the offset is covered by the section. + return offset >= it->first && offset < it->second.regions.back().offset + + it->second.regions.back().length; +} + +} // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/binary_annotator.h b/contrib/libs/flatbuffers/src/binary_annotator.h new file mode 100644 index 00000000000..df8a17bed4f --- /dev/null +++ b/contrib/libs/flatbuffers/src/binary_annotator.h @@ -0,0 +1,406 @@ +/* + * Copyright 2021 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_BINARY_ANNOTATOR_H_ +#define FLATBUFFERS_BINARY_ANNOTATOR_H_ + +#include <list> +#include <map> +#include <string> +#include <vector> + +#include "flatbuffers/base.h" +#include "flatbuffers/reflection.h" +#include "flatbuffers/stl_emulation.h" +#include "flatbuffers/util.h" + +namespace flatbuffers { + +enum class BinaryRegionType { + Unknown = 0, + UOffset = 1, + SOffset = 2, + VOffset = 3, + Bool = 4, + Byte = 5, + Char = 6, + Uint8 = 7, + Int8 = 8, + Uint16 = 9, + Int16 = 10, + Uint32 = 11, + Int32 = 12, + Uint64 = 13, + Int64 = 14, + Float = 15, + Double = 16, + UType = 17, + UOffset64 = 18, +}; + +template<typename T> +static inline std::string ToHex(T i, size_t width = sizeof(T)) { + std::stringstream stream; + stream << std::hex << std::uppercase << std::setfill('0') + << std::setw(static_cast<int>(width)) << i; + return stream.str(); +} + +// Specialized version for uint8_t that don't work well with std::hex. +static inline std::string ToHex(uint8_t i) { + return ToHex<int>(static_cast<int>(i), 2); +} + +enum class BinaryRegionStatus { + OK = 0, + WARN = 100, + WARN_NO_REFERENCES, + WARN_CORRUPTED_PADDING, + WARN_PADDING_LENGTH, + ERROR = 200, + // An offset is pointing outside the binary bounds. + ERROR_OFFSET_OUT_OF_BINARY, + // Expecting to read N bytes but not enough remain in the binary. + ERROR_INCOMPLETE_BINARY, + // When a length of a vtable/vector is longer than possible. + ERROR_LENGTH_TOO_LONG, + // When a length of a vtable/vector is shorter than possible. + ERROR_LENGTH_TOO_SHORT, + // A field mark required is not present in the vtable. + ERROR_REQUIRED_FIELD_NOT_PRESENT, + // A realized union type is not within the enum bounds. + ERROR_INVALID_UNION_TYPE, + // Occurs when there is a cycle in offsets. + ERROR_CYCLE_DETECTED, +}; + +enum class BinaryRegionCommentType { + Unknown = 0, + SizePrefix, + // The offset to the root table. + RootTableOffset, + // The optional 4-char file identifier. + FileIdentifier, + // Generic 0-filled padding + Padding, + // The size of the vtable. + VTableSize, + // The size of the referring table. + VTableRefferingTableLength, + // Offsets to vtable fields. + VTableFieldOffset, + // Offsets to unknown vtable fields. + VTableUnknownFieldOffset, + // The vtable offset of a table. + TableVTableOffset, + // A "inline" table field value. + TableField, + // A table field that is unknown. + TableUnknownField, + // A table field value that points to another section. + TableOffsetField, + // A struct field value. + StructField, + // A array field value. + ArrayField, + // The length of the string. + StringLength, + // The string contents. + StringValue, + // The explicit string terminator. + StringTerminator, + // The length of the vector (# of items). + VectorLength, + // A "inline" value of a vector. + VectorValue, + // A vector value that points to another section. + VectorTableValue, + VectorStringValue, + VectorUnionValue, +}; + +struct BinaryRegionComment { + BinaryRegionStatus status = BinaryRegionStatus::OK; + + // If status is non OK, this may be filled in with additional details. + std::string status_message; + + BinaryRegionCommentType type = BinaryRegionCommentType::Unknown; + + std::string name; + + std::string default_value; + + size_t index = 0; +}; + +struct BinaryRegion { + // Offset into the binary where this region begins. + uint64_t offset = 0; + + // The length of this region in bytes. + uint64_t length = 0; + + // The underlying datatype of this region + BinaryRegionType type = BinaryRegionType::Unknown; + + // If `type` is an array/vector, this is the number of those types this region + // encompasses. + uint64_t array_length = 0; + + // If the is an offset to some other region, this is what it points to. The + // offset is relative to overall binary, not to this region. + uint64_t points_to_offset = 0; + + // The comment on the region. + BinaryRegionComment comment; +}; + +enum class BinarySectionType { + Unknown = 0, + Header = 1, + Table = 2, + RootTable = 3, + VTable = 4, + Struct = 5, + String = 6, + Vector = 7, + Union = 8, + Padding = 9, + Vector64 = 10, +}; + +// A section of the binary that is grouped together in some logical manner, and +// often is pointed too by some other offset BinaryRegion. Sections include +// `tables`, `vtables`, `strings`, `vectors`, etc.. +struct BinarySection { + // User-specified name of the section, if applicable. + std::string name; + + // The type of this section. + BinarySectionType type = BinarySectionType::Unknown; + + // The binary regions that make up this section, in order of their offsets. + std::vector<BinaryRegion> regions; +}; + +inline static BinaryRegionType GetRegionType(reflection::BaseType base_type) { + switch (base_type) { + case reflection::UType: return BinaryRegionType::UType; + case reflection::Bool: return BinaryRegionType::Uint8; + case reflection::Byte: return BinaryRegionType::Uint8; + case reflection::UByte: return BinaryRegionType::Uint8; + case reflection::Short: return BinaryRegionType::Int16; + case reflection::UShort: return BinaryRegionType::Uint16; + case reflection::Int: return BinaryRegionType::Uint32; + case reflection::UInt: return BinaryRegionType::Uint32; + case reflection::Long: return BinaryRegionType::Int64; + case reflection::ULong: return BinaryRegionType::Uint64; + case reflection::Float: return BinaryRegionType::Float; + case reflection::Double: return BinaryRegionType::Double; + default: return BinaryRegionType::Unknown; + } +} + +inline static std::string ToString(const BinaryRegionType type) { + switch (type) { + case BinaryRegionType::UOffset: return "UOffset32"; + case BinaryRegionType::UOffset64: return "UOffset64"; + case BinaryRegionType::SOffset: return "SOffset32"; + case BinaryRegionType::VOffset: return "VOffset16"; + case BinaryRegionType::Bool: return "bool"; + case BinaryRegionType::Char: return "char"; + case BinaryRegionType::Byte: return "int8_t"; + case BinaryRegionType::Uint8: return "uint8_t"; + case BinaryRegionType::Uint16: return "uint16_t"; + case BinaryRegionType::Uint32: return "uint32_t"; + case BinaryRegionType::Uint64: return "uint64_t"; ; + case BinaryRegionType::Int8: return "int8_t"; + case BinaryRegionType::Int16: return "int16_t"; + case BinaryRegionType::Int32: return "int32_t"; + case BinaryRegionType::Int64: return "int64_t"; + case BinaryRegionType::Double: return "double"; + case BinaryRegionType::Float: return "float"; + case BinaryRegionType::UType: return "UType8"; + case BinaryRegionType::Unknown: return "?uint8_t"; + default: return "todo"; + } +} + +class BinaryAnnotator { + public: + explicit BinaryAnnotator(const uint8_t *const bfbs, + const uint64_t bfbs_length, + const uint8_t *const binary, + const uint64_t binary_length, + const bool is_size_prefixed) + : bfbs_(bfbs), + bfbs_length_(bfbs_length), + schema_(reflection::GetSchema(bfbs)), + binary_(binary), + binary_length_(binary_length), + is_size_prefixed_(is_size_prefixed) {} + + std::map<uint64_t, BinarySection> Annotate(); + + private: + struct VTable { + struct Entry { + const reflection::Field *field = nullptr; + uint16_t offset_from_table = 0; + }; + + const reflection::Object *referring_table = nullptr; + + // Field ID -> {field def, offset from table} + std::map<uint16_t, Entry> fields; + + uint16_t vtable_size = 0; + uint16_t table_size = 0; + }; + + uint64_t BuildHeader(uint64_t offset); + + // VTables can be shared across instances or even across objects. This + // attempts to get an existing vtable given the offset and table type, + // otherwise it will built the vtable, memorize it, and return the built + // VTable. Returns nullptr if building the VTable fails. + VTable *GetOrBuildVTable(uint64_t offset, const reflection::Object *table, + uint64_t offset_of_referring_table); + + void BuildTable(uint64_t offset, const BinarySectionType type, + const reflection::Object *table); + + uint64_t BuildStruct(uint64_t offset, std::vector<BinaryRegion> ®ions, + const std::string referring_field_name, + const reflection::Object *structure); + + void BuildString(uint64_t offset, const reflection::Object *table, + const reflection::Field *field); + + void BuildVector(uint64_t offset, const reflection::Object *table, + const reflection::Field *field, uint64_t parent_table_offset, + const std::map<uint16_t, VTable::Entry> vtable_fields); + + std::string BuildUnion(uint64_t offset, uint8_t realized_type, + const reflection::Field *field); + + void FixMissingRegions(); + void FixMissingSections(); + + inline bool IsValidOffset(const uint64_t offset) const { + return offset < binary_length_; + } + + // Determines if performing a GetScalar request for `T` at `offset` would read + // passed the end of the binary. + template<typename T> inline bool IsValidRead(const uint64_t offset) const { + return IsValidRead(offset, sizeof(T)); + } + + inline bool IsValidRead(const uint64_t offset, const uint64_t length) const { + return length < binary_length_ && IsValidOffset(offset + length - 1); + } + + // Calculate the number of bytes remaining from the given offset. If offset is + // > binary_length, 0 is returned. + uint64_t RemainingBytes(const uint64_t offset) const { + return IsValidOffset(offset) ? binary_length_ - offset : 0; + } + + template<typename T> + flatbuffers::Optional<T> ReadScalar(const uint64_t offset) const { + if (!IsValidRead<T>(offset)) { return flatbuffers::nullopt; } + + return flatbuffers::ReadScalar<T>(binary_ + offset); + } + + // Adds the provided `section` keyed by the `offset` it occurs at. If a + // section is already added at that offset, it doesn't replace the exisiting + // one. + void AddSection(const uint64_t offset, const BinarySection §ion) { + sections_.insert(std::make_pair(offset, section)); + } + + bool IsInlineField(const reflection::Field *const field) { + if (field->type()->base_type() == reflection::BaseType::Obj) { + return schema_->objects()->Get(field->type()->index())->is_struct(); + } + return IsScalar(field->type()->base_type()); + } + + bool IsUnionType(const reflection::BaseType type) { + return (type == reflection::BaseType::UType || + type == reflection::BaseType::Union); + } + + bool IsUnionType(const reflection::Field *const field) { + return IsUnionType(field->type()->base_type()) && + field->type()->index() >= 0; + } + + bool IsValidUnionValue(const reflection::Field *const field, + const uint8_t value) { + return IsUnionType(field) && + IsValidUnionValue(field->type()->index(), value); + } + + bool IsValidUnionValue(const uint32_t enum_id, const uint8_t value) { + if (enum_id >= schema_->enums()->size()) { return false; } + + const reflection::Enum *enum_def = schema_->enums()->Get(enum_id); + + if (enum_def == nullptr) { return false; } + + return value < enum_def->values()->size(); + } + + uint64_t GetElementSize(const reflection::Field *const field) { + if (IsScalar(field->type()->element())) { + return GetTypeSize(field->type()->element()); + } + + switch (field->type()->element()) { + case reflection::BaseType::Obj: { + auto obj = schema_->objects()->Get(field->type()->index()); + return obj->is_struct() ? obj->bytesize() : sizeof(uint32_t); + } + default: return sizeof(uint32_t); + } + } + + bool ContainsSection(const uint64_t offset); + + // The schema for the binary file + const uint8_t *bfbs_; + const uint64_t bfbs_length_; + const reflection::Schema *schema_; + + // The binary data itself. + const uint8_t *binary_; + const uint64_t binary_length_; + const bool is_size_prefixed_; + + // Map of binary offset to vtables, to dedupe vtables. + std::map<uint64_t, std::list<VTable>> vtables_; + + // The annotated binary sections, index by their absolute offset. + std::map<uint64_t, BinarySection> sections_; +}; + +} // namespace flatbuffers + +#endif // FLATBUFFERS_BINARY_ANNOTATOR_H_ diff --git a/contrib/libs/flatbuffers/src/code_generators.cpp b/contrib/libs/flatbuffers/src/code_generators.cpp index 745406ba95b..c5efb222b1c 100644 --- a/contrib/libs/flatbuffers/src/code_generators.cpp +++ b/contrib/libs/flatbuffers/src/code_generators.cpp @@ -30,6 +30,52 @@ namespace flatbuffers { +namespace { + +static std::string JavaCSharpMakeRule(const bool java, const Parser &parser, + const std::string &path, + const std::string &file_name) { + const std::string file_extension = java ? ".java" : ".cs"; + std::string make_rule; + + for (auto it = parser.enums_.vec.begin(); it != parser.enums_.vec.end(); + ++it) { + auto &enum_def = **it; + if (!make_rule.empty()) make_rule += " "; + std::string directory = + BaseGenerator::NamespaceDir(parser, path, *enum_def.defined_namespace); + make_rule += directory + enum_def.name + file_extension; + } + + for (auto it = parser.structs_.vec.begin(); it != parser.structs_.vec.end(); + ++it) { + auto &struct_def = **it; + if (!make_rule.empty()) make_rule += " "; + std::string directory = BaseGenerator::NamespaceDir( + parser, path, *struct_def.defined_namespace); + make_rule += directory + struct_def.name + file_extension; + } + + make_rule += ": "; + auto included_files = parser.GetIncludedFilesRecursive(file_name); + for (auto it = included_files.begin(); it != included_files.end(); ++it) { + make_rule += " " + *it; + } + return make_rule; +} + + +static std::string BinaryFileName(const Parser &parser, const std::string &path, + const std::string &file_name) { + auto ext = parser.file_extension_.length() ? parser.file_extension_ : "bin"; + return path + file_name + "." + ext; +} + +} // namespace + + + + void CodeWriter::operator+=(std::string text) { if (!ignore_ident_ && !text.empty()) AppendIdent(stream_); @@ -60,7 +106,7 @@ void CodeWriter::operator+=(std::string text) { // Update the text to everything after the }}. text = text.substr(end + 2); } - if (!text.empty() && string_back(text) == '\\') { + if (!text.empty() && text.back() == '\\') { text.pop_back(); ignore_ident_ = true; stream_ << text; @@ -91,7 +137,8 @@ std::string BaseGenerator::NamespaceDir(const Parser &parser, std::string namespace_dir = path; // Either empty or ends in separator. auto &namespaces = ns.components; for (auto it = namespaces.begin(); it != namespaces.end(); ++it) { - namespace_dir += !dasherize ? *it : ToDasherizedCase(*it); + namespace_dir += + !dasherize ? *it : ConvertCase(*it, Case::kDasher, Case::kUpperCamel); namespace_dir += kPathSeparator; EnsureDirExists(namespace_dir); } @@ -103,22 +150,6 @@ std::string BaseGenerator::NamespaceDir(const Namespace &ns, return BaseGenerator::NamespaceDir(parser_, path_, ns, dasherize); } -std::string BaseGenerator::ToDasherizedCase(const std::string pascal_case) { - std::string dasherized_case; - char p = 0; - for (size_t i = 0; i < pascal_case.length(); i++) { - char const &c = pascal_case[i]; - if (is_alpha_upper(c)) { - if (i > 0 && p != kPathSeparator) dasherized_case += "-"; - dasherized_case += CharToLower(c); - } else { - dasherized_case += c; - } - p = c; - } - return dasherized_case; -} - std::string BaseGenerator::FullNamespace(const char *separator, const Namespace &ns) { std::string namespace_name; @@ -146,8 +177,9 @@ std::string BaseGenerator::WrapInNameSpace(const Namespace *ns, return qualified_name + name; } -std::string BaseGenerator::WrapInNameSpace(const Definition &def) const { - return WrapInNameSpace(def.defined_namespace, def.name); +std::string BaseGenerator::WrapInNameSpace(const Definition &def, + const std::string &suffix) const { + return WrapInNameSpace(def.defined_namespace, def.name + suffix); } std::string BaseGenerator::GetNameSpace(const Definition &def) const { @@ -314,46 +346,14 @@ std::string SimpleFloatConstantGenerator::NaN(float v) const { return this->NaN(static_cast<double>(v)); } -std::string JavaCSharpMakeRule(const Parser &parser, const std::string &path, - const std::string &file_name) { - FLATBUFFERS_ASSERT(parser.opts.lang == IDLOptions::kJava || - parser.opts.lang == IDLOptions::kCSharp); - - std::string file_extension = - (parser.opts.lang == IDLOptions::kJava) ? ".java" : ".cs"; - - std::string make_rule; - - for (auto it = parser.enums_.vec.begin(); it != parser.enums_.vec.end(); - ++it) { - auto &enum_def = **it; - if (!make_rule.empty()) make_rule += " "; - std::string directory = - BaseGenerator::NamespaceDir(parser, path, *enum_def.defined_namespace); - make_rule += directory + enum_def.name + file_extension; - } - - for (auto it = parser.structs_.vec.begin(); it != parser.structs_.vec.end(); - ++it) { - auto &struct_def = **it; - if (!make_rule.empty()) make_rule += " "; - std::string directory = BaseGenerator::NamespaceDir( - parser, path, *struct_def.defined_namespace); - make_rule += directory + struct_def.name + file_extension; - } - make_rule += ": "; - auto included_files = parser.GetIncludedFilesRecursive(file_name); - for (auto it = included_files.begin(); it != included_files.end(); ++it) { - make_rule += " " + *it; - } - return make_rule; +std::string JavaMakeRule(const Parser &parser, const std::string &path, + const std::string &file_name) { + return JavaCSharpMakeRule(true, parser, path, file_name); } - -std::string BinaryFileName(const Parser &parser, const std::string &path, +std::string CSharpMakeRule(const Parser &parser, const std::string &path, const std::string &file_name) { - auto ext = parser.file_extension_.length() ? parser.file_extension_ : "bin"; - return path + file_name + "." + ext; + return JavaCSharpMakeRule(false, parser, path, file_name); } bool GenerateBinary(const Parser &parser, const std::string &path, diff --git a/contrib/libs/flatbuffers/src/file_binary_writer.cpp b/contrib/libs/flatbuffers/src/file_binary_writer.cpp new file mode 100644 index 00000000000..69d83e77c9d --- /dev/null +++ b/contrib/libs/flatbuffers/src/file_binary_writer.cpp @@ -0,0 +1,49 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fstream> +#include <set> +#include <string> + +#include "flatbuffers/file_manager.h" + +namespace flatbuffers { + +class FileBinaryWriter : public FileManager { + public: + bool SaveFile(const std::string &absolute_file_name, + const std::string &content) override { + std::ofstream ofs(absolute_file_name, std::ofstream::binary); + if (!ofs.is_open()) return false; + ofs.write(content.c_str(), content.size()); + return !ofs.bad(); + } + + bool Loadfile(const std::string &absolute_file_name, std::string *output) { + if (DirExists(absolute_file_name.c_str())) return false; + std::ifstream ifs(absolute_file_name, std::ifstream::binary); + if (!ifs.is_open()) return false; + // The fastest way to read a file into a string. + ifs.seekg(0, std::ios::end); + auto size = ifs.tellg(); + (*output).resize(static_cast<size_t>(size)); + ifs.seekg(0, std::ios::beg); + ifs.read(&(*output)[0], (*output).size()); + return !ifs.bad(); + } +}; + +} // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/file_name_saving_file_manager.cpp b/contrib/libs/flatbuffers/src/file_name_saving_file_manager.cpp new file mode 100644 index 00000000000..fc4a4aa1ec7 --- /dev/null +++ b/contrib/libs/flatbuffers/src/file_name_saving_file_manager.cpp @@ -0,0 +1,49 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fstream> +#include <set> +#include <string> + +#include "flatbuffers/file_manager.h" + +namespace flatbuffers { + +class FileNameSavingFileManager : public FileManager { + public: + FileNameSavingFileManager(std::set<std::string> file_names) + : file_names_(file_names) {} + + bool SaveFile(const std::string &absolute_file_name, + const std::string &content) override { + (void)content; + auto pair = file_names_.insert(absolute_file_name); + // pair.second indicates whether the insertion is + // successful or not. + return pair.second; + } + + bool Loadfile(const std::string &absolute_file_name, std::string *content) { + (void) absolute_file_name; + (void) content; + return false; + } + + private: + std::set<std::string> file_names_; +}; + +} // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/file_writer.cpp b/contrib/libs/flatbuffers/src/file_writer.cpp new file mode 100644 index 00000000000..bd34545b00d --- /dev/null +++ b/contrib/libs/flatbuffers/src/file_writer.cpp @@ -0,0 +1,47 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fstream> +#include <set> +#include <string> + +#include "flatbuffers/file_manager.h" + +namespace flatbuffers { + +class FileWriter : public FileManager { + public: + bool SaveFile(const std::string &absolute_file_name, + const std::string &content) override { + std::ofstream ofs(absolute_file_name, std::ofstream::out); + if (!ofs.is_open()) return false; + ofs.write(content.c_str(), content.size()); + return !ofs.bad(); + } + + bool Loadfile(const std::string &absolute_file_name, std::string *output) { + if (DirExists(absolute_file_name.c_str())) return false; + std::ifstream ifs(absolute_file_name, std::ifstream::in); + if (!ifs.is_open()) return false; + // This is slower, but works correctly on all platforms for text files. + std::ostringstream oss; + oss << ifs.rdbuf(); + *output = oss.str(); + return !ifs.bad(); + } +}; + +} // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/flatc.cpp b/contrib/libs/flatbuffers/src/flatc.cpp index 221b88676df..a1501712e59 100644 --- a/contrib/libs/flatbuffers/src/flatc.cpp +++ b/contrib/libs/flatbuffers/src/flatc.cpp @@ -16,26 +16,38 @@ #include "flatbuffers/flatc.h" +#include <algorithm> +#include <limits> #include <list> +#include <memory> +#include <sstream> + +#include "annotated_binary_text_gen.h" +#include "binary_annotator.h" +#include "flatbuffers/code_generator.h" +#include "flatbuffers/idl.h" +#include "flatbuffers/util.h" namespace flatbuffers { -const char *FLATC_VERSION() { return FLATBUFFERS_VERSION(); } +static const char *FLATC_VERSION() { return FLATBUFFERS_VERSION(); } void FlatCompiler::ParseFile( flatbuffers::Parser &parser, const std::string &filename, const std::string &contents, - std::vector<const char *> &include_directories) const { + const std::vector<const char *> &include_directories) const { auto local_include_directory = flatbuffers::StripFileName(filename); - include_directories.push_back(local_include_directory.c_str()); - include_directories.push_back(nullptr); - if (!parser.Parse(contents.c_str(), &include_directories[0], - filename.c_str())) { + + std::vector<const char *> inc_directories; + inc_directories.insert(inc_directories.end(), include_directories.begin(), + include_directories.end()); + inc_directories.push_back(local_include_directory.c_str()); + inc_directories.push_back(nullptr); + + if (!parser.Parse(contents.c_str(), &inc_directories[0], filename.c_str())) { Error(parser.error_, false, false); } if (!parser.error_.empty()) { Warn(parser.error_, false); } - include_directories.pop_back(); - include_directories.pop_back(); } void FlatCompiler::LoadBinarySchema(flatbuffers::Parser &parser, @@ -56,181 +68,399 @@ void FlatCompiler::Error(const std::string &err, bool usage, params_.error_fn(this, err, usage, show_exe_name); } -std::string FlatCompiler::GetUsageString(const char *program_name) const { +const static FlatCOption flatc_options[] = { + { "o", "", "PATH", "Prefix PATH to all generated files." }, + { "I", "", "PATH", "Search for includes in the specified path." }, + { "M", "", "", "Print make rules for generated files." }, + { "", "version", "", "Print the version number of flatc and exit." }, + { "h", "help", "", "Prints this help text and exit." }, + { "", "strict-json", "", + "Strict JSON: field names must be / will be quoted, no trailing commas in " + "tables/vectors." }, + { "", "allow-non-utf8", "", + "Pass non-UTF-8 input through parser and emit nonstandard \\x escapes in " + "JSON. (Default is to raise parse error on non-UTF-8 input.)" }, + { "", "natural-utf8", "", + "Output strings with UTF-8 as human-readable strings. By default, UTF-8 " + "characters are printed as \\uXXXX escapes." }, + { "", "defaults-json", "", + "Output fields whose value is the default when writing JSON" }, + { "", "unknown-json", "", + "Allow fields in JSON that are not defined in the schema. These fields " + "will be discared when generating binaries." }, + { "", "no-prefix", "", + "Don't prefix enum values with the enum type in C++." }, + { "", "scoped-enums", "", + "Use C++11 style scoped and strongly typed enums. Also implies " + "--no-prefix." }, + { "", "no-emit-min-max-enum-values", "", + "Disable generation of MIN and MAX enumerated values for scoped enums " + "and prefixed enums." }, + { "", "swift-implementation-only", "", + "Adds a @_implementationOnly to swift imports" }, + { "", "gen-includes", "", + "(deprecated), this is the default behavior. If the original behavior is " + "required (no include statements) use --no-includes." }, + { "", "no-includes", "", + "Don't generate include statements for included schemas the generated " + "file depends on (C++, Python, Proto-to-Fbs)." }, + { "", "gen-mutable", "", + "Generate accessors that can mutate buffers in-place." }, + { "", "gen-onefile", "", + "Generate a single output file for C#, Go, Java, Kotlin and Python. " + "Implies --no-include." }, + { "", "gen-name-strings", "", + "Generate type name functions for C++ and Rust." }, + { "", "gen-object-api", "", "Generate an additional object-based API." }, + { "", "gen-compare", "", "Generate operator== for object-based API types." }, + { "", "gen-nullable", "", + "Add Clang _Nullable for C++ pointer. or @Nullable for Java" }, + { "", "java-package-prefix", "", + "Add a prefix to the generated package name for Java." }, + { "", "java-checkerframework", "", "Add @Pure for Java." }, + { "", "gen-generated", "", "Add @Generated annotation for Java." }, + { "", "gen-jvmstatic", "", + "Add @JvmStatic annotation for Kotlin methods in companion object for " + "interop from Java to Kotlin." }, + { "", "gen-all", "", + "Generate not just code for the current schema files, but for all files it " + "includes as well. If the language uses a single file for output (by " + "default the case for C++ and JS), all code will end up in this one " + "file." }, + { "", "gen-json-emit", "", + "Generates encoding code which emits Flatbuffers into JSON" }, + { "", "cpp-include", "", "Adds an #include in generated file." }, + { "", "cpp-ptr-type", "T", + "Set object API pointer type (default std::unique_ptr)." }, + { "", "cpp-str-type", "T", + "Set object API string type (default std::string). T::c_str(), T::length() " + "and T::empty() must be supported. The custom type also needs to be " + "constructible from std::string (see the --cpp-str-flex-ctor option to " + "change this behavior)" }, + { "", "cpp-str-flex-ctor", "", + "Don't construct custom string types by passing std::string from " + "Flatbuffers, but (char* + length)." }, + { "", "cpp-field-case-style", "STYLE", + "Generate C++ fields using selected case style. Supported STYLE values: * " + "'unchanged' - leave unchanged (default) * 'upper' - schema snake_case " + "emits UpperCamel; * 'lower' - schema snake_case emits lowerCamel." }, + { "", "cpp-std", "CPP_STD", + "Generate a C++ code using features of selected C++ standard. Supported " + "CPP_STD values: * 'c++0x' - generate code compatible with old compilers; " + "'c++11' - use C++11 code generator (default); * 'c++17' - use C++17 " + "features in generated code (experimental)." }, + { "", "cpp-static-reflection", "", + "When using C++17, generate extra code to provide compile-time (static) " + "reflection of Flatbuffers types. Requires --cpp-std to be \"c++17\" or " + "higher." }, + { "", "object-prefix", "PREFIX", + "Customize class prefix for C++ object-based API." }, + { "", "object-suffix", "SUFFIX", + "Customize class suffix for C++ object-based API. Default Value is " + "\"T\"." }, + { "", "go-namespace", "", "Generate the overriding namespace in Golang." }, + { "", "go-import", "IMPORT", + "Generate the overriding import for flatbuffers in Golang (default is " + "\"github.com/google/flatbuffers/go\")." }, + { "", "go-module-name", "", + "Prefix local import paths of generated go code with the module name" }, + { "", "raw-binary", "", + "Allow binaries without file_identifier to be read. This may crash flatc " + "given a mismatched schema." }, + { "", "size-prefixed", "", "Input binaries are size prefixed buffers." }, + { "", "proto-namespace-suffix", "SUFFIX", + "Add this namespace to any flatbuffers generated from protobufs." }, + { "", "oneof-union", "", "Translate .proto oneofs to flatbuffer unions." }, + { "", "keep-proto-id", "", "Keep protobuf field ids in generated fbs file." }, + { "", "proto-id-gap", "", + "Action that should be taken when a gap between protobuf ids found. " + "Supported values: * " + "'nop' - do not care about gap * 'warn' - A warning message will be shown " + "about the gap in protobuf ids" + "(default) " + "* 'error' - An error message will be shown and the fbs generation will be " + "interrupted." }, + { "", "grpc", "", "Generate GRPC interfaces for the specified languages." }, + { "", "schema", "", "Serialize schemas instead of JSON (use with -b)." }, + { "", "bfbs-filenames", "PATH", + "Sets the root path where reflection filenames in reflection.fbs are " + "relative to. The 'root' is denoted with `//`. E.g. if PATH=/a/b/c " + "then /a/d/e.fbs will be serialized as //../d/e.fbs. (PATH defaults to the " + "directory of the first provided schema file." }, + { "", "bfbs-comments", "", "Add doc comments to the binary schema files." }, + { "", "bfbs-builtins", "", + "Add builtin attributes to the binary schema files." }, + { "", "bfbs-gen-embed", "", + "Generate code to embed the bfbs schema to the source." }, + { "", "conform", "FILE", + "Specify a schema the following schemas should be an evolution of. Gives " + "errors if not." }, + { "", "conform-includes", "PATH", + "Include path for the schema given with --conform PATH" }, + { "", "filename-suffix", "SUFFIX", + "The suffix appended to the generated file names (Default is " + "'_generated')." }, + { "", "filename-ext", "EXT", + "The extension appended to the generated file names. Default is " + "language-specific (e.g., '.h' for C++)" }, + { "", "include-prefix", "PATH", + "Prefix this PATH to any generated include statements." }, + { "", "keep-prefix", "", + "Keep original prefix of schema include statement." }, + { "", "reflect-types", "", + "Add minimal type reflection to code generation." }, + { "", "reflect-names", "", "Add minimal type/name reflection." }, + { "", "rust-serialize", "", + "Implement serde::Serialize on generated Rust types." }, + { "", "rust-module-root-file", "", + "Generate rust code in individual files with a module root file." }, + { "", "root-type", "T", "Select or override the default root_type." }, + { "", "require-explicit-ids", "", + "When parsing schemas, require explicit ids (id: x)." }, + { "", "force-defaults", "", + "Emit default values in binary output from JSON" }, + { "", "force-empty", "", + "When serializing from object API representation, force strings and " + "vectors to empty rather than null." }, + { "", "force-empty-vectors", "", + "When serializing from object API representation, force vectors to empty " + "rather than null." }, + { "", "flexbuffers", "", + "Used with \"binary\" and \"json\" options, it generates data using " + "schema-less FlexBuffers." }, + { "", "no-warnings", "", "Inhibit all warnings messages." }, + { "", "warnings-as-errors", "", "Treat all warnings as errors." }, + { "", "cs-global-alias", "", + "Prepend \"global::\" to all user generated csharp classes and " + "structs." }, + { "", "cs-gen-json-serializer", "", + "Allows (de)serialization of JSON text in the Object API. (requires " + "--gen-object-api)." }, + { "", "json-nested-bytes", "", + "Allow a nested_flatbuffer field to be parsed as a vector of bytes " + "in JSON, which is unsafe unless checked by a verifier afterwards." }, + { "", "ts-flat-files", "", + "Generate a single typescript file per .fbs file. Implies " + "ts_entry_points." }, + { "", "ts-entry-points", "", + "Generate entry point typescript per namespace. Implies gen-all." }, + { "", "annotate-sparse-vectors", "", "Don't annotate every vector element." }, + { "", "annotate", "SCHEMA", + "Annotate the provided BINARY_FILE with the specified SCHEMA file." }, + { "", "no-leak-private-annotation", "", + "Prevents multiple type of annotations within a Fbs SCHEMA file. " + "Currently this is required to generate private types in Rust" }, + { "", "python-no-type-prefix-suffix", "", + "Skip emission of Python functions that are prefixed with typenames" }, + { "", "python-typing", "", "Generate Python type annotations" }, + { "", "file-names-only", "", + "Print out generated file names without writing to the files" }, +}; + +auto cmp = [](FlatCOption a, FlatCOption b) { return a.long_opt < b.long_opt; }; +static std::set<FlatCOption, decltype(cmp)> language_options(cmp); + +static void AppendTextWrappedString(std::stringstream &ss, std::string &text, + size_t max_col, size_t start_col) { + size_t max_line_length = max_col - start_col; + + if (text.length() > max_line_length) { + size_t ideal_break_location = text.rfind(' ', max_line_length); + size_t length = std::min(max_line_length, ideal_break_location); + ss << text.substr(0, length) << "\n"; + ss << std::string(start_col, ' '); + std::string rest_of_description = text.substr( + ((ideal_break_location < max_line_length || text.at(length) == ' ') + ? length + 1 + : length)); + AppendTextWrappedString(ss, rest_of_description, max_col, start_col); + } else { + ss << text; + } +} + +static void AppendOption(std::stringstream &ss, const FlatCOption &option, + size_t max_col, size_t min_col_for_description) { + size_t chars = 2; + ss << " "; + if (!option.short_opt.empty()) { + chars += 2 + option.short_opt.length(); + ss << "-" << option.short_opt; + if (!option.long_opt.empty()) { + chars++; + ss << ","; + } + ss << " "; + } + if (!option.long_opt.empty()) { + chars += 3 + option.long_opt.length(); + ss << "--" << option.long_opt << " "; + } + if (!option.parameter.empty()) { + chars += 1 + option.parameter.length(); + ss << option.parameter << " "; + } + size_t start_of_description = chars; + if (start_of_description > min_col_for_description) { + ss << "\n"; + start_of_description = min_col_for_description; + ss << std::string(start_of_description, ' '); + } else { + while (start_of_description < min_col_for_description) { + ss << " "; + start_of_description++; + } + } + if (!option.description.empty()) { + std::string description = option.description; + AppendTextWrappedString(ss, description, max_col, start_of_description); + } + ss << "\n"; +} + +static void AppendShortOption(std::stringstream &ss, + const FlatCOption &option) { + if (!option.short_opt.empty()) { + ss << "-" << option.short_opt; + if (!option.long_opt.empty()) { ss << "|"; } + } + if (!option.long_opt.empty()) { ss << "--" << option.long_opt; } +} + +std::string FlatCompiler::GetShortUsageString( + const std::string &program_name) const { std::stringstream ss; - ss << "Usage: " << program_name << " [OPTION]... FILE... [-- FILE...]\n"; - for (size_t i = 0; i < params_.num_generators; ++i) { - const Generator &g = params_.generators[i]; + ss << "Usage: " << program_name << " ["; + + for (const FlatCOption &option : language_options) { + AppendShortOption(ss, option); + ss << ", "; + } - std::stringstream full_name; - full_name << std::setw(16) << std::left << g.generator_opt_long; - const char *name = g.generator_opt_short ? g.generator_opt_short : " "; - const char *help = g.generator_help; + for (const FlatCOption &option : flatc_options) { + AppendShortOption(ss, option); + ss << ", "; + } + + ss.seekp(-2, ss.cur); + ss << "]... FILE... [-- BINARY_FILE...]"; + std::string help = ss.str(); + std::stringstream ss_textwrap; + AppendTextWrappedString(ss_textwrap, help, 80, 0); + return ss_textwrap.str(); +} + +std::string FlatCompiler::GetUsageString( + const std::string &program_name) const { + std::stringstream ss; + ss << "Usage: " << program_name + << " [OPTION]... FILE... [-- BINARY_FILE...]\n"; + + for (const FlatCOption &option : language_options) { + AppendOption(ss, option, 80, 25); + } + ss << "\n"; - ss << " " << full_name.str() << " " << name << " " << help << ".\n"; + for (const FlatCOption &option : flatc_options) { + AppendOption(ss, option, 80, 25); } - // clang-format off - - // Output width - // 12345678901234567890123456789012345678901234567890123456789012345678901234567890 - ss << - " -o PATH Prefix PATH to all generated files.\n" - " -I PATH Search for includes in the specified path.\n" - " -M Print make rules for generated files.\n" - " --version Print the version number of flatc and exit.\n" - " --strict-json Strict JSON: field names must be / will be quoted,\n" - " no trailing commas in tables/vectors.\n" - " --allow-non-utf8 Pass non-UTF-8 input through parser and emit nonstandard\n" - " \\x escapes in JSON. (Default is to raise parse error on\n" - " non-UTF-8 input.)\n" - " --natural-utf8 Output strings with UTF-8 as human-readable strings.\n" - " By default, UTF-8 characters are printed as \\uXXXX escapes.\n" - " --defaults-json Output fields whose value is the default when\n" - " writing JSON\n" - " --unknown-json Allow fields in JSON that are not defined in the\n" - " schema. These fields will be discared when generating\n" - " binaries.\n" - " --no-prefix Don\'t prefix enum values with the enum type in C++.\n" - " --scoped-enums Use C++11 style scoped and strongly typed enums.\n" - " also implies --no-prefix.\n" - " --gen-includes (deprecated), this is the default behavior.\n" - " If the original behavior is required (no include\n" - " statements) use --no-includes.\n" - " --no-includes Don\'t generate include statements for included\n" - " schemas the generated file depends on (C++ / Python).\n" - " --gen-mutable Generate accessors that can mutate buffers in-place.\n" - " --gen-onefile Generate single output file for C# and Go.\n" - " --gen-name-strings Generate type name functions for C++ and Rust.\n" - " --gen-object-api Generate an additional object-based API.\n" - " --gen-compare Generate operator== for object-based API types.\n" - " --gen-nullable Add Clang _Nullable for C++ pointer. or @Nullable for Java\n" - " --java-checkerframe work Add @Pure for Java.\n" - " --gen-generated Add @Generated annotation for Java\n" - " --gen-jvmstatic Add @JvmStatic annotation for Kotlin methods\n" - " in companion object for interop from Java to Kotlin.\n" - " --gen-all Generate not just code for the current schema files,\n" - " but for all files it includes as well.\n" - " If the language uses a single file for output (by default\n" - " the case for C++ and JS), all code will end up in this one\n" - " file.\n" - " --cpp-include Adds an #include in generated file.\n" - " --cpp-ptr-type T Set object API pointer type (default std::unique_ptr).\n" - " --cpp-str-type T Set object API string type (default std::string).\n" - " T::c_str(), T::length() and T::empty() must be supported.\n" - " The custom type also needs to be constructible from std::string\n" - " (see the --cpp-str-flex-ctor option to change this behavior).\n" - " --cpp-str-flex-ctor Don't construct custom string types by passing std::string\n" - " from Flatbuffers, but (char* + length).\n" - " --cpp-std CPP_STD Generate a C++ code using features of selected C++ standard.\n" - " Supported CPP_STD values:\n" - " * 'c++0x' - generate code compatible with old compilers;\n" - " * 'c++11' - use C++11 code generator (default);\n" - " * 'c++17' - use C++17 features in generated code (experimental).\n" - " --cpp-static-reflection When using C++17, generate extra code to provide compile-time\n" - " (static) reflection of Flatbuffers types. Requires --cpp-std\n" - " to be \"c++17\" or higher.\n" - " --object-prefix Customise class prefix for C++ object-based API.\n" - " --object-suffix Customise class suffix for C++ object-based API.\n" - " Default value is \"T\".\n" - " --go-namespace Generate the overriding namespace in Golang.\n" - " --go-import Generate the overriding import for flatbuffers in Golang\n" - " (default is \"github.com/google/flatbuffers/go\").\n" - " --raw-binary Allow binaries without file_identifier to be read.\n" - " This may crash flatc given a mismatched schema.\n" - " --size-prefixed Input binaries are size prefixed buffers.\n" - " --proto Input is a .proto, translate to .fbs.\n" - " --proto-namespace-suffix Add this namespace to any flatbuffers generated\n" - " SUFFIX from protobufs.\n" - " --oneof-union Translate .proto oneofs to flatbuffer unions.\n" - " --grpc Generate GRPC interfaces for the specified languages.\n" - " --schema Serialize schemas instead of JSON (use with -b).\n" - " --bfbs-comments Add doc comments to the binary schema files.\n" - " --bfbs-builtins Add builtin attributes to the binary schema files.\n" - " --bfbs-gen-embed Generate code to embed the bfbs schema to the source.\n" - " --conform FILE Specify a schema the following schemas should be\n" - " an evolution of. Gives errors if not.\n" - " --conform-includes Include path for the schema given with --conform PATH\n" - " --filename-suffix The suffix appended to the generated file names.\n" - " Default is '_generated'.\n" - " --filename-ext The extension appended to the generated file names.\n" - " Default is language-specific (e.g., '.h' for C++)\n" - " --include-prefix Prefix this path to any generated include statements.\n" - " PATH\n" - " --keep-prefix Keep original prefix of schema include statement.\n" - " --reflect-types Add minimal type reflection to code generation.\n" - " --reflect-names Add minimal type/name reflection.\n" - " --root-type T Select or override the default root_type\n" - " --require-explicit-ids When parsing schemas, require explicit ids (id: x).\n" - " --force-defaults Emit default values in binary output from JSON\n" - " --force-empty When serializing from object API representation,\n" - " force strings and vectors to empty rather than null.\n" - " --force-empty-vectors When serializing from object API representation,\n" - " force vectors to empty rather than null.\n" - " --flexbuffers Used with \"binary\" and \"json\" options, it generates\n" - " data using schema-less FlexBuffers.\n" - " --no-warnings Inhibit all warning messages.\n" - "FILEs may be schemas (must end in .fbs), binary schemas (must end in .bfbs),\n" - "or JSON files (conforming to preceding schema). FILEs after the -- must be\n" - "binary flatbuffer format files.\n" - "Output files are named using the base file name of the input,\n" - "and written to the current directory or the path given by -o.\n" - "example: " << program_name << " -c -b schema1.fbs schema2.fbs data.json\n"; - // 12345678901234567890123456789012345678901234567890123456789012345678901234567890 - // clang-format on + ss << "\n"; + + std::string files_description = + "FILEs may be schemas (must end in .fbs), binary schemas (must end in " + ".bfbs) or JSON files (conforming to preceding schema). BINARY_FILEs " + "after the -- must be binary flatbuffer format files. Output files are " + "named using the base file name of the input, and written to the current " + "directory or the path given by -o. example: " + + program_name + " -c -b schema1.fbs schema2.fbs data.json"; + AppendTextWrappedString(ss, files_description, 80, 0); + ss << "\n"; return ss.str(); } -int FlatCompiler::Compile(int argc, const char **argv) { - if (params_.generators == nullptr || params_.num_generators == 0) { - return 0; +void FlatCompiler::AnnotateBinaries(const uint8_t *binary_schema, + const uint64_t binary_schema_size, + const FlatCOptions &options) { + const std::string &schema_filename = options.annotate_schema; + + for (const std::string &filename : options.filenames) { + std::string binary_contents; + if (!flatbuffers::LoadFile(filename.c_str(), true, &binary_contents)) { + Warn("unable to load binary file: " + filename); + continue; + } + + const uint8_t *binary = + reinterpret_cast<const uint8_t *>(binary_contents.c_str()); + const size_t binary_size = binary_contents.size(); + const bool is_size_prefixed = options.opts.size_prefixed; + + flatbuffers::BinaryAnnotator binary_annotator( + binary_schema, binary_schema_size, binary, binary_size, + is_size_prefixed); + + auto annotations = binary_annotator.Annotate(); + + flatbuffers::AnnotatedBinaryTextGenerator::Options text_gen_opts; + text_gen_opts.include_vector_contents = + options.annotate_include_vector_contents; + + // TODO(dbaileychess): Right now we just support a single text-based + // output of the annotated binary schema, which we generate here. We + // could output the raw annotations instead and have third-party tools + // use them to generate their own output. + flatbuffers::AnnotatedBinaryTextGenerator text_generator( + text_gen_opts, annotations, binary, binary_size); + + text_generator.Generate(filename, schema_filename); } +} + +FlatCOptions FlatCompiler::ParseFromCommandLineArguments(int argc, + const char **argv) { + if (argc <= 1) { Error("Need to provide at least one argument."); } + + FlatCOptions options; + + options.program_name = std::string(argv[0]); + + IDLOptions &opts = options.opts; - flatbuffers::IDLOptions opts; - std::string output_path; - - bool any_generator = false; - bool print_make_rules = false; - bool raw_binary = false; - bool schema_binary = false; - bool grpc_enabled = false; - std::vector<std::string> filenames; - std::list<std::string> include_directories_storage; - std::vector<const char *> include_directories; - std::vector<const char *> conform_include_directories; - std::vector<bool> generator_enabled(params_.num_generators, false); - size_t binary_files_from = std::numeric_limits<size_t>::max(); - std::string conform_to_schema; - - for (int argi = 0; argi < argc; argi++) { + for (int argi = 1; argi < argc; argi++) { std::string arg = argv[argi]; if (arg[0] == '-') { - if (filenames.size() && arg[1] != '-') + if (options.filenames.size() && arg[1] != '-') Error("invalid option location: " + arg, true); if (arg == "-o") { if (++argi >= argc) Error("missing path following: " + arg, true); - output_path = flatbuffers::ConCatPathFileName( + options.output_path = flatbuffers::ConCatPathFileName( flatbuffers::PosixPath(argv[argi]), ""); } else if (arg == "-I") { if (++argi >= argc) Error("missing path following: " + arg, true); - include_directories_storage.push_back( + options.include_directories_storage.push_back( flatbuffers::PosixPath(argv[argi])); - include_directories.push_back( - include_directories_storage.back().c_str()); + options.include_directories.push_back( + options.include_directories_storage.back().c_str()); + } else if (arg == "--bfbs-filenames") { + if (++argi > argc) Error("missing path following: " + arg, true); + opts.project_root = argv[argi]; + if (!DirExists(opts.project_root.c_str())) + Error(arg + " is not a directory: " + opts.project_root); } else if (arg == "--conform") { if (++argi >= argc) Error("missing path following: " + arg, true); - conform_to_schema = flatbuffers::PosixPath(argv[argi]); + options.conform_to_schema = flatbuffers::PosixPath(argv[argi]); } else if (arg == "--conform-includes") { if (++argi >= argc) Error("missing path following: " + arg, true); - include_directories_storage.push_back( + options.include_directories_storage.push_back( flatbuffers::PosixPath(argv[argi])); - conform_include_directories.push_back( - include_directories_storage.back().c_str()); + options.conform_include_directories.push_back( + options.include_directories_storage.back().c_str()); } else if (arg == "--include-prefix") { if (++argi >= argc) Error("missing path following: " + arg, true); opts.include_prefix = flatbuffers::ConCatPathFileName( flatbuffers::PosixPath(argv[argi]), ""); } else if (arg == "--keep-prefix") { - opts.keep_include_path = true; + opts.keep_prefix = true; } else if (arg == "--strict-json") { opts.strict_json = true; } else if (arg == "--allow-non-utf8") { @@ -243,15 +473,22 @@ int FlatCompiler::Compile(int argc, const char **argv) { } else if (arg == "--go-import") { if (++argi >= argc) Error("missing golang import" + arg, true); opts.go_import = argv[argi]; + } else if (arg == "--go-module-name") { + if (++argi >= argc) Error("missing golang module name" + arg, true); + opts.go_module_name = argv[argi]; } else if (arg == "--defaults-json") { opts.output_default_scalars_in_json = true; } else if (arg == "--unknown-json") { opts.skip_unexpected_fields_in_json = true; } else if (arg == "--no-prefix") { opts.prefixed_enums = false; + } else if (arg == "--cpp-minify-enums") { + opts.cpp_minify_enums = true; } else if (arg == "--scoped-enums") { opts.prefixed_enums = false; opts.scoped_enums = true; + } else if (arg == "--no-emit-min-max-enum-values") { + opts.emit_min_max_enum_values = false; } else if (arg == "--no-union-value-namespacing") { opts.union_value_namespacing = false; } else if (arg == "--gen-mutable") { @@ -275,12 +512,30 @@ int FlatCompiler::Compile(int argc, const char **argv) { opts.cpp_object_api_string_flexible_constructor = true; } else if (arg == "--no-cpp-direct-copy") { opts.cpp_direct_copy = false; + } else if (arg == "--cpp-field-case-style") { + if (++argi >= argc) Error("missing case style following: " + arg, true); + if (!strcmp(argv[argi], "unchanged")) + opts.cpp_object_api_field_case_style = + IDLOptions::CaseStyle_Unchanged; + else if (!strcmp(argv[argi], "upper")) + opts.cpp_object_api_field_case_style = IDLOptions::CaseStyle_Upper; + else if (!strcmp(argv[argi], "lower")) + opts.cpp_object_api_field_case_style = IDLOptions::CaseStyle_Lower; + else + Error("unknown case style: " + std::string(argv[argi]), true); } else if (arg == "--gen-nullable") { opts.gen_nullable = true; + } else if (arg == "--java-package-prefix") { + if (++argi >= argc) Error("missing prefix following: " + arg, true); + opts.java_package_prefix = argv[argi]; } else if (arg == "--java-checkerframework") { opts.java_checkerframework = true; } else if (arg == "--gen-generated") { opts.gen_generated = true; + } else if (arg == "--swift-implementation-only") { + opts.swift_implementation_only = true; + } else if (arg == "--gen-json-emit") { + opts.gen_json_coders = true; } else if (arg == "--object-prefix") { if (++argi >= argc) Error("missing prefix following: " + arg, true); opts.object_prefix = argv[argi]; @@ -297,28 +552,42 @@ int FlatCompiler::Compile(int argc, const char **argv) { opts.include_dependence_headers = false; } else if (arg == "--gen-onefile") { opts.one_file = true; + opts.include_dependence_headers = false; } else if (arg == "--raw-binary") { - raw_binary = true; + options.raw_binary = true; } else if (arg == "--size-prefixed") { opts.size_prefixed = true; } else if (arg == "--") { // Separator between text and binary inputs. - binary_files_from = filenames.size(); - } else if (arg == "--proto") { - opts.proto_mode = true; + options.binary_files_from = options.filenames.size(); } else if (arg == "--proto-namespace-suffix") { if (++argi >= argc) Error("missing namespace suffix" + arg, true); opts.proto_namespace_suffix = argv[argi]; } else if (arg == "--oneof-union") { opts.proto_oneof_union = true; + } else if (arg == "--keep-proto-id") { + opts.keep_proto_id = true; + } else if (arg == "--proto-id-gap") { + if (++argi >= argc) Error("missing case style following: " + arg, true); + if (!strcmp(argv[argi], "nop")) + opts.proto_id_gap_action = IDLOptions::ProtoIdGapAction::NO_OP; + else if (!strcmp(argv[argi], "warn")) + opts.proto_id_gap_action = IDLOptions::ProtoIdGapAction::WARNING; + else if (!strcmp(argv[argi], "error")) + opts.proto_id_gap_action = IDLOptions::ProtoIdGapAction::ERROR; + else + Error("unknown case style: " + std::string(argv[argi]), true); } else if (arg == "--schema") { - schema_binary = true; + options.schema_binary = true; } else if (arg == "-M") { - print_make_rules = true; + options.print_make_rules = true; } else if (arg == "--version") { printf("flatc version %s\n", FLATC_VERSION()); exit(0); + } else if (arg == "--help" || arg == "-h") { + printf("%s\n", GetUsageString(options.program_name).c_str()); + exit(0); } else if (arg == "--grpc") { - grpc_enabled = true; + options.grpc_enabled = true; } else if (arg == "--bfbs-comments") { opts.binary_schema_comments = true; } else if (arg == "--bfbs-builtins") { @@ -329,6 +598,10 @@ int FlatCompiler::Compile(int argc, const char **argv) { opts.mini_reflect = IDLOptions::kTypes; } else if (arg == "--reflect-names") { opts.mini_reflect = IDLOptions::kTypesAndNames; + } else if (arg == "--rust-serialize") { + opts.rust_serialize = true; + } else if (arg == "--rust-module-root-file") { + opts.rust_module_root_file = true; } else if (arg == "--require-explicit-ids") { opts.require_explicit_ids = true; } else if (arg == "--root-type") { @@ -357,6 +630,8 @@ int FlatCompiler::Compile(int argc, const char **argv) { opts.gen_jvmstatic = true; } else if (arg == "--no-warnings") { opts.no_warnings = true; + } else if (arg == "--warnings-as-errors") { + opts.warnings_as_errors = true; } else if (arg == "--cpp-std") { if (++argi >= argc) Error("missing C++ standard specification" + arg, true); @@ -365,69 +640,132 @@ int FlatCompiler::Compile(int argc, const char **argv) { opts.cpp_std = arg.substr(std::string("--cpp-std=").size()); } else if (arg == "--cpp-static-reflection") { opts.cpp_static_reflection = true; + } else if (arg == "--cs-global-alias") { + opts.cs_global_alias = true; + } else if (arg == "--json-nested-bytes") { + opts.json_nested_legacy_flatbuffers = true; + } else if (arg == "--ts-flat-files") { + opts.ts_flat_files = true; + opts.ts_entry_points = true; + opts.generate_all = true; + } else if (arg == "--ts-entry-points") { + opts.ts_entry_points = true; + opts.generate_all = true; + } else if (arg == "--ts-no-import-ext") { + opts.ts_no_import_ext = true; + } else if (arg == "--no-leak-private-annotation") { + opts.no_leak_private_annotations = true; + } else if (arg == "--python-no-type-prefix-suffix") { + opts.python_no_type_prefix_suffix = true; + } else if (arg == "--python-typing") { + opts.python_typing = true; + } else if (arg == "--annotate-sparse-vectors") { + options.annotate_include_vector_contents = false; + } else if (arg == "--annotate") { + if (++argi >= argc) Error("missing path following: " + arg, true); + options.annotate_schema = flatbuffers::PosixPath(argv[argi]); + } else if (arg == "--file-names-only") { + // TODO (khhn): Provide 2 implementation + options.file_names_only = true; } else { - for (size_t i = 0; i < params_.num_generators; ++i) { - if (arg == params_.generators[i].generator_opt_long || - (params_.generators[i].generator_opt_short && - arg == params_.generators[i].generator_opt_short)) { - generator_enabled[i] = true; - any_generator = true; - opts.lang_to_generate |= params_.generators[i].lang; - goto found; - } + if (arg == "--proto") { opts.proto_mode = true; } + + auto code_generator_it = code_generators_.find(arg); + if (code_generator_it == code_generators_.end()) { + Error("unknown commandline argument: " + arg, true); + return options; } - Error("unknown commandline argument: " + arg, true); - found:; + + std::shared_ptr<CodeGenerator> code_generator = + code_generator_it->second; + + // TODO(derekbailey): remove in favor of just checking if + // generators.empty(). + options.any_generator = true; + opts.lang_to_generate |= code_generator->Language(); + + auto is_binary_schema = code_generator->SupportsBfbsGeneration(); + opts.binary_schema_comments = is_binary_schema; + options.requires_bfbs = is_binary_schema; + options.generators.push_back(std::move(code_generator)); } } else { - filenames.push_back(flatbuffers::PosixPath(argv[argi])); + options.filenames.push_back(flatbuffers::PosixPath(argv[argi])); } } - if (!filenames.size()) Error("missing input files", false, true); + return options; +} + +void FlatCompiler::ValidateOptions(const FlatCOptions &options) { + const IDLOptions &opts = options.opts; + + if (!options.filenames.size()) Error("missing input files", false, true); if (opts.proto_mode) { - if (any_generator) + if (options.any_generator) Error("cannot generate code directly from .proto files", true); - } else if (!any_generator && conform_to_schema.empty()) { + } else if (!options.any_generator && options.conform_to_schema.empty() && + options.annotate_schema.empty()) { Error("no options: specify at least one generator.", true); } + if (opts.cs_gen_json_serializer && !opts.generate_object_based_api) { + Error( + "--cs-gen-json-serializer requires --gen-object-api to be set as " + "well."); + } +} + +flatbuffers::Parser FlatCompiler::GetConformParser( + const FlatCOptions &options) { flatbuffers::Parser conform_parser; - if (!conform_to_schema.empty()) { + if (!options.conform_to_schema.empty()) { std::string contents; - if (!flatbuffers::LoadFile(conform_to_schema.c_str(), true, &contents)) - Error("unable to load schema: " + conform_to_schema); + if (!flatbuffers::LoadFile(options.conform_to_schema.c_str(), true, + &contents)) { + Error("unable to load schema: " + options.conform_to_schema); + } - if (flatbuffers::GetExtension(conform_to_schema) == + if (flatbuffers::GetExtension(options.conform_to_schema) == reflection::SchemaExtension()) { - LoadBinarySchema(conform_parser, conform_to_schema, contents); + LoadBinarySchema(conform_parser, options.conform_to_schema, contents); } else { - ParseFile(conform_parser, conform_to_schema, contents, - conform_include_directories); + ParseFile(conform_parser, options.conform_to_schema, contents, + options.conform_include_directories); } } + return conform_parser; +} - std::unique_ptr<flatbuffers::Parser> parser(new flatbuffers::Parser(opts)); +std::unique_ptr<Parser> FlatCompiler::GenerateCode(const FlatCOptions &options, + Parser &conform_parser) { + std::unique_ptr<Parser> parser = + std::unique_ptr<Parser>(new Parser(options.opts)); + + for (auto file_it = options.filenames.begin(); + file_it != options.filenames.end(); ++file_it) { + IDLOptions opts = options.opts; - for (auto file_it = filenames.begin(); file_it != filenames.end(); - ++file_it) { auto &filename = *file_it; std::string contents; if (!flatbuffers::LoadFile(filename.c_str(), true, &contents)) Error("unable to load file: " + filename); - bool is_binary = - static_cast<size_t>(file_it - filenames.begin()) >= binary_files_from; + bool is_binary = static_cast<size_t>(file_it - options.filenames.begin()) >= + options.binary_files_from; auto ext = flatbuffers::GetExtension(filename); - auto is_schema = ext == "fbs" || ext == "proto"; - auto is_binary_schema = ext == reflection::SchemaExtension(); + const bool is_schema = ext == "fbs" || ext == "proto"; + if (is_schema && opts.project_root.empty()) { + opts.project_root = StripFileName(filename); + } + const bool is_binary_schema = ext == reflection::SchemaExtension(); if (is_binary) { parser->builder_.Clear(); parser->builder_.PushFlatBuffer( reinterpret_cast<const uint8_t *>(contents.c_str()), contents.length()); - if (!raw_binary) { + if (!options.raw_binary) { // Generally reading binaries that do not correspond to the schema // will crash, and sadly there's no way around that when the binary // does not contain a file identifier. @@ -453,26 +791,30 @@ int FlatCompiler::Compile(int argc, const char **argv) { contents.length() != strlen(contents.c_str())) { Error("input file appears to be binary: " + filename, true); } - if (is_schema) { + if (is_schema || is_binary_schema) { // If we're processing multiple schemas, make sure to start each // one from scratch. If it depends on previous schemas it must do // so explicitly using an include. - parser.reset(new flatbuffers::Parser(opts)); + parser.reset(new Parser(opts)); } + // Try to parse the file contents (binary schema/flexbuffer/textual + // schema) if (is_binary_schema) { - LoadBinarySchema(*parser.get(), filename, contents); - } - if (opts.use_flexbuffers) { + LoadBinarySchema(*parser, filename, contents); + } else if (opts.use_flexbuffers) { if (opts.lang_to_generate == IDLOptions::kJson) { - parser->flex_root_ = flexbuffers::GetRoot( - reinterpret_cast<const uint8_t *>(contents.c_str()), - contents.size()); + auto data = reinterpret_cast<const uint8_t *>(contents.c_str()); + auto size = contents.size(); + std::vector<uint8_t> reuse_tracker; + if (!flexbuffers::VerifyBuffer(data, size, &reuse_tracker)) + Error("flexbuffers file failed to verify: " + filename, false); + parser->flex_root_ = flexbuffers::GetRoot(data, size); } else { parser->flex_builder_.Clear(); - ParseFile(*parser.get(), filename, contents, include_directories); + ParseFile(*parser, filename, contents, options.include_directories); } } else { - ParseFile(*parser.get(), filename, contents, include_directories); + ParseFile(*parser, filename, contents, options.include_directories); if (!is_schema && !parser->builder_.GetSize()) { // If a file doesn't end in .fbs, it must be json/binary. Ensure we // didn't just parse a schema with a different extension. @@ -481,56 +823,78 @@ int FlatCompiler::Compile(int argc, const char **argv) { true); } } - if ((is_schema || is_binary_schema) && !conform_to_schema.empty()) { + if ((is_schema || is_binary_schema) && + !options.conform_to_schema.empty()) { auto err = parser->ConformTo(conform_parser); - if (!err.empty()) Error("schemas don\'t conform: " + err); + if (!err.empty()) Error("schemas don\'t conform: " + err, false); } - if (schema_binary || opts.binary_schema_gen_embed) { + if (options.schema_binary || opts.binary_schema_gen_embed) { parser->Serialize(); } - if (schema_binary) { + if (options.schema_binary) { parser->file_extension_ = reflection::SchemaExtension(); } } - std::string filebase = flatbuffers::StripPath(flatbuffers::StripExtension(filename)); - for (size_t i = 0; i < params_.num_generators; ++i) { - parser->opts.lang = params_.generators[i].lang; - if (generator_enabled[i]) { - if (!print_make_rules) { - flatbuffers::EnsureDirExists(output_path); - if ((!params_.generators[i].schema_only || - (is_schema || is_binary_schema)) && - !params_.generators[i].generate(*parser.get(), output_path, - filebase)) { - Error(std::string("Unable to generate ") + - params_.generators[i].lang_name + " for " + filebase); + // If one of the generators uses bfbs, serialize the parser and get + // the serialized buffer and length. + const uint8_t *bfbs_buffer = nullptr; + int64_t bfbs_length = 0; + if (options.requires_bfbs) { + parser->Serialize(); + bfbs_buffer = parser->builder_.GetBufferPointer(); + bfbs_length = parser->builder_.GetSize(); + } + + for (const std::shared_ptr<CodeGenerator> &code_generator : + options.generators) { + if (options.print_make_rules) { + std::string make_rule; + const CodeGenerator::Status status = code_generator->GenerateMakeRule( + *parser, options.output_path, filename, make_rule); + if (status == CodeGenerator::Status::OK && !make_rule.empty()) { + printf("%s\n", + flatbuffers::WordWrap(make_rule, 80, " ", " \\").c_str()); + } else { + Error("Cannot generate make rule for " + + code_generator->LanguageName()); + } + } else { + flatbuffers::EnsureDirExists(options.output_path); + + // Prefer bfbs generators if present. + if (code_generator->SupportsBfbsGeneration()) { + const CodeGenerator::Status status = + code_generator->GenerateCode(bfbs_buffer, bfbs_length); + if (status != CodeGenerator::Status::OK) { + Error("Unable to generate " + code_generator->LanguageName() + + " for " + filebase + code_generator->status_detail + + " using bfbs generator."); } } else { - if (params_.generators[i].make_rule == nullptr) { - Error(std::string("Cannot generate make rule for ") + - params_.generators[i].lang_name); - } else { - std::string make_rule = params_.generators[i].make_rule( - *parser.get(), output_path, filename); - if (!make_rule.empty()) - printf("%s\n", - flatbuffers::WordWrap(make_rule, 80, " ", " \\").c_str()); + if ((!code_generator->IsSchemaOnly() || + (is_schema || is_binary_schema)) && + code_generator->GenerateCode(*parser, options.output_path, + filebase) != + CodeGenerator::Status::OK) { + Error("Unable to generate " + code_generator->LanguageName() + + " for " + filebase + code_generator->status_detail); } } - if (grpc_enabled) { - if (params_.generators[i].generateGRPC != nullptr) { - if (!params_.generators[i].generateGRPC(*parser.get(), output_path, - filebase)) { - Error(std::string("Unable to generate GRPC interface for") + - params_.generators[i].lang_name); - } - } else { - Warn(std::string("GRPC interface generator not implemented for ") + - params_.generators[i].lang_name); - } + } + + if (options.grpc_enabled) { + const CodeGenerator::Status status = code_generator->GenerateGrpcCode( + *parser, options.output_path, filebase); + + if (status == CodeGenerator::Status::NOT_IMPLEMENTED) { + Warn("GRPC interface generator not implemented for " + + code_generator->LanguageName()); + } else if (status == CodeGenerator::Status::ERROR) { + Error("Unable to generate GRPC interface for " + + code_generator->LanguageName()); } } } @@ -542,13 +906,111 @@ int FlatCompiler::Compile(int argc, const char **argv) { Error("root type must be a table"); } - if (opts.proto_mode) GenerateFBS(*parser.get(), output_path, filebase); - // We do not want to generate code for the definitions in this file // in any files coming up next. parser->MarkGenerated(); } + + return parser; +} + +int FlatCompiler::Compile(const FlatCOptions &options) { + // TODO(derekbailey): change to std::optional<Parser> + Parser conform_parser = GetConformParser(options); + + // TODO(derekbailey): split to own method. + if (!options.annotate_schema.empty()) { + const std::string ext = flatbuffers::GetExtension(options.annotate_schema); + if (!(ext == reflection::SchemaExtension() || ext == "fbs")) { + Error("Expected a `.bfbs` or `.fbs` schema, got: " + + options.annotate_schema); + } + + const bool is_binary_schema = ext == reflection::SchemaExtension(); + + std::string schema_contents; + if (!flatbuffers::LoadFile(options.annotate_schema.c_str(), + /*binary=*/is_binary_schema, &schema_contents)) { + Error("unable to load schema: " + options.annotate_schema); + } + + const uint8_t *binary_schema = nullptr; + uint64_t binary_schema_size = 0; + + IDLOptions binary_opts; + binary_opts.lang_to_generate |= flatbuffers::IDLOptions::kBinary; + Parser parser(binary_opts); + + if (is_binary_schema) { + binary_schema = + reinterpret_cast<const uint8_t *>(schema_contents.c_str()); + binary_schema_size = schema_contents.size(); + } else { + // If we need to generate the .bfbs file from the provided schema file + // (.fbs) + ParseFile(parser, options.annotate_schema, schema_contents, + options.include_directories); + parser.Serialize(); + + binary_schema = parser.builder_.GetBufferPointer(); + binary_schema_size = parser.builder_.GetSize(); + } + + if (binary_schema == nullptr || !binary_schema_size) { + Error("could not parse a value binary schema from: " + + options.annotate_schema); + } + + // Annotate the provided files with the binary_schema. + AnnotateBinaries(binary_schema, binary_schema_size, options); + + // We don't support doing anything else after annotating a binary. + return 0; + } + + if (options.generators.empty()) { + Error("No generator registered"); + return -1; + } + + std::unique_ptr<Parser> parser = GenerateCode(options, conform_parser); + + for (const auto &code_generator : options.generators) { + if (code_generator->SupportsRootFileGeneration()) { + code_generator->GenerateRootFile(*parser, options.output_path); + } + } + return 0; } +bool FlatCompiler::RegisterCodeGenerator( + const FlatCOption &option, std::shared_ptr<CodeGenerator> code_generator) { + if (!option.short_opt.empty() && + code_generators_.find("-" + option.short_opt) != code_generators_.end()) { + Error("multiple generators registered under: -" + option.short_opt, false, + false); + return false; + } + + if (!option.short_opt.empty()) { + code_generators_["-" + option.short_opt] = code_generator; + } + + if (!option.long_opt.empty() && + code_generators_.find("--" + option.long_opt) != code_generators_.end()) { + Error("multiple generators registered under: --" + option.long_opt, false, + false); + return false; + } + + if (!option.long_opt.empty()) { + code_generators_["--" + option.long_opt] = code_generator; + } + + language_options.insert(option); + + return true; +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/flatc_main.cpp b/contrib/libs/flatbuffers/src/flatc_main.cpp index 31ccbc71851..f2dde958d37 100644 --- a/contrib/libs/flatbuffers/src/flatc_main.cpp +++ b/contrib/libs/flatbuffers/src/flatc_main.cpp @@ -14,8 +14,32 @@ * limitations under the License. */ +#include <cstdio> +#include <memory> + +#include "bfbs_gen_lua.h" +#include "bfbs_gen_nim.h" +#include "flatbuffers/base.h" +#include "flatbuffers/code_generator.h" #include "flatbuffers/flatc.h" #include "flatbuffers/util.h" +#include "idl_gen_binary.h" +#include "idl_gen_cpp.h" + #include "idl_gen_cpp_yandex_maps_iter.h" +#include "idl_gen_csharp.h" +#include "idl_gen_dart.h" +#include "idl_gen_fbs.h" +#include "idl_gen_go.h" +#include "idl_gen_java.h" +#include "idl_gen_json_schema.h" +#include "idl_gen_kotlin.h" +#include "idl_gen_lobster.h" +#include "idl_gen_php.h" +#include "idl_gen_python.h" +#include "idl_gen_rust.h" +#include "idl_gen_swift.h" +#include "idl_gen_text.h" +#include "idl_gen_ts.h" static const char *g_program_name = nullptr; @@ -23,16 +47,16 @@ static void Warn(const flatbuffers::FlatCompiler *flatc, const std::string &warn, bool show_exe_name) { (void)flatc; if (show_exe_name) { printf("%s: ", g_program_name); } - printf("warning: %s\n", warn.c_str()); + fprintf(stderr, "\nwarning:\n %s\n\n", warn.c_str()); } static void Error(const flatbuffers::FlatCompiler *flatc, const std::string &err, bool usage, bool show_exe_name) { if (show_exe_name) { printf("%s: ", g_program_name); } - printf("error: %s\n", err.c_str()); if (usage && flatc) { - printf("%s", flatc->GetUsageString(g_program_name).c_str()); + fprintf(stderr, "%s\n", flatc->GetShortUsageString(g_program_name).c_str()); } + fprintf(stderr, "\nerror:\n %s\n\n", err.c_str()); exit(1); } @@ -47,75 +71,115 @@ void LogCompilerError(const std::string &err) { } // namespace flatbuffers int main(int argc, const char *argv[]) { - // Prevent Appveyor-CI hangs. - flatbuffers::SetupDefaultCRTReportMode(); + const std::string flatbuffers_version(flatbuffers::FLATBUFFERS_VERSION()); g_program_name = argv[0]; - const flatbuffers::FlatCompiler::Generator generators[] = { - { flatbuffers::GenerateBinary, "-b", "--binary", "binary", false, nullptr, - flatbuffers::IDLOptions::kBinary, - "Generate wire format binaries for any data definitions", - flatbuffers::BinaryMakeRule }, - { flatbuffers::GenerateTextFile, "-t", "--json", "text", false, nullptr, - flatbuffers::IDLOptions::kJson, - "Generate text output for any data definitions", - flatbuffers::TextMakeRule }, - { flatbuffers::GenerateCPP, "-c", "--cpp", "C++", true, - flatbuffers::GenerateCppGRPC, flatbuffers::IDLOptions::kCpp, - "Generate C++ headers for tables/structs", flatbuffers::CPPMakeRule }, - { flatbuffers::GenerateGo, "-g", "--go", "Go", true, - flatbuffers::GenerateGoGRPC, flatbuffers::IDLOptions::kGo, - "Generate Go files for tables/structs", nullptr }, - { flatbuffers::GenerateJava, "-j", "--java", "Java", true, - flatbuffers::GenerateJavaGRPC, flatbuffers::IDLOptions::kJava, - "Generate Java classes for tables/structs", - flatbuffers::JavaCSharpMakeRule }, - { flatbuffers::GenerateDart, "-d", "--dart", "Dart", true, nullptr, - flatbuffers::IDLOptions::kDart, - "Generate Dart classes for tables/structs", flatbuffers::DartMakeRule }, - { flatbuffers::GenerateTS, "-T", "--ts", "TypeScript", true, - flatbuffers::GenerateTSGRPC, flatbuffers::IDLOptions::kTs, - "Generate TypeScript code for tables/structs", flatbuffers::TSMakeRule }, - { flatbuffers::GenerateCSharp, "-n", "--csharp", "C#", true, nullptr, - flatbuffers::IDLOptions::kCSharp, - "Generate C# classes for tables/structs", - flatbuffers::JavaCSharpMakeRule }, - { flatbuffers::GeneratePython, "-p", "--python", "Python", true, - flatbuffers::GeneratePythonGRPC, flatbuffers::IDLOptions::kPython, - "Generate Python files for tables/structs", nullptr }, - { flatbuffers::GenerateLobster, nullptr, "--lobster", "Lobster", true, - nullptr, flatbuffers::IDLOptions::kLobster, - "Generate Lobster files for tables/structs", nullptr }, - { flatbuffers::GenerateLua, "-l", "--lua", "Lua", true, nullptr, - flatbuffers::IDLOptions::kLua, "Generate Lua files for tables/structs", - nullptr }, - { flatbuffers::GenerateRust, "-r", "--rust", "Rust", true, nullptr, - flatbuffers::IDLOptions::kRust, "Generate Rust files for tables/structs", - flatbuffers::RustMakeRule }, - { flatbuffers::GeneratePhp, nullptr, "--php", "PHP", true, nullptr, - flatbuffers::IDLOptions::kPhp, "Generate PHP files for tables/structs", - nullptr }, - { flatbuffers::GenerateKotlin, nullptr, "--kotlin", "Kotlin", true, nullptr, - flatbuffers::IDLOptions::kKotlin, - "Generate Kotlin classes for tables/structs", nullptr }, - { flatbuffers::GenerateJsonSchema, nullptr, "--jsonschema", "JsonSchema", - true, nullptr, flatbuffers::IDLOptions::kJsonSchema, - "Generate Json schema", nullptr }, - { flatbuffers::GenerateSwift, nullptr, "--swift", "swift", true, - flatbuffers::GenerateSwiftGRPC, flatbuffers::IDLOptions::kSwift, - "Generate Swift files for tables/structs", nullptr }, - { flatbuffers::GenerateCPPYandexMapsIter, nullptr, "--yandex-maps-iter", "C++Iter", - true, nullptr, flatbuffers::IDLOptions::kCppYandexMapsIter, - "Generate C++ template headers for tables/structs", nullptr }, - }; - flatbuffers::FlatCompiler::InitParams params; - params.generators = generators; - params.num_generators = sizeof(generators) / sizeof(generators[0]); params.warn_fn = Warn; params.error_fn = Error; flatbuffers::FlatCompiler flatc(params); - return flatc.Compile(argc - 1, argv + 1); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ + "b", "binary", "", + "Generate wire format binaries for any data definitions" }, + flatbuffers::NewBinaryCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "c", "cpp", "", + "Generate C++ headers for tables/structs" }, + flatbuffers::NewCppCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "n", "csharp", "", + "Generate C# classes for tables/structs" }, + flatbuffers::NewCSharpCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "d", "dart", "", + "Generate Dart classes for tables/structs" }, + flatbuffers::NewDartCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "", "proto", "", + "Input is a .proto, translate to .fbs" }, + flatbuffers::NewFBSCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "g", "go", "", + "Generate Go files for tables/structs" }, + flatbuffers::NewGoCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "j", "java", "", + "Generate Java classes for tables/structs" }, + flatbuffers::NewJavaCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "", "jsonschema", "", "Generate Json schema" }, + flatbuffers::NewJsonSchemaCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "", "kotlin", "", + "Generate Kotlin classes for tables/structs" }, + flatbuffers::NewKotlinCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "", "lobster", "", + "Generate Lobster files for tables/structs" }, + flatbuffers::NewLobsterCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "l", "lua", "", + "Generate Lua files for tables/structs" }, + flatbuffers::NewLuaBfbsGenerator(flatbuffers_version)); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "", "nim", "", + "Generate Nim files for tables/structs" }, + flatbuffers::NewNimBfbsGenerator(flatbuffers_version)); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "p", "python", "", + "Generate Python files for tables/structs" }, + flatbuffers::NewPythonCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "", "php", "", + "Generate PHP files for tables/structs" }, + flatbuffers::NewPhpCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "r", "rust", "", + "Generate Rust files for tables/structs" }, + flatbuffers::NewRustCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ + "t", "json", "", "Generate text output for any data definitions" }, + flatbuffers::NewTextCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "", "swift", "", + "Generate Swift files for tables/structs" }, + flatbuffers::NewSwiftCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "", "yandex-maps-iter", "", + "Generate C++ template headers for tables/structs" }, + flatbuffers::NewCppYandexMapsIterCodeGenerator()); + + flatc.RegisterCodeGenerator( + flatbuffers::FlatCOption{ "T", "ts", "", + "Generate TypeScript code for tables/structs" }, + flatbuffers::NewTsCodeGenerator()); + + // Create the FlatC options by parsing the command line arguments. + const flatbuffers::FlatCOptions &options = + flatc.ParseFromCommandLineArguments(argc, argv); + + // Compile with the extracted FlatC options. + return flatc.Compile(options); } diff --git a/contrib/libs/flatbuffers/src/idl_gen_binary.cpp b/contrib/libs/flatbuffers/src/idl_gen_binary.cpp new file mode 100644 index 00000000000..feb4e2f55e7 --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_binary.cpp @@ -0,0 +1,92 @@ +/* + * Copyright 2014 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// independent from idl_parser, since this code is not needed for most clients + +#include "idl_gen_binary.h" + +#include <limits> +#include <memory> +#include <string> +#include <unordered_set> + +#include "flatbuffers/base.h" +#include "flatbuffers/code_generators.h" +#include "flatbuffers/flatbuffers.h" +#include "flatbuffers/flatc.h" +#include "flatbuffers/idl.h" +#include "flatbuffers/util.h" + +namespace flatbuffers { + +namespace { + +class BinaryCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateBinary(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + // Generate code from the provided `buffer` of given `length`. The buffer is a + // serialized reflection.fbs. + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + output = BinaryMakeRule(parser, path, filename); + return Status::OK; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return false; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kBinary; } + + std::string LanguageName() const override { return "binary"; } +}; + +} // namespace + +std::unique_ptr<CodeGenerator> NewBinaryCodeGenerator() { + return std::unique_ptr<BinaryCodeGenerator>(new BinaryCodeGenerator()); +} + +} // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_binary.h b/contrib/libs/flatbuffers/src/idl_gen_binary.h new file mode 100644 index 00000000000..a7c93b9d9b0 --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_binary.h @@ -0,0 +1,32 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_BINARY_H_ +#define FLATBUFFERS_IDL_GEN_BINARY_H_ + +#include <memory> +#include <string> + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Binary code generator. +std::unique_ptr<CodeGenerator> NewBinaryCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_BINARY_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_cpp.cpp b/contrib/libs/flatbuffers/src/idl_gen_cpp.cpp index a33697eaed1..6e22c9d2195 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_cpp.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_cpp.cpp @@ -16,8 +16,15 @@ // independent from idl_parser, since this code is not needed for most clients +#include "idl_gen_cpp.h" + +#include <limits> +#include <memory> +#include <string> #include <unordered_set> +#include <utility> +#include "flatbuffers/base.h" #include "flatbuffers/code_generators.h" #include "flatbuffers/flatbuffers.h" #include "flatbuffers/flatc.h" @@ -57,9 +64,8 @@ static std::string GenIncludeGuard(const std::string &file_name, guard = "FLATBUFFERS_GENERATED_" + guard; guard += "_"; // For further uniqueness, also add the namespace. - for (auto it = name_space.components.begin(); - it != name_space.components.end(); ++it) { - guard += *it + "_"; + for (const std::string &component : name_space.components) { + guard += component + "_"; } // Anything extra to add to the guard? if (!postfix.empty()) { guard += postfix + "_"; } @@ -68,6 +74,18 @@ static std::string GenIncludeGuard(const std::string &file_name, return guard; } +static bool IsVectorOfPointers(const FieldDef &field) { + const auto &type = field.value.type; + const auto &vector_type = type.VectorType(); + return IsVector(type) && vector_type.base_type == BASE_TYPE_STRUCT && + !vector_type.struct_def->fixed && !field.native_inline; +} + +static bool IsPointer(const FieldDef &field) { + return field.value.type.base_type == BASE_TYPE_STRUCT && + !IsStruct(field.value.type); +} + namespace cpp { enum CppStandard { CPP_STD_X0 = 0, CPP_STD_11, CPP_STD_17 }; @@ -75,7 +93,7 @@ enum CppStandard { CPP_STD_X0 = 0, CPP_STD_11, CPP_STD_17 }; // Define a style of 'struct' constructor if it has 'Array' fields. enum GenArrayArgMode { kArrayArgModeNone, // don't generate initialization args - kArrayArgModeSpanStatic, // generate flatbuffers::span<T,N> + kArrayArgModeSpanStatic, // generate ::flatbuffers::span<T,N> }; // Extension of IDLOptions for cpp-generator. @@ -88,6 +106,21 @@ struct IDLOptionsCpp : public IDLOptions { : IDLOptions(opts), g_cpp_std(CPP_STD_11), g_only_fixed_enums(true) {} }; +// Iterates over all the fields of the object first by Offset type (Offset64 +// before Offset32) and then by definition order. +static void ForAllFieldsOrderedByOffset( + const StructDef &object, std::function<void(const FieldDef *field)> func) { + // Loop over all the fields and call the func on all offset64 fields. + for (const FieldDef *field_def : object.fields.vec) { + if (field_def->offset64) { func(field_def); } + } + // Loop over all the fields a second time and call the func on all offset + // fields. + for (const FieldDef *field_def : object.fields.vec) { + if (!field_def->offset64) { func(field_def); } + } +} + class CppGenerator : public BaseGenerator { public: CppGenerator(const Parser &parser, const std::string &path, @@ -199,40 +232,125 @@ class CppGenerator : public BaseGenerator { for (auto kw = keywords; *kw; kw++) keywords_.insert(*kw); } + // Adds code to check that the included flatbuffers.h is of the same version + // as the generated code. This check currently looks for exact version match, + // as we would guarantee that they are compatible, but in theory a newer + // version of flatbuffers.h should work with a old code gen if we do proper + // backwards support. + void GenFlatbuffersVersionCheck() { + code_ += + "// Ensure the included flatbuffers.h is the same version as when this " + "file was"; + code_ += "// generated, otherwise it may not be compatible."; + code_ += "static_assert(FLATBUFFERS_VERSION_MAJOR == " + + std::to_string(FLATBUFFERS_VERSION_MAJOR) + " &&"; + code_ += " FLATBUFFERS_VERSION_MINOR == " + + std::to_string(FLATBUFFERS_VERSION_MINOR) + " &&"; + code_ += " FLATBUFFERS_VERSION_REVISION == " + + std::to_string(FLATBUFFERS_VERSION_REVISION) + ","; + code_ += " \"Non-compatible flatbuffers version included\");"; + } + void GenIncludeDependencies() { - int num_includes = 0; if (opts_.generate_object_based_api) { - for (auto it = parser_.native_included_files_.begin(); - it != parser_.native_included_files_.end(); ++it) { - code_ += "#include \"" + *it + "\""; - num_includes++; + for (const std::string &native_included_file : + parser_.native_included_files_) { + code_ += "#include \"" + native_included_file + "\""; } } - for (auto it = parser_.included_files_.begin(); - it != parser_.included_files_.end(); ++it) { - if (it->second.empty()) continue; - auto noext = flatbuffers::StripExtension(it->second); - auto basename = flatbuffers::StripPath(noext); - auto includeName = - GeneratedFileName(opts_.include_prefix, - opts_.keep_include_path ? noext : basename, opts_); - code_ += "#include \"" + includeName + "\""; - num_includes++; + + // Get the directly included file of the file being parsed. + std::vector<IncludedFile> included_files(parser_.GetIncludedFiles()); + + // We are safe to sort them alphabetically, since there shouldn't be any + // interdependence between them. + std::stable_sort(included_files.begin(), included_files.end()); + + for (const IncludedFile &included_file : included_files) { + // Get the name of the included file as defined by the schema, and strip + // the .fbs extension. + const std::string name_without_ext = + StripExtension(included_file.schema_name); + + // If we are told to keep the prefix of the included schema, leave it + // unchanged, otherwise strip the leading path off so just the "basename" + // of the include is retained. + const std::string basename = + opts_.keep_prefix ? name_without_ext : StripPath(name_without_ext); + + code_ += "#include \"" + + GeneratedFileName(opts_.include_prefix, basename, opts_) + "\""; + } + + if (!parser_.native_included_files_.empty() || !included_files.empty()) { + code_ += ""; } - if (num_includes) code_ += ""; + } + + void MarkIf64BitBuilderIsNeeded() { + if (needs_64_bit_builder_) { return; } + for (auto t : parser_.structs_.vec) { + if (t == nullptr) continue; + for (auto f : t->fields.vec) { + if (f == nullptr) continue; + if (f->offset64) { + needs_64_bit_builder_ = true; + break; + } + } + } + } + + std::string GetBuilder() { + return std::string("::flatbuffers::FlatBufferBuilder") + + (needs_64_bit_builder_ ? "64" : ""); } void GenExtraIncludes() { - for (std::size_t i = 0; i < opts_.cpp_includes.size(); ++i) { - code_ += "#include \"" + opts_.cpp_includes[i] + "\""; + for (const std::string &cpp_include : opts_.cpp_includes) { + code_ += "#include \"" + cpp_include + "\""; } if (!opts_.cpp_includes.empty()) { code_ += ""; } } + void GenEmbeddedIncludes() { + if (parser_.opts.binary_schema_gen_embed && parser_.root_struct_def_) { + const std::string file_path = + GeneratedFileName(opts_.include_prefix, file_name_ + "_bfbs", opts_); + code_ += "// For access to the binary schema that produced this file."; + code_ += "#include \"" + file_path + "\""; + code_ += ""; + } + } + std::string EscapeKeyword(const std::string &name) const { return keywords_.find(name) == keywords_.end() ? name : name + "_"; } + std::string Name(const FieldDef &field) const { + // the union type field suffix is immutable. + static size_t union_suffix_len = strlen(UnionTypeFieldSuffix()); + const bool is_union_type = field.value.type.base_type == BASE_TYPE_UTYPE; + // early return if no case transformation required + if (opts_.cpp_object_api_field_case_style == + IDLOptions::CaseStyle_Unchanged) + return EscapeKeyword(field.name); + std::string name = field.name; + // do not change the case style of the union type field suffix + if (is_union_type) { + FLATBUFFERS_ASSERT(name.length() > union_suffix_len); + name.erase(name.length() - union_suffix_len, union_suffix_len); + } + if (opts_.cpp_object_api_field_case_style == IDLOptions::CaseStyle_Upper) + name = ConvertCase(name, Case::kUpperCamel); + else if (opts_.cpp_object_api_field_case_style == + IDLOptions::CaseStyle_Lower) + name = ConvertCase(name, Case::kLowerCamel); + // restore the union field type suffix + if (is_union_type) name.append(UnionTypeFieldSuffix(), union_suffix_len); + return EscapeKeyword(name); + } + std::string Name(const Definition &def) const { return EscapeKeyword(def.name); } @@ -259,6 +377,11 @@ class CppGenerator : public BaseGenerator { code_ += "#pragma clang system_header\n\n"; } + code_ += "#include \"flatbuffers/flatbuffers.h\""; + code_ += ""; + GenFlatbuffersVersionCheck(); + code_ += ""; + SetNameSpace(struct_def.defined_namespace); auto name = Name(struct_def); code_.SetValue("STRUCT_NAME", name); @@ -306,6 +429,9 @@ class CppGenerator : public BaseGenerator { // Iterate through all definitions we haven't generate code for (enums, // structs, and tables) and output them to a single file. bool generate() { + // Check if we require a 64-bit flatbuffer builder. + MarkIf64BitBuilderIsNeeded(); + code_.Clear(); code_ += "// " + std::string(FlatBuffersGeneratedWarning()) + "\n\n"; @@ -320,28 +446,30 @@ class CppGenerator : public BaseGenerator { code_ += "#include <contrib/libs/flatbuffers/include/flatbuffers/flatbuffers.h>"; if (parser_.uses_flexbuffers_) { code_ += "#include <contrib/libs/flatbuffers/include/flatbuffers/flexbuffers.h>"; + code_ += "#include \"flatbuffers/flex_flat_util.h\""; } code_ += ""; + GenFlatbuffersVersionCheck(); + code_ += ""; if (opts_.include_dependence_headers) { GenIncludeDependencies(); } GenExtraIncludes(); + GenEmbeddedIncludes(); FLATBUFFERS_ASSERT(!cur_name_space_); // Generate forward declarations for all structs/tables, since they may // have circular references. - for (auto it = parser_.structs_.vec.begin(); - it != parser_.structs_.vec.end(); ++it) { - const auto &struct_def = **it; - if (!struct_def.generated) { - SetNameSpace(struct_def.defined_namespace); - code_ += "struct " + Name(struct_def) + ";"; - if (!struct_def.fixed) { - code_ += "struct " + Name(struct_def) + "Builder;"; + for (const auto &struct_def : parser_.structs_.vec) { + if (!struct_def->generated) { + SetNameSpace(struct_def->defined_namespace); + code_ += "struct " + Name(*struct_def) + ";"; + if (!struct_def->fixed) { + code_ += "struct " + Name(*struct_def) + "Builder;"; } if (opts_.generate_object_based_api) { - auto nativeName = NativeName(Name(struct_def), &struct_def, opts_); - if (!struct_def.fixed) { code_ += "struct " + nativeName + ";"; } + auto nativeName = NativeName(Name(*struct_def), struct_def, opts_); + if (!struct_def->fixed) { code_ += "struct " + nativeName + ";"; } } code_ += ""; } @@ -349,12 +477,10 @@ class CppGenerator : public BaseGenerator { // Generate forward declarations for all equal operators if (opts_.generate_object_based_api && opts_.gen_compare) { - for (auto it = parser_.structs_.vec.begin(); - it != parser_.structs_.vec.end(); ++it) { - const auto &struct_def = **it; - if (!struct_def.generated) { - SetNameSpace(struct_def.defined_namespace); - auto nativeName = NativeName(Name(struct_def), &struct_def, opts_); + for (const auto &struct_def : parser_.structs_.vec) { + if (!struct_def->generated) { + SetNameSpace(struct_def->defined_namespace); + auto nativeName = NativeName(Name(*struct_def), struct_def, opts_); code_ += "bool operator==(const " + nativeName + " &lhs, const " + nativeName + " &rhs);"; code_ += "bool operator!=(const " + nativeName + " &lhs, const " + @@ -367,80 +493,64 @@ class CppGenerator : public BaseGenerator { // Generate preablmle code for mini reflection. if (opts_.mini_reflect != IDLOptions::kNone) { // To break cyclic dependencies, first pre-declare all tables/structs. - for (auto it = parser_.structs_.vec.begin(); - it != parser_.structs_.vec.end(); ++it) { - const auto &struct_def = **it; - if (!struct_def.generated) { - SetNameSpace(struct_def.defined_namespace); - GenMiniReflectPre(&struct_def); + for (const auto &struct_def : parser_.structs_.vec) { + if (!struct_def->generated) { + SetNameSpace(struct_def->defined_namespace); + GenMiniReflectPre(struct_def); } } } // Generate code for all the enum declarations. - for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end(); - ++it) { - const auto &enum_def = **it; - if (!enum_def.generated) { - SetNameSpace(enum_def.defined_namespace); - GenEnum(enum_def); + for (const auto &enum_def : parser_.enums_.vec) { + if (!enum_def->generated) { + SetNameSpace(enum_def->defined_namespace); + GenEnum(*enum_def); } } // Generate code for all structs, then all tables. - for (auto it = parser_.structs_.vec.begin(); - it != parser_.structs_.vec.end(); ++it) { - const auto &struct_def = **it; - if (struct_def.fixed && !struct_def.generated) { - SetNameSpace(struct_def.defined_namespace); - GenStruct(struct_def); + for (const auto &struct_def : parser_.structs_.vec) { + if (struct_def->fixed && !struct_def->generated) { + SetNameSpace(struct_def->defined_namespace); + GenStruct(*struct_def); } } - for (auto it = parser_.structs_.vec.begin(); - it != parser_.structs_.vec.end(); ++it) { - const auto &struct_def = **it; - if (!struct_def.fixed && !struct_def.generated) { - SetNameSpace(struct_def.defined_namespace); - GenTable(struct_def); + for (const auto &struct_def : parser_.structs_.vec) { + if (!struct_def->fixed && !struct_def->generated) { + SetNameSpace(struct_def->defined_namespace); + GenTable(*struct_def); } } - for (auto it = parser_.structs_.vec.begin(); - it != parser_.structs_.vec.end(); ++it) { - const auto &struct_def = **it; - if (!struct_def.fixed && !struct_def.generated) { - SetNameSpace(struct_def.defined_namespace); - GenTablePost(struct_def); + for (const auto &struct_def : parser_.structs_.vec) { + if (!struct_def->fixed && !struct_def->generated) { + SetNameSpace(struct_def->defined_namespace); + GenTablePost(*struct_def); } } // Generate code for union verifiers. - for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end(); - ++it) { - const auto &enum_def = **it; - if (enum_def.is_union && !enum_def.generated) { - SetNameSpace(enum_def.defined_namespace); - GenUnionPost(enum_def); + for (const auto &enum_def : parser_.enums_.vec) { + if (enum_def->is_union && !enum_def->generated) { + SetNameSpace(enum_def->defined_namespace); + GenUnionPost(*enum_def); } } // Generate code for mini reflection. if (opts_.mini_reflect != IDLOptions::kNone) { // Then the unions/enums that may refer to them. - for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end(); - ++it) { - const auto &enum_def = **it; - if (!enum_def.generated) { - SetNameSpace(enum_def.defined_namespace); - GenMiniReflect(nullptr, &enum_def); + for (const auto &enum_def : parser_.enums_.vec) { + if (!enum_def->generated) { + SetNameSpace(enum_def->defined_namespace); + GenMiniReflect(nullptr, enum_def); } } // Then the full tables/structs. - for (auto it = parser_.structs_.vec.begin(); - it != parser_.structs_.vec.end(); ++it) { - const auto &struct_def = **it; - if (!struct_def.generated) { - SetNameSpace(struct_def.defined_namespace); - GenMiniReflect(&struct_def, nullptr); + for (const auto &struct_def : parser_.structs_.vec) { + if (!struct_def->generated) { + SetNameSpace(struct_def->defined_namespace); + GenMiniReflect(struct_def, nullptr); } } } @@ -456,13 +566,15 @@ class CppGenerator : public BaseGenerator { code_.SetValue("STRUCT_NAME", name); code_.SetValue("CPP_NAME", cpp_name); code_.SetValue("NULLABLE_EXT", NullableExtension()); + code_.SetValue( + "SIZE_T", needs_64_bit_builder_ ? ",::flatbuffers::uoffset64_t" : ""); // The root datatype accessor: code_ += "inline \\"; code_ += "const {{CPP_NAME}} *{{NULLABLE_EXT}}Get{{STRUCT_NAME}}(const void " "*buf) {"; - code_ += " return flatbuffers::GetRoot<{{CPP_NAME}}>(buf);"; + code_ += " return ::flatbuffers::GetRoot<{{CPP_NAME}}>(buf);"; code_ += "}"; code_ += ""; @@ -471,14 +583,29 @@ class CppGenerator : public BaseGenerator { "const {{CPP_NAME}} " "*{{NULLABLE_EXT}}GetSizePrefixed{{STRUCT_NAME}}(const void " "*buf) {"; - code_ += " return flatbuffers::GetSizePrefixedRoot<{{CPP_NAME}}>(buf);"; + code_ += + " return " + "::flatbuffers::GetSizePrefixedRoot<{{CPP_NAME}}{{SIZE_T}}>(buf);"; code_ += "}"; code_ += ""; if (opts_.mutable_buffer) { code_ += "inline \\"; code_ += "{{STRUCT_NAME}} *GetMutable{{STRUCT_NAME}}(void *buf) {"; - code_ += " return flatbuffers::GetMutableRoot<{{STRUCT_NAME}}>(buf);"; + code_ += + " return ::flatbuffers::GetMutableRoot<{{STRUCT_NAME}}>(buf);"; + code_ += "}"; + code_ += ""; + + code_ += "inline \\"; + code_ += + "{{CPP_NAME}} " + "*{{NULLABLE_EXT}}GetMutableSizePrefixed{{STRUCT_NAME}}(void " + "*buf) {"; + code_ += + " return " + "::flatbuffers::GetMutableSizePrefixedRoot<{{CPP_NAME}}{{SIZE_T}}>(" + "buf);"; code_ += "}"; code_ += ""; } @@ -493,10 +620,20 @@ class CppGenerator : public BaseGenerator { // Check if a buffer has the identifier. code_ += "inline \\"; code_ += "bool {{STRUCT_NAME}}BufferHasIdentifier(const void *buf) {"; - code_ += " return flatbuffers::BufferHasIdentifier("; + code_ += " return ::flatbuffers::BufferHasIdentifier("; code_ += " buf, {{STRUCT_NAME}}Identifier());"; code_ += "}"; code_ += ""; + + // Check if a size-prefixed buffer has the identifier. + code_ += "inline \\"; + code_ += + "bool SizePrefixed{{STRUCT_NAME}}BufferHasIdentifier(const void " + "*buf) {"; + code_ += " return ::flatbuffers::BufferHasIdentifier("; + code_ += " buf, {{STRUCT_NAME}}Identifier(), true);"; + code_ += "}"; + code_ += ""; } // The root verifier. @@ -507,15 +644,16 @@ class CppGenerator : public BaseGenerator { } code_ += "inline bool Verify{{STRUCT_NAME}}Buffer("; - code_ += " flatbuffers::Verifier &verifier) {"; + code_ += " ::flatbuffers::Verifier &verifier) {"; code_ += " return verifier.VerifyBuffer<{{CPP_NAME}}>({{ID}});"; code_ += "}"; code_ += ""; code_ += "inline bool VerifySizePrefixed{{STRUCT_NAME}}Buffer("; - code_ += " flatbuffers::Verifier &verifier) {"; + code_ += " ::flatbuffers::Verifier &verifier) {"; code_ += - " return verifier.VerifySizePrefixedBuffer<{{CPP_NAME}}>({{ID}});"; + " return " + "verifier.VerifySizePrefixedBuffer<{{CPP_NAME}}{{SIZE_T}}>({{ID}});"; code_ += "}"; code_ += ""; @@ -529,8 +667,8 @@ class CppGenerator : public BaseGenerator { // Finish a buffer with a given root object: code_ += "inline void Finish{{STRUCT_NAME}}Buffer("; - code_ += " flatbuffers::FlatBufferBuilder &fbb,"; - code_ += " flatbuffers::Offset<{{CPP_NAME}}> root) {"; + code_ += " " + GetBuilder() + " &fbb,"; + code_ += " ::flatbuffers::Offset<{{CPP_NAME}}> root) {"; if (parser_.file_identifier_.length()) code_ += " fbb.Finish(root, {{STRUCT_NAME}}Identifier());"; else @@ -539,8 +677,8 @@ class CppGenerator : public BaseGenerator { code_ += ""; code_ += "inline void FinishSizePrefixed{{STRUCT_NAME}}Buffer("; - code_ += " flatbuffers::FlatBufferBuilder &fbb,"; - code_ += " flatbuffers::Offset<{{CPP_NAME}}> root) {"; + code_ += " " + GetBuilder() + " &fbb,"; + code_ += " ::flatbuffers::Offset<{{CPP_NAME}}> root) {"; if (parser_.file_identifier_.length()) code_ += " fbb.FinishSizePrefixed(root, {{STRUCT_NAME}}Identifier());"; else @@ -558,7 +696,8 @@ class CppGenerator : public BaseGenerator { code_ += "inline {{UNPACK_RETURN}} UnPack{{STRUCT_NAME}}("; code_ += " const void *buf,"; - code_ += " const flatbuffers::resolver_function_t *res = nullptr) {"; + code_ += + " const ::flatbuffers::resolver_function_t *res = nullptr) {"; code_ += " return {{UNPACK_TYPE}}\\"; code_ += "(Get{{STRUCT_NAME}}(buf)->UnPack(res));"; code_ += "}"; @@ -566,7 +705,8 @@ class CppGenerator : public BaseGenerator { code_ += "inline {{UNPACK_RETURN}} UnPackSizePrefixed{{STRUCT_NAME}}("; code_ += " const void *buf,"; - code_ += " const flatbuffers::resolver_function_t *res = nullptr) {"; + code_ += + " const ::flatbuffers::resolver_function_t *res = nullptr) {"; code_ += " return {{UNPACK_TYPE}}\\"; code_ += "(GetSizePrefixed{{STRUCT_NAME}}(buf)->UnPack(res));"; code_ += "}"; @@ -597,6 +737,7 @@ class CppGenerator : public BaseGenerator { const IDLOptionsCpp opts_; const TypedFloatConstantGenerator float_const_gen_; + bool needs_64_bit_builder_ = false; const Namespace *CurrentNameSpace() const { return cur_name_space_; } @@ -614,17 +755,16 @@ class CppGenerator : public BaseGenerator { bool TypeHasKey(const Type &type) { if (type.base_type != BASE_TYPE_STRUCT) { return false; } - for (auto it = type.struct_def->fields.vec.begin(); - it != type.struct_def->fields.vec.end(); ++it) { - const auto &field = **it; - if (field.key) { return true; } + for (auto &field : type.struct_def->fields.vec) { + if (field->key) { return true; } } return false; } bool VectorElementUserFacing(const Type &type) const { - return opts_.g_cpp_std >= cpp::CPP_STD_17 && opts_.g_only_fixed_enums && - IsEnum(type); + return (opts_.scoped_enums && IsEnum(type)) || + (opts_.g_cpp_std >= cpp::CPP_STD_17 && opts_.g_only_fixed_enums && + IsEnum(type)); } void GenComment(const std::vector<std::string> &dc, const char *prefix = "") { @@ -635,19 +775,11 @@ class CppGenerator : public BaseGenerator { // Return a C++ type from the table in idl.h std::string GenTypeBasic(const Type &type, bool user_facing_type) const { - // clang-format off - static const char *const ctypename[] = { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \ - #CTYPE, - FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) - #undef FLATBUFFERS_TD - }; - // clang-format on if (user_facing_type) { if (type.enum_def) return WrapInNameSpace(*type.enum_def); if (type.base_type == BASE_TYPE_BOOL) return "bool"; } - return ctypename[type.base_type]; + return StringOf(type.base_type); } // Return a C++ pointer type, specialized to the actual struct/table types, @@ -655,12 +787,16 @@ class CppGenerator : public BaseGenerator { std::string GenTypePointer(const Type &type) const { switch (type.base_type) { case BASE_TYPE_STRING: { - return "flatbuffers::String"; + return "::flatbuffers::String"; } + case BASE_TYPE_VECTOR64: case BASE_TYPE_VECTOR: { const auto type_name = GenTypeWire( type.VectorType(), "", VectorElementUserFacing(type.VectorType())); - return "flatbuffers::Vector<" + type_name + ">"; + return "::flatbuffers::Vector" + + std::string((type.base_type == BASE_TYPE_VECTOR64) ? "64<" + : "<") + + type_name + ">"; } case BASE_TYPE_STRUCT: { return WrapInNameSpace(*type.struct_def); @@ -676,13 +812,15 @@ class CppGenerator : public BaseGenerator { // Return a C++ type for any type (scalar/pointer) specifically for // building a flatbuffer. std::string GenTypeWire(const Type &type, const char *postfix, - bool user_facing_type) const { + bool user_facing_type, + bool _64_bit_offset = false) const { if (IsScalar(type.base_type)) { return GenTypeBasic(type, user_facing_type) + postfix; } else if (IsStruct(type)) { return "const " + GenTypePointer(type) + " *"; } else { - return "flatbuffers::Offset<" + GenTypePointer(type) + ">" + postfix; + return "::flatbuffers::Offset" + std::string(_64_bit_offset ? "64" : "") + + "<" + GenTypePointer(type) + ">" + postfix; } } @@ -694,7 +832,7 @@ class CppGenerator : public BaseGenerator { } else if (IsStruct(type)) { return GenTypePointer(type); } else { - return "flatbuffers::uoffset_t"; + return "::flatbuffers::uoffset_t"; } } @@ -727,9 +865,8 @@ class CppGenerator : public BaseGenerator { } bool FlexibleStringConstructor(const FieldDef *field) { - auto attr = field - ? (field->attributes.Lookup("cpp_str_flex_ctor") != nullptr) - : false; + auto attr = field != nullptr && + (field->attributes.Lookup("cpp_str_flex_ctor") != nullptr); auto ret = attr ? attr : opts_.cpp_object_api_string_flexible_constructor; return ret && NativeString(field) != "std::string"; // Only for custom string types. @@ -757,18 +894,19 @@ class CppGenerator : public BaseGenerator { return ptr_type == "naked" ? "" : ".get()"; } - std::string GenOptionalNull() { return "flatbuffers::nullopt"; } + std::string GenOptionalNull() { return "::flatbuffers::nullopt"; } std::string GenOptionalDecl(const Type &type) { - return "flatbuffers::Optional<" + GenTypeBasic(type, true) + ">"; + return "::flatbuffers::Optional<" + GenTypeBasic(type, true) + ">"; } std::string GenTypeNative(const Type &type, bool invector, - const FieldDef &field) { + const FieldDef &field, bool forcopy = false) { switch (type.base_type) { case BASE_TYPE_STRING: { return NativeString(&field); } + case BASE_TYPE_VECTOR64: case BASE_TYPE_VECTOR: { const auto type_name = GenTypeNative(type.VectorType(), true, field); if (type.struct_def && @@ -777,23 +915,25 @@ class CppGenerator : public BaseGenerator { type.struct_def->attributes.Lookup("native_custom_alloc"); return "std::vector<" + type_name + "," + native_custom_alloc->constant + "<" + type_name + ">>"; - } else + } else { return "std::vector<" + type_name + ">"; + } } case BASE_TYPE_STRUCT: { auto type_name = WrapInNameSpace(*type.struct_def); if (IsStruct(type)) { auto native_type = type.struct_def->attributes.Lookup("native_type"); if (native_type) { type_name = native_type->constant; } - if (invector || field.native_inline) { + if (invector || field.native_inline || forcopy) { return type_name; } else { return GenTypeNativePtr(type_name, &field, false); } } else { - return GenTypeNativePtr( - WrapNativeNameInNameSpace(*type.struct_def, opts_), &field, - false); + const auto nn = WrapNativeNameInNameSpace(*type.struct_def, opts_); + return (forcopy || field.native_inline) + ? nn + : GenTypeNativePtr(nn, &field, false); } } case BASE_TYPE_UNION: { @@ -833,10 +973,10 @@ class CppGenerator : public BaseGenerator { } std::string GenTypeSpan(const Type &type, bool immutable, size_t extent) { - // Generate "flatbuffers::span<const U, extent>". + // Generate "::flatbuffers::span<const U, extent>". FLATBUFFERS_ASSERT(IsSeries(type) && "unexpected type"); auto element_type = type.VectorType(); - std::string text = "flatbuffers::span<"; + std::string text = "::flatbuffers::span<"; text += immutable ? "const " : ""; if (IsScalar(element_type.base_type)) { text += GenTypeBasic(element_type, IsEnum(element_type)); @@ -856,7 +996,7 @@ class CppGenerator : public BaseGenerator { break; } } - if (extent != flatbuffers::dynamic_extent) { + if (extent != dynamic_extent) { text += ", "; text += NumToString(extent); } @@ -887,13 +1027,13 @@ class CppGenerator : public BaseGenerator { std::string GetUnionElement(const EnumVal &ev, bool native_type, const IDLOptions &opts) { if (ev.union_type.base_type == BASE_TYPE_STRUCT) { - auto name = ev.union_type.struct_def->name; + std::string name = ev.union_type.struct_def->name; if (native_type) { - name = NativeName(name, ev.union_type.struct_def, opts); + name = NativeName(std::move(name), ev.union_type.struct_def, opts); } return WrapInNameSpace(ev.union_type.struct_def->defined_namespace, name); } else if (IsString(ev.union_type)) { - return native_type ? "std::string" : "flatbuffers::String"; + return native_type ? "std::string" : "::flatbuffers::String"; } else { FLATBUFFERS_ASSERT(false); return Name(ev); @@ -902,48 +1042,51 @@ class CppGenerator : public BaseGenerator { std::string UnionVerifySignature(const EnumDef &enum_def) { return "bool Verify" + Name(enum_def) + - "(flatbuffers::Verifier &verifier, const void *obj, " + + "(::flatbuffers::Verifier &verifier, const void *obj, " + Name(enum_def) + " type)"; } std::string UnionVectorVerifySignature(const EnumDef &enum_def) { - return "bool Verify" + Name(enum_def) + "Vector" + - "(flatbuffers::Verifier &verifier, " + - "const flatbuffers::Vector<flatbuffers::Offset<void>> *values, " + - "const flatbuffers::Vector<uint8_t> *types)"; + const std::string name = Name(enum_def); + const std::string &type = opts_.scoped_enums ? name : "uint8_t"; + return "bool Verify" + name + "Vector" + + "(::flatbuffers::Verifier &verifier, " + + "const ::flatbuffers::Vector<::flatbuffers::Offset<void>> " + "*values, " + + "const ::flatbuffers::Vector<" + type + "> *types)"; } std::string UnionUnPackSignature(const EnumDef &enum_def, bool inclass) { return (inclass ? "static " : "") + std::string("void *") + (inclass ? "" : Name(enum_def) + "Union::") + "UnPack(const void *obj, " + Name(enum_def) + - " type, const flatbuffers::resolver_function_t *resolver)"; + " type, const ::flatbuffers::resolver_function_t *resolver)"; } std::string UnionPackSignature(const EnumDef &enum_def, bool inclass) { - return "flatbuffers::Offset<void> " + - (inclass ? "" : Name(enum_def) + "Union::") + - "Pack(flatbuffers::FlatBufferBuilder &_fbb, " + - "const flatbuffers::rehasher_function_t *_rehasher" + + return "::flatbuffers::Offset<void> " + + (inclass ? "" : Name(enum_def) + "Union::") + "Pack(" + + GetBuilder() + " &_fbb, " + + "const ::flatbuffers::rehasher_function_t *_rehasher" + (inclass ? " = nullptr" : "") + ") const"; } std::string TableCreateSignature(const StructDef &struct_def, bool predecl, const IDLOptions &opts) { - return "flatbuffers::Offset<" + Name(struct_def) + "> Create" + - Name(struct_def) + "(flatbuffers::FlatBufferBuilder &_fbb, const " + + return "::flatbuffers::Offset<" + Name(struct_def) + "> Create" + + Name(struct_def) + "(" + GetBuilder() + " &_fbb, const " + NativeName(Name(struct_def), &struct_def, opts) + - " *_o, const flatbuffers::rehasher_function_t *_rehasher" + + " *_o, const ::flatbuffers::rehasher_function_t *_rehasher" + (predecl ? " = nullptr" : "") + ")"; } std::string TablePackSignature(const StructDef &struct_def, bool inclass, const IDLOptions &opts) { - return std::string(inclass ? "static " : "") + "flatbuffers::Offset<" + + return std::string(inclass ? "static " : "") + "::flatbuffers::Offset<" + Name(struct_def) + "> " + (inclass ? "" : Name(struct_def) + "::") + - "Pack(flatbuffers::FlatBufferBuilder &_fbb, " + "const " + + "Pack(" + GetBuilder() + " &_fbb, " + "const " + NativeName(Name(struct_def), &struct_def, opts) + "* _o, " + - "const flatbuffers::rehasher_function_t *_rehasher" + + "const ::flatbuffers::rehasher_function_t *_rehasher" + (inclass ? " = nullptr" : "") + ")"; } @@ -951,7 +1094,7 @@ class CppGenerator : public BaseGenerator { const IDLOptions &opts) { return NativeName(Name(struct_def), &struct_def, opts) + " *" + (inclass ? "" : Name(struct_def) + "::") + - "UnPack(const flatbuffers::resolver_function_t *_resolver" + + "UnPack(const ::flatbuffers::resolver_function_t *_resolver" + (inclass ? " = nullptr" : "") + ") const"; } @@ -959,13 +1102,13 @@ class CppGenerator : public BaseGenerator { const IDLOptions &opts) { return "void " + (inclass ? "" : Name(struct_def) + "::") + "UnPackTo(" + NativeName(Name(struct_def), &struct_def, opts) + " *" + - "_o, const flatbuffers::resolver_function_t *_resolver" + + "_o, const ::flatbuffers::resolver_function_t *_resolver" + (inclass ? " = nullptr" : "") + ") const"; } void GenMiniReflectPre(const StructDef *struct_def) { code_.SetValue("NAME", struct_def->name); - code_ += "inline const flatbuffers::TypeTable *{{NAME}}TypeTable();"; + code_ += "inline const ::flatbuffers::TypeTable *{{NAME}}TypeTable();"; code_ += ""; } @@ -981,11 +1124,9 @@ class CppGenerator : public BaseGenerator { std::vector<Type> types; if (struct_def) { - for (auto it = struct_def->fields.vec.begin(); - it != struct_def->fields.vec.end(); ++it) { - const auto &field = **it; - names.push_back(Name(field)); - types.push_back(field.value.type); + for (const auto &field : struct_def->fields.vec) { + names.push_back(Name(*field)); + types.push_back(field->value.type); } } else { for (auto it = enum_def->Vals().begin(); it != enum_def->Vals().end(); @@ -999,8 +1140,7 @@ class CppGenerator : public BaseGenerator { std::string ts; std::vector<std::string> type_refs; std::vector<uint16_t> array_sizes; - for (auto it = types.begin(); it != types.end(); ++it) { - auto &type = *it; + for (auto &type : types) { if (!ts.empty()) ts += ",\n "; auto is_vector = IsVector(type); auto is_array = IsArray(type); @@ -1009,10 +1149,9 @@ class CppGenerator : public BaseGenerator { ? bt - BASE_TYPE_UTYPE + ET_UTYPE : ET_SEQUENCE; int ref_idx = -1; - std::string ref_name = - type.struct_def - ? WrapInNameSpace(*type.struct_def) - : type.enum_def ? WrapInNameSpace(*type.enum_def) : ""; + std::string ref_name = type.struct_def ? WrapInNameSpace(*type.struct_def) + : type.enum_def ? WrapInNameSpace(*type.enum_def) + : ""; if (!ref_name.empty()) { auto rit = type_refs.begin(); for (; rit != type_refs.end(); ++rit) { @@ -1027,24 +1166,24 @@ class CppGenerator : public BaseGenerator { } } if (is_array) { array_sizes.push_back(type.fixed_length); } - ts += "{ flatbuffers::" + std::string(ElementaryTypeNames()[et]) + ", " + - NumToString(is_vector || is_array) + ", " + NumToString(ref_idx) + - " }"; + ts += "{ ::flatbuffers::" + std::string(ElementaryTypeNames()[et]) + + ", " + NumToString(is_vector || is_array) + ", " + + NumToString(ref_idx) + " }"; } std::string rs; - for (auto it = type_refs.begin(); it != type_refs.end(); ++it) { + for (auto &type_ref : type_refs) { if (!rs.empty()) rs += ",\n "; - rs += *it + "TypeTable"; + rs += type_ref + "TypeTable"; } std::string as; - for (auto it = array_sizes.begin(); it != array_sizes.end(); ++it) { - as += NumToString(*it); + for (auto &array_size : array_sizes) { + as += NumToString(array_size); as += ", "; } std::string ns; - for (auto it = names.begin(); it != names.end(); ++it) { + for (auto &name : names) { if (!ns.empty()) ns += ",\n "; - ns += "\"" + *it + "\""; + ns += "\"" + name + "\""; } std::string vs; const auto consecutive_enum_from_zero = @@ -1059,10 +1198,8 @@ class CppGenerator : public BaseGenerator { enum_def->underlying_type.base_type); } } else if (struct_def && struct_def->fixed) { - for (auto it = struct_def->fields.vec.begin(); - it != struct_def->fields.vec.end(); ++it) { - const auto &field = **it; - vs += NumToString(field.value.offset); + for (const auto field : struct_def->fields.vec) { + vs += NumToString(field->value.offset); vs += ", "; } vs += NumToString(struct_def->bytesize); @@ -1072,14 +1209,14 @@ class CppGenerator : public BaseGenerator { code_.SetValue("ARRAYSIZES", as); code_.SetValue("NAMES", ns); code_.SetValue("VALUES", vs); - code_ += "inline const flatbuffers::TypeTable *{{NAME}}TypeTable() {"; + code_ += "inline const ::flatbuffers::TypeTable *{{NAME}}TypeTable() {"; if (num_fields) { - code_ += " static const flatbuffers::TypeCode type_codes[] = {"; + code_ += " static const ::flatbuffers::TypeCode type_codes[] = {"; code_ += " {{TYPES}}"; code_ += " };"; } if (!type_refs.empty()) { - code_ += " static const flatbuffers::TypeFunction type_refs[] = {"; + code_ += " static const ::flatbuffers::TypeFunction type_refs[] = {"; code_ += " {{REFS}}"; code_ += " };"; } @@ -1097,8 +1234,8 @@ class CppGenerator : public BaseGenerator { code_ += " {{NAMES}}"; code_ += " };"; } - code_ += " static const flatbuffers::TypeTable tt = {"; - code_ += std::string(" flatbuffers::{{SEQ_TYPE}}, {{NUM_FIELDS}}, ") + + code_ += " static const ::flatbuffers::TypeTable tt = {"; + code_ += std::string(" ::flatbuffers::{{SEQ_TYPE}}, {{NUM_FIELDS}}, ") + (num_fields ? "type_codes, " : "nullptr, ") + (!type_refs.empty() ? "type_refs, " : "nullptr, ") + (!as.empty() ? "array_sizes, " : "nullptr, ") + @@ -1126,17 +1263,21 @@ class CppGenerator : public BaseGenerator { code_.SetValue("SEP", ","); auto add_sep = false; - for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { - const auto &ev = **it; + for (const auto ev : enum_def.Vals()) { if (add_sep) code_ += "{{SEP}}"; - GenComment(ev.doc_comment, " "); - code_.SetValue("KEY", GenEnumValDecl(enum_def, Name(ev))); + GenComment(ev->doc_comment, " "); + code_.SetValue("KEY", GenEnumValDecl(enum_def, Name(*ev))); code_.SetValue("VALUE", - NumToStringCpp(enum_def.ToString(ev), + NumToStringCpp(enum_def.ToString(*ev), enum_def.underlying_type.base_type)); code_ += " {{KEY}} = {{VALUE}}\\"; add_sep = true; } + if (opts_.cpp_minify_enums) { + code_ += ""; + code_ += "};"; + return; + } const EnumVal *minv = enum_def.MinValue(); const EnumVal *maxv = enum_def.MaxValue(); @@ -1144,6 +1285,8 @@ class CppGenerator : public BaseGenerator { FLATBUFFERS_ASSERT(minv && maxv); code_.SetValue("SEP", ",\n"); + + // MIN & MAX are useless for bit_flags if (enum_def.attributes.Lookup("bit_flags")) { code_.SetValue("KEY", GenEnumValDecl(enum_def, "NONE")); code_.SetValue("VALUE", "0"); @@ -1154,7 +1297,7 @@ class CppGenerator : public BaseGenerator { NumToStringCpp(enum_def.AllFlags(), enum_def.underlying_type.base_type)); code_ += "{{SEP}} {{KEY}} = {{VALUE}}\\"; - } else { // MIN & MAX are useless for bit_flags + } else if (opts_.emit_min_max_enum_values) { code_.SetValue("KEY", GenEnumValDecl(enum_def, "MIN")); code_.SetValue("VALUE", GenEnumValDecl(enum_def, Name(*minv))); code_ += "{{SEP}} {{KEY}} = {{VALUE}}\\"; @@ -1172,8 +1315,173 @@ class CppGenerator : public BaseGenerator { "FLATBUFFERS_DEFINE_BITMASK_OPERATORS({{ENUM_NAME}}, {{BASE_TYPE}})"; } code_ += ""; + GenEnumArray(enum_def); + GenEnumStringTable(enum_def); + + // Generate type traits for unions to map from a type to union enum value. + if (enum_def.is_union && !enum_def.uses_multiple_type_instances) { + for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); + ++it) { + const auto &ev = **it; + + if (it == enum_def.Vals().begin()) { + code_ += "template<typename T> struct {{ENUM_NAME}}Traits {"; + } else { + auto name = GetUnionElement(ev, false, opts_); + code_ += "template<> struct {{ENUM_NAME}}Traits<" + name + "> {"; + } + + auto value = GetEnumValUse(enum_def, ev); + code_ += " static const {{ENUM_NAME}} enum_value = " + value + ";"; + code_ += "};"; + code_ += ""; + } + } - // Generate an array of all enumeration values + GenEnumObjectBasedAPI(enum_def); + + if (enum_def.is_union) { + code_ += UnionVerifySignature(enum_def) + ";"; + code_ += UnionVectorVerifySignature(enum_def) + ";"; + code_ += ""; + } + } + + // Generate a union type and a trait type for it. + void GenEnumObjectBasedAPI(const EnumDef &enum_def) { + if (!(opts_.generate_object_based_api && enum_def.is_union)) { return; } + code_.SetValue("NAME", Name(enum_def)); + FLATBUFFERS_ASSERT(enum_def.Lookup("NONE")); + code_.SetValue("NONE", GetEnumValUse(enum_def, *enum_def.Lookup("NONE"))); + + if (!enum_def.uses_multiple_type_instances) { + for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); + ++it) { + const auto &ev = **it; + + if (it == enum_def.Vals().begin()) { + code_ += "template<typename T> struct {{NAME}}UnionTraits {"; + } else { + auto name = GetUnionElement(ev, true, opts_); + code_ += "template<> struct {{NAME}}UnionTraits<" + name + "> {"; + } + + auto value = GetEnumValUse(enum_def, ev); + code_ += " static const {{ENUM_NAME}} enum_value = " + value + ";"; + code_ += "};"; + code_ += ""; + } + } + + code_ += "struct {{NAME}}Union {"; + code_ += " {{NAME}} type;"; + code_ += " void *value;"; + code_ += ""; + code_ += " {{NAME}}Union() : type({{NONE}}), value(nullptr) {}"; + code_ += " {{NAME}}Union({{NAME}}Union&& u) FLATBUFFERS_NOEXCEPT :"; + code_ += " type({{NONE}}), value(nullptr)"; + code_ += " { std::swap(type, u.type); std::swap(value, u.value); }"; + code_ += " {{NAME}}Union(const {{NAME}}Union &);"; + code_ += " {{NAME}}Union &operator=(const {{NAME}}Union &u)"; + code_ += + " { {{NAME}}Union t(u); std::swap(type, t.type); std::swap(value, " + "t.value); return *this; }"; + code_ += + " {{NAME}}Union &operator=({{NAME}}Union &&u) FLATBUFFERS_NOEXCEPT"; + code_ += + " { std::swap(type, u.type); std::swap(value, u.value); return " + "*this; }"; + code_ += " ~{{NAME}}Union() { Reset(); }"; + code_ += ""; + code_ += " void Reset();"; + code_ += ""; + if (!enum_def.uses_multiple_type_instances) { + code_ += " template <typename T>"; + code_ += " void Set(T&& val) {"; + code_ += " typedef typename std::remove_reference<T>::type RT;"; + code_ += " Reset();"; + code_ += " type = {{NAME}}UnionTraits<RT>::enum_value;"; + code_ += " if (type != {{NONE}}) {"; + code_ += " value = new RT(std::forward<T>(val));"; + code_ += " }"; + code_ += " }"; + code_ += ""; + } + code_ += " " + UnionUnPackSignature(enum_def, true) + ";"; + code_ += " " + UnionPackSignature(enum_def, true) + ";"; + code_ += ""; + + for (const auto ev : enum_def.Vals()) { + if (ev->IsZero()) { continue; } + + const auto native_type = GetUnionElement(*ev, true, opts_); + code_.SetValue("NATIVE_TYPE", native_type); + code_.SetValue("NATIVE_NAME", Name(*ev)); + code_.SetValue("NATIVE_ID", GetEnumValUse(enum_def, *ev)); + + code_ += " {{NATIVE_TYPE}} *As{{NATIVE_NAME}}() {"; + code_ += " return type == {{NATIVE_ID}} ?"; + code_ += " reinterpret_cast<{{NATIVE_TYPE}} *>(value) : nullptr;"; + code_ += " }"; + + code_ += " const {{NATIVE_TYPE}} *As{{NATIVE_NAME}}() const {"; + code_ += " return type == {{NATIVE_ID}} ?"; + code_ += + " reinterpret_cast<const {{NATIVE_TYPE}} *>(value) : nullptr;"; + code_ += " }"; + } + code_ += "};"; + code_ += ""; + + GenEnumEquals(enum_def); + } + + void GenEnumEquals(const EnumDef &enum_def) { + if (opts_.gen_compare) { + code_ += ""; + code_ += + "inline bool operator==(const {{NAME}}Union &lhs, const " + "{{NAME}}Union &rhs) {"; + code_ += " if (lhs.type != rhs.type) return false;"; + code_ += " switch (lhs.type) {"; + + for (const auto &ev : enum_def.Vals()) { + code_.SetValue("NATIVE_ID", GetEnumValUse(enum_def, *ev)); + if (ev->IsNonZero()) { + const auto native_type = GetUnionElement(*ev, true, opts_); + code_.SetValue("NATIVE_TYPE", native_type); + code_ += " case {{NATIVE_ID}}: {"; + code_ += + " return *(reinterpret_cast<const {{NATIVE_TYPE}} " + "*>(lhs.value)) =="; + code_ += + " *(reinterpret_cast<const {{NATIVE_TYPE}} " + "*>(rhs.value));"; + code_ += " }"; + } else { + code_ += " case {{NATIVE_ID}}: {"; + code_ += " return true;"; // "NONE" enum value. + code_ += " }"; + } + } + code_ += " default: {"; + code_ += " return false;"; + code_ += " }"; + code_ += " }"; + code_ += "}"; + + code_ += ""; + code_ += + "inline bool operator!=(const {{NAME}}Union &lhs, const " + "{{NAME}}Union &rhs) {"; + code_ += " return !(lhs == rhs);"; + code_ += "}"; + code_ += ""; + } + } + + // Generate an array of all enumeration values + void GenEnumArray(const EnumDef &enum_def) { auto num_fields = NumToString(enum_def.size()); code_ += "inline const {{ENUM_NAME}} (&EnumValues{{ENUM_NAME}}())[" + num_fields + "] {"; @@ -1188,11 +1496,13 @@ class CppGenerator : public BaseGenerator { code_ += " return values;"; code_ += "}"; code_ += ""; + } - // Generate a generate string table for enum values. - // Problem is, if values are very sparse that could generate really big - // tables. Ideally in that case we generate a map lookup instead, but for - // the moment we simply don't output a table at all. + // Generate a string table for enum values. + // Problem is, if values are very sparse that could generate huge tables. + // Ideally in that case we generate a map lookup instead, but for the moment + // we simply don't output a table at all. + void GenEnumStringTable(const EnumDef &enum_def) { auto range = enum_def.Distance(); // Average distance between values above which we consider a table // "too sparse". Change at will. @@ -1203,14 +1513,12 @@ class CppGenerator : public BaseGenerator { NumToString(range + 1 + 1) + "] = {"; auto val = enum_def.Vals().front(); - for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); - ++it) { - auto ev = *it; - for (auto k = enum_def.Distance(val, ev); k > 1; --k) { + for (const auto &enum_value : enum_def.Vals()) { + for (auto k = enum_def.Distance(val, enum_value); k > 1; --k) { code_ += " \"\","; } - val = ev; - code_ += " \"" + Name(*ev) + "\","; + val = enum_value; + code_ += " \"" + Name(*enum_value) + "\","; } code_ += " nullptr"; code_ += " };"; @@ -1221,7 +1529,7 @@ class CppGenerator : public BaseGenerator { code_ += "inline const char *EnumName{{ENUM_NAME}}({{ENUM_NAME}} e) {"; - code_ += " if (flatbuffers::IsOutRange(e, " + + code_ += " if (::flatbuffers::IsOutRange(e, " + GetEnumValUse(enum_def, *enum_def.MinValue()) + ", " + GetEnumValUse(enum_def, *enum_def.MaxValue()) + ")) return \"\";"; @@ -1238,164 +1546,16 @@ class CppGenerator : public BaseGenerator { code_ += ""; } else { code_ += "inline const char *EnumName{{ENUM_NAME}}({{ENUM_NAME}} e) {"; - code_ += " switch (e) {"; - - for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); - ++it) { - const auto &ev = **it; - code_ += " case " + GetEnumValUse(enum_def, ev) + ": return \"" + - Name(ev) + "\";"; + for (const auto &ev : enum_def.Vals()) { + code_ += " case " + GetEnumValUse(enum_def, *ev) + ": return \"" + + Name(*ev) + "\";"; } - code_ += " default: return \"\";"; code_ += " }"; - code_ += "}"; code_ += ""; } - - // Generate type traits for unions to map from a type to union enum value. - if (enum_def.is_union && !enum_def.uses_multiple_type_instances) { - for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); - ++it) { - const auto &ev = **it; - - if (it == enum_def.Vals().begin()) { - code_ += "template<typename T> struct {{ENUM_NAME}}Traits {"; - } else { - auto name = GetUnionElement(ev, false, opts_); - code_ += "template<> struct {{ENUM_NAME}}Traits<" + name + "> {"; - } - - auto value = GetEnumValUse(enum_def, ev); - code_ += " static const {{ENUM_NAME}} enum_value = " + value + ";"; - code_ += "};"; - code_ += ""; - } - } - - if (opts_.generate_object_based_api && enum_def.is_union) { - // Generate a union type - code_.SetValue("NAME", Name(enum_def)); - FLATBUFFERS_ASSERT(enum_def.Lookup("NONE")); - code_.SetValue("NONE", GetEnumValUse(enum_def, *enum_def.Lookup("NONE"))); - - code_ += "struct {{NAME}}Union {"; - code_ += " {{NAME}} type;"; - code_ += " void *value;"; - code_ += ""; - code_ += " {{NAME}}Union() : type({{NONE}}), value(nullptr) {}"; - code_ += " {{NAME}}Union({{NAME}}Union&& u) FLATBUFFERS_NOEXCEPT :"; - code_ += " type({{NONE}}), value(nullptr)"; - code_ += " { std::swap(type, u.type); std::swap(value, u.value); }"; - code_ += " {{NAME}}Union(const {{NAME}}Union &);"; - code_ += " {{NAME}}Union &operator=(const {{NAME}}Union &u)"; - code_ += - " { {{NAME}}Union t(u); std::swap(type, t.type); std::swap(value, " - "t.value); return *this; }"; - code_ += - " {{NAME}}Union &operator=({{NAME}}Union &&u) FLATBUFFERS_NOEXCEPT"; - code_ += - " { std::swap(type, u.type); std::swap(value, u.value); return " - "*this; }"; - code_ += " ~{{NAME}}Union() { Reset(); }"; - code_ += ""; - code_ += " void Reset();"; - code_ += ""; - if (!enum_def.uses_multiple_type_instances) { - code_ += "#ifndef FLATBUFFERS_CPP98_STL"; - code_ += " template <typename T>"; - code_ += " void Set(T&& val) {"; - code_ += " using RT = typename std::remove_reference<T>::type;"; - code_ += " Reset();"; - code_ += - " type = {{NAME}}Traits<typename RT::TableType>::enum_value;"; - code_ += " if (type != {{NONE}}) {"; - code_ += " value = new RT(std::forward<T>(val));"; - code_ += " }"; - code_ += " }"; - code_ += "#endif // FLATBUFFERS_CPP98_STL"; - code_ += ""; - } - code_ += " " + UnionUnPackSignature(enum_def, true) + ";"; - code_ += " " + UnionPackSignature(enum_def, true) + ";"; - code_ += ""; - - for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); - ++it) { - const auto &ev = **it; - if (ev.IsZero()) { continue; } - - const auto native_type = GetUnionElement(ev, true, opts_); - code_.SetValue("NATIVE_TYPE", native_type); - code_.SetValue("NATIVE_NAME", Name(ev)); - code_.SetValue("NATIVE_ID", GetEnumValUse(enum_def, ev)); - - code_ += " {{NATIVE_TYPE}} *As{{NATIVE_NAME}}() {"; - code_ += " return type == {{NATIVE_ID}} ?"; - code_ += " reinterpret_cast<{{NATIVE_TYPE}} *>(value) : nullptr;"; - code_ += " }"; - - code_ += " const {{NATIVE_TYPE}} *As{{NATIVE_NAME}}() const {"; - code_ += " return type == {{NATIVE_ID}} ?"; - code_ += - " reinterpret_cast<const {{NATIVE_TYPE}} *>(value) : nullptr;"; - code_ += " }"; - } - code_ += "};"; - code_ += ""; - - if (opts_.gen_compare) { - code_ += ""; - code_ += - "inline bool operator==(const {{NAME}}Union &lhs, const " - "{{NAME}}Union &rhs) {"; - code_ += " if (lhs.type != rhs.type) return false;"; - code_ += " switch (lhs.type) {"; - - for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); - ++it) { - const auto &ev = **it; - code_.SetValue("NATIVE_ID", GetEnumValUse(enum_def, ev)); - if (ev.IsNonZero()) { - const auto native_type = GetUnionElement(ev, true, opts_); - code_.SetValue("NATIVE_TYPE", native_type); - code_ += " case {{NATIVE_ID}}: {"; - code_ += - " return *(reinterpret_cast<const {{NATIVE_TYPE}} " - "*>(lhs.value)) =="; - code_ += - " *(reinterpret_cast<const {{NATIVE_TYPE}} " - "*>(rhs.value));"; - code_ += " }"; - } else { - code_ += " case {{NATIVE_ID}}: {"; - code_ += " return true;"; // "NONE" enum value. - code_ += " }"; - } - } - code_ += " default: {"; - code_ += " return false;"; - code_ += " }"; - code_ += " }"; - code_ += "}"; - - code_ += ""; - code_ += - "inline bool operator!=(const {{NAME}}Union &lhs, const " - "{{NAME}}Union &rhs) {"; - code_ += " return !(lhs == rhs);"; - code_ += "}"; - code_ += ""; - } - } - - if (enum_def.is_union) { - code_ += UnionVerifySignature(enum_def) + ";"; - code_ += UnionVectorVerifySignature(enum_def) + ";"; - code_ += ""; - } } void GenUnionPost(const EnumDef &enum_def) { @@ -1419,9 +1579,11 @@ class CppGenerator : public BaseGenerator { " auto ptr = reinterpret_cast<const {{TYPE}} *>(obj);"; if (ev.union_type.base_type == BASE_TYPE_STRUCT) { if (ev.union_type.struct_def->fixed) { + code_.SetValue("ALIGN", + NumToString(ev.union_type.struct_def->minalign)); code_ += - " return verifier.Verify<{{TYPE}}>(static_cast<const " - "uint8_t *>(obj), 0);"; + " return verifier.VerifyField<{{TYPE}}>(" + "static_cast<const uint8_t *>(obj), 0, {{ALIGN}});"; } else { code_ += getptr; code_ += " return verifier.VerifyTable(ptr);"; @@ -1447,7 +1609,8 @@ class CppGenerator : public BaseGenerator { code_ += "inline " + UnionVectorVerifySignature(enum_def) + " {"; code_ += " if (!values || !types) return !values && !types;"; code_ += " if (values->size() != types->size()) return false;"; - code_ += " for (flatbuffers::uoffset_t i = 0; i < values->size(); ++i) {"; + code_ += + " for (::flatbuffers::uoffset_t i = 0; i < values->size(); ++i) {"; code_ += " if (!Verify" + Name(enum_def) + "("; code_ += " verifier, values->Get(i), types->GetEnum<" + Name(enum_def) + ">(i))) {"; @@ -1461,6 +1624,7 @@ class CppGenerator : public BaseGenerator { if (opts_.generate_object_based_api) { // Generate union Unpack() and Pack() functions. code_ += "inline " + UnionUnPackSignature(enum_def, false) + " {"; + code_ += " (void)resolver;"; code_ += " switch (type) {"; for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { @@ -1491,6 +1655,7 @@ class CppGenerator : public BaseGenerator { code_ += ""; code_ += "inline " + UnionPackSignature(enum_def, false) + " {"; + code_ += " (void)_rehasher;"; code_ += " switch (type) {"; for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { @@ -1526,23 +1691,20 @@ class CppGenerator : public BaseGenerator { "inline {{ENUM_NAME}}Union::{{ENUM_NAME}}Union(const " "{{ENUM_NAME}}Union &u) : type(u.type), value(nullptr) {"; code_ += " switch (type) {"; - for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); - ++it) { - const auto &ev = **it; - if (ev.IsZero()) { continue; } - code_.SetValue("LABEL", GetEnumValUse(enum_def, ev)); - code_.SetValue("TYPE", GetUnionElement(ev, true, opts_)); + for (const auto &ev : enum_def.Vals()) { + if (ev->IsZero()) { continue; } + code_.SetValue("LABEL", GetEnumValUse(enum_def, *ev)); + code_.SetValue("TYPE", GetUnionElement(*ev, true, opts_)); code_ += " case {{LABEL}}: {"; bool copyable = true; - if (ev.union_type.base_type == BASE_TYPE_STRUCT && - !ev.union_type.struct_def->fixed) { + if (opts_.g_cpp_std < cpp::CPP_STD_11 && + ev->union_type.base_type == BASE_TYPE_STRUCT && + !ev->union_type.struct_def->fixed) { // Don't generate code to copy if table is not copyable. // TODO(wvo): make tables copyable instead. - for (auto fit = ev.union_type.struct_def->fields.vec.begin(); - fit != ev.union_type.struct_def->fields.vec.end(); ++fit) { - const auto &field = **fit; - if (!field.deprecated && field.value.type.struct_def && - !field.native_inline) { + for (const auto &field : ev->union_type.struct_def->fields.vec) { + if (!field->deprecated && field->value.type.struct_def && + !field->native_inline) { copyable = false; break; } @@ -1621,7 +1783,7 @@ class CppGenerator : public BaseGenerator { if (!opts_.generate_name_strings) { return; } auto fullname = struct_def.defined_namespace->GetFullyQualifiedName(name); code_.SetValue("NAME", fullname); - code_.SetValue("CONSTEXPR", "FLATBUFFERS_CONSTEXPR"); + code_.SetValue("CONSTEXPR", "FLATBUFFERS_CONSTEXPR_CPP11"); code_ += " static {{CONSTEXPR}} const char *GetFullyQualifiedName() {"; code_ += " return \"{{NAME}}\";"; code_ += " }"; @@ -1659,6 +1821,8 @@ class CppGenerator : public BaseGenerator { } else { return "0"; } + } else if (IsStruct(type) && (field.value.constant == "0")) { + return "nullptr"; } else { return GenDefaultConstant(field); } @@ -1676,7 +1840,8 @@ class CppGenerator : public BaseGenerator { if (IsStruct(vtype)) { type = WrapInNameSpace(*vtype.struct_def); } else { - type = GenTypeWire(vtype, "", VectorElementUserFacing(vtype)); + type = GenTypeWire(vtype, "", VectorElementUserFacing(vtype), + field.offset64); } if (TypeHasKey(vtype)) { code_.SetValue("PARAM_TYPE", "std::vector<" + type + "> *"); @@ -1690,7 +1855,8 @@ class CppGenerator : public BaseGenerator { if (field.IsScalarOptional()) code_.SetValue("PARAM_TYPE", GenOptionalDecl(type) + " "); else - code_.SetValue("PARAM_TYPE", GenTypeWire(type, " ", true)); + code_.SetValue("PARAM_TYPE", + GenTypeWire(type, " ", true, field.offset64)); } code_ += "{{PRE}}{{PARAM_TYPE}}{{PARAM_NAME}} = {{PARAM_VALUE}}\\"; } @@ -1699,11 +1865,11 @@ class CppGenerator : public BaseGenerator { void GenMember(const FieldDef &field) { if (!field.deprecated && // Deprecated fields won't be accessible. field.value.type.base_type != BASE_TYPE_UTYPE && - (field.value.type.base_type != BASE_TYPE_VECTOR || + (!IsVector(field.value.type) || field.value.type.element != BASE_TYPE_UTYPE)) { auto type = GenTypeNative(field.value.type, false, field); auto cpp_type = field.attributes.Lookup("cpp_type"); - auto full_type = + const std::string &full_type = (cpp_type ? (IsVector(field.value.type) ? "std::vector<" + @@ -1713,7 +1879,7 @@ class CppGenerator : public BaseGenerator { : GenTypeNativePtr(cpp_type->constant, &field, false)) : type + " "); // Generate default member initializers for >= C++11. - std::string field_di = ""; + std::string field_di; if (opts_.g_cpp_std >= cpp::CPP_STD_11) { field_di = "{}"; auto native_default = field.attributes.Lookup("native_default"); @@ -1735,13 +1901,47 @@ class CppGenerator : public BaseGenerator { } } + // Returns true if `struct_def` needs a copy constructor and assignment + // operator because it has one or more table members, struct members with a + // custom cpp_type and non-naked pointer type, or vector members of those. + bool NeedsCopyCtorAssignOp(const StructDef &struct_def) { + for (const auto &field : struct_def.fields.vec) { + const auto &type = field->value.type; + if (field->deprecated) continue; + if (type.base_type == BASE_TYPE_STRUCT) { + const auto cpp_type = field->attributes.Lookup("cpp_type"); + const auto cpp_ptr_type = field->attributes.Lookup("cpp_ptr_type"); + const bool is_ptr = !(IsStruct(type) && field->native_inline) || + (cpp_type && cpp_ptr_type->constant != "naked"); + if (is_ptr) { return true; } + } else if (IsVector(type)) { + const auto vec_type = type.VectorType(); + if (vec_type.base_type == BASE_TYPE_UTYPE) continue; + const auto cpp_type = field->attributes.Lookup("cpp_type"); + const auto cpp_ptr_type = field->attributes.Lookup("cpp_ptr_type"); + const bool is_ptr = IsVectorOfPointers(*field) || + (cpp_type && cpp_ptr_type->constant != "naked"); + if (is_ptr) { return true; } + } + } + return false; + } + // Generate the default constructor for this struct. Properly initialize all // scalar members with default values. void GenDefaultConstructor(const StructDef &struct_def) { code_.SetValue("NATIVE_NAME", NativeName(Name(struct_def), &struct_def, opts_)); - // In >= C++11, default member initializers are generated. - if (opts_.g_cpp_std >= cpp::CPP_STD_11) { return; } + // In >= C++11, default member initializers are generated. To allow for + // aggregate initialization, do not emit a default constructor at all, with + // the exception of types that need a copy/move ctors and assignment + // operators. + if (opts_.g_cpp_std >= cpp::CPP_STD_11) { + if (NeedsCopyCtorAssignOp(struct_def)) { + code_ += " {{NATIVE_NAME}}() = default;"; + } + return; + } std::string initializer_list; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { @@ -1769,7 +1969,7 @@ class CppGenerator : public BaseGenerator { Name(field) + "(" + native_default->constant + ")"; } } - } else if (cpp_type && field.value.type.base_type != BASE_TYPE_VECTOR) { + } else if (cpp_type && !IsVector(field.value.type)) { if (!initializer_list.empty()) { initializer_list += ",\n "; } initializer_list += Name(field) + "(0)"; } @@ -1785,19 +1985,169 @@ class CppGenerator : public BaseGenerator { code_ += " }"; } + // Generate the >= C++11 copy/move constructor and assignment operator + // declarations if required. Tables that are default-copyable do not get + // user-provided copy/move constructors and assignment operators so they + // remain aggregates. + void GenCopyMoveCtorAndAssigOpDecls(const StructDef &struct_def) { + if (opts_.g_cpp_std < cpp::CPP_STD_11) return; + if (!NeedsCopyCtorAssignOp(struct_def)) return; + code_.SetValue("NATIVE_NAME", + NativeName(Name(struct_def), &struct_def, opts_)); + code_ += " {{NATIVE_NAME}}(const {{NATIVE_NAME}} &o);"; + code_ += + " {{NATIVE_NAME}}({{NATIVE_NAME}}&&) FLATBUFFERS_NOEXCEPT = " + "default;"; + code_ += + " {{NATIVE_NAME}} &operator=({{NATIVE_NAME}} o) FLATBUFFERS_NOEXCEPT;"; + } + + // Generate the >= C++11 copy constructor and assignment operator definitions. + void GenCopyCtorAssignOpDefs(const StructDef &struct_def) { + if (opts_.g_cpp_std < cpp::CPP_STD_11) return; + if (!NeedsCopyCtorAssignOp(struct_def)) return; + std::string initializer_list; + std::string vector_copies; + std::string swaps; + for (const auto &field : struct_def.fields.vec) { + const auto &type = field->value.type; + if (field->deprecated || type.base_type == BASE_TYPE_UTYPE) continue; + if (type.base_type == BASE_TYPE_STRUCT) { + if (!initializer_list.empty()) { initializer_list += ",\n "; } + const auto cpp_type = field->attributes.Lookup("cpp_type"); + const auto cpp_ptr_type = field->attributes.Lookup("cpp_ptr_type"); + const std::string &type_name = + (cpp_type) ? cpp_type->constant + : GenTypeNative(type, /*invector*/ false, *field, + /*forcopy*/ true); + const bool is_ptr = !(IsStruct(type) && field->native_inline) || + (cpp_type && cpp_ptr_type->constant != "naked"); + CodeWriter cw; + cw.SetValue("FIELD", Name(*field)); + cw.SetValue("TYPE", type_name); + if (is_ptr) { + cw += + "{{FIELD}}((o.{{FIELD}}) ? new {{TYPE}}(*o.{{FIELD}}) : " + "nullptr)\\"; + initializer_list += cw.ToString(); + } else { + cw += "{{FIELD}}(o.{{FIELD}})\\"; + initializer_list += cw.ToString(); + } + } else if (IsVector(type)) { + const auto vec_type = type.VectorType(); + if (vec_type.base_type == BASE_TYPE_UTYPE) continue; + const auto cpp_type = field->attributes.Lookup("cpp_type"); + const auto cpp_ptr_type = field->attributes.Lookup("cpp_ptr_type"); + const std::string &type_name = + (cpp_type) ? cpp_type->constant + : GenTypeNative(vec_type, /*invector*/ true, *field, + /*forcopy*/ true); + const bool is_ptr = IsVectorOfPointers(*field) || + (cpp_type && cpp_ptr_type->constant != "naked"); + CodeWriter cw(" "); + cw.SetValue("FIELD", Name(*field)); + cw.SetValue("TYPE", type_name); + if (is_ptr) { + // Use emplace_back to construct the potentially-smart pointer element + // from a raw pointer to a new-allocated copy. + cw.IncrementIdentLevel(); + cw += "{{FIELD}}.reserve(o.{{FIELD}}.size());"; + cw += + "for (const auto &{{FIELD}}_ : o.{{FIELD}}) { " + "{{FIELD}}.emplace_back(({{FIELD}}_) ? new {{TYPE}}(*{{FIELD}}_) " + ": nullptr); }"; + vector_copies += cw.ToString(); + } else { + // For non-pointer elements, use std::vector's copy constructor in the + // initializer list. This will yield better performance than an insert + // range loop for trivially-copyable element types. + if (!initializer_list.empty()) { initializer_list += ",\n "; } + cw += "{{FIELD}}(o.{{FIELD}})\\"; + initializer_list += cw.ToString(); + } + } else { + if (!initializer_list.empty()) { initializer_list += ",\n "; } + CodeWriter cw; + cw.SetValue("FIELD", Name(*field)); + cw += "{{FIELD}}(o.{{FIELD}})\\"; + initializer_list += cw.ToString(); + } + { + if (!swaps.empty()) { swaps += "\n "; } + CodeWriter cw; + cw.SetValue("FIELD", Name(*field)); + cw += "std::swap({{FIELD}}, o.{{FIELD}});\\"; + swaps += cw.ToString(); + } + } + if (!initializer_list.empty()) { + initializer_list = "\n : " + initializer_list; + } + if (!swaps.empty()) { swaps = " " + swaps; } + + code_.SetValue("NATIVE_NAME", + NativeName(Name(struct_def), &struct_def, opts_)); + code_.SetValue("INIT_LIST", initializer_list); + code_.SetValue("VEC_COPY", vector_copies); + code_.SetValue("SWAPS", swaps); + + code_ += + "inline {{NATIVE_NAME}}::{{NATIVE_NAME}}(const {{NATIVE_NAME}} &o)" + "{{INIT_LIST}} {"; + code_ += "{{VEC_COPY}}}\n"; + code_ += + "inline {{NATIVE_NAME}} &{{NATIVE_NAME}}::operator=" + "({{NATIVE_NAME}} o) FLATBUFFERS_NOEXCEPT {"; + code_ += "{{SWAPS}}"; + code_ += " return *this;\n}\n"; + } + void GenCompareOperator(const StructDef &struct_def, - std::string accessSuffix = "") { + const std::string &accessSuffix = "") { std::string compare_op; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { const auto &field = **it; + const auto accessor = Name(field) + accessSuffix; + const auto lhs_accessor = "lhs." + accessor; + const auto rhs_accessor = "rhs." + accessor; if (!field.deprecated && // Deprecated fields won't be accessible. field.value.type.base_type != BASE_TYPE_UTYPE && - (field.value.type.base_type != BASE_TYPE_VECTOR || + (!IsVector(field.value.type) || field.value.type.element != BASE_TYPE_UTYPE)) { if (!compare_op.empty()) { compare_op += " &&\n "; } - auto accessor = Name(field) + accessSuffix; - compare_op += "(lhs." + accessor + " == rhs." + accessor + ")"; + if (struct_def.fixed || field.native_inline || + field.value.type.base_type != BASE_TYPE_STRUCT) { + // If the field is a vector of tables, the table need to be compared + // by value, instead of by the default unique_ptr == operator which + // compares by address. + if (IsVectorOfPointers(field)) { + const auto type = + GenTypeNative(field.value.type.VectorType(), true, field); + const auto equal_length = + lhs_accessor + ".size() == " + rhs_accessor + ".size()"; + const auto elements_equal = + "std::equal(" + lhs_accessor + ".cbegin(), " + lhs_accessor + + ".cend(), " + rhs_accessor + ".cbegin(), [](" + type + + " const &a, " + type + + " const &b) { return (a == b) || (a && b && *a == *b); })"; + + compare_op += "(" + equal_length + " && " + elements_equal + ")"; + } else if (field.value.type.base_type == BASE_TYPE_ARRAY) { + compare_op += "(*" + lhs_accessor + " == *" + rhs_accessor + ")"; + } else { + compare_op += "(" + lhs_accessor + " == " + rhs_accessor + ")"; + } + } else { + // Deep compare of std::unique_ptr. Null is not equal to empty. + std::string both_null = + "(" + lhs_accessor + " == " + rhs_accessor + ")"; + std::string not_null_and_equal = "(lhs." + accessor + " && rhs." + + accessor + " && *lhs." + accessor + + " == *rhs." + accessor + ")"; + compare_op += "(" + both_null + " || " + not_null_and_equal + ")"; + } } } @@ -1853,20 +2203,36 @@ class CppGenerator : public BaseGenerator { code_.SetValue("NATIVE_NAME", native_name); // Generate a C++ object that can hold an unpacked version of this table. - code_ += "struct {{NATIVE_NAME}} : public flatbuffers::NativeTable {"; + code_ += "struct {{NATIVE_NAME}} : public ::flatbuffers::NativeTable {"; code_ += " typedef {{STRUCT_NAME}} TableType;"; GenFullyQualifiedNameGetter(struct_def, native_name); - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - GenMember(**it); - } + for (const auto field : struct_def.fields.vec) { GenMember(*field); } GenOperatorNewDelete(struct_def); GenDefaultConstructor(struct_def); + GenCopyMoveCtorAndAssigOpDecls(struct_def); code_ += "};"; - if (opts_.gen_compare) GenCompareOperator(struct_def); code_ += ""; } + // Adds a typedef to the binary schema type so one could get the bfbs based + // on the type at runtime. + void GenBinarySchemaTypeDef(const StructDef *struct_def) { + if (struct_def && opts_.binary_schema_gen_embed) { + code_ += " typedef " + WrapInNameSpace(*struct_def) + + "BinarySchema BinarySchema;"; + } + } + + void GenNativeTablePost(const StructDef &struct_def) { + if (opts_.gen_compare) { + const auto native_name = NativeName(Name(struct_def), &struct_def, opts_); + code_.SetValue("STRUCT_NAME", Name(struct_def)); + code_.SetValue("NATIVE_NAME", native_name); + GenCompareOperator(struct_def); + code_ += ""; + } + } + // Generate the code to call the appropriate Verify function(s) for a field. void GenVerifyCall(const FieldDef &field, const char *prefix) { code_.SetValue("PRE", prefix); @@ -1875,10 +2241,15 @@ class CppGenerator : public BaseGenerator { code_.SetValue("SIZE", GenTypeSize(field.value.type)); code_.SetValue("OFFSET", GenFieldOffsetName(field)); if (IsScalar(field.value.type.base_type) || IsStruct(field.value.type)) { + code_.SetValue("ALIGN", NumToString(InlineAlignment(field.value.type))); code_ += - "{{PRE}}VerifyField{{REQUIRED}}<{{SIZE}}>(verifier, {{OFFSET}})\\"; + "{{PRE}}VerifyField{{REQUIRED}}<{{SIZE}}>(verifier, " + "{{OFFSET}}, {{ALIGN}})\\"; } else { - code_ += "{{PRE}}VerifyOffset{{REQUIRED}}(verifier, {{OFFSET}})\\"; + code_.SetValue("OFFSET_SIZE", field.offset64 ? "64" : ""); + code_ += + "{{PRE}}VerifyOffset{{OFFSET_SIZE}}{{REQUIRED}}(verifier, " + "{{OFFSET}})\\"; } switch (field.value.type.base_type) { @@ -1900,6 +2271,7 @@ class CppGenerator : public BaseGenerator { code_ += "{{PRE}}verifier.VerifyString({{NAME}}())\\"; break; } + case BASE_TYPE_VECTOR64: case BASE_TYPE_VECTOR: { code_ += "{{PRE}}verifier.VerifyVector({{NAME}}())\\"; @@ -1923,6 +2295,19 @@ class CppGenerator : public BaseGenerator { } default: break; } + + auto nfn = GetNestedFlatBufferName(field); + if (!nfn.empty()) { + code_.SetValue("CPP_NAME", nfn); + // FIXME: file_identifier. + code_ += + "{{PRE}}verifier.VerifyNestedFlatBuffer<{{CPP_NAME}}>" + "({{NAME}}(), nullptr)\\"; + } else if (field.flexbuffer) { + code_ += + "{{PRE}}flexbuffers::VerifyNestedFlexBuffer" + "({{NAME}}(), verifier)\\"; + } break; } default: { @@ -1931,24 +2316,153 @@ class CppGenerator : public BaseGenerator { } } + void GenComparatorForStruct(const StructDef &struct_def, size_t space_size, + const std::string lhs_struct_literal, + const std::string rhs_struct_literal) { + code_.SetValue("LHS_PREFIX", lhs_struct_literal); + code_.SetValue("RHS_PREFIX", rhs_struct_literal); + std::string space(space_size, ' '); + for (const auto &curr_field : struct_def.fields.vec) { + const auto curr_field_name = Name(*curr_field); + code_.SetValue("CURR_FIELD_NAME", curr_field_name); + code_.SetValue("LHS", lhs_struct_literal + "_" + curr_field_name); + code_.SetValue("RHS", rhs_struct_literal + "_" + curr_field_name); + const bool is_scalar = IsScalar(curr_field->value.type.base_type); + const bool is_array = IsArray(curr_field->value.type); + const bool is_struct = IsStruct(curr_field->value.type); + + // If encouter a key field, call KeyCompareWithValue to compare this + // field. + if (curr_field->key) { + code_ += space + + "const auto {{RHS}} = {{RHS_PREFIX}}.{{CURR_FIELD_NAME}}();"; + code_ += space + + "const auto {{CURR_FIELD_NAME}}_compare_result = " + "{{LHS_PREFIX}}.KeyCompareWithValue({{RHS}});"; + + code_ += space + "if ({{CURR_FIELD_NAME}}_compare_result != 0)"; + code_ += space + " return {{CURR_FIELD_NAME}}_compare_result;"; + continue; + } + + code_ += + space + "const auto {{LHS}} = {{LHS_PREFIX}}.{{CURR_FIELD_NAME}}();"; + code_ += + space + "const auto {{RHS}} = {{RHS_PREFIX}}.{{CURR_FIELD_NAME}}();"; + if (is_scalar) { + code_ += space + "if ({{LHS}} != {{RHS}})"; + code_ += space + + " return static_cast<int>({{LHS}} > {{RHS}}) - " + "static_cast<int>({{LHS}} < {{RHS}});"; + } else if (is_array) { + const auto &elem_type = curr_field->value.type.VectorType(); + code_ += + space + + "for (::flatbuffers::uoffset_t i = 0; i < {{LHS}}->size(); i++) {"; + code_ += space + " const auto {{LHS}}_elem = {{LHS}}->Get(i);"; + code_ += space + " const auto {{RHS}}_elem = {{RHS}}->Get(i);"; + if (IsScalar(elem_type.base_type)) { + code_ += space + " if ({{LHS}}_elem != {{RHS}}_elem)"; + code_ += space + + " return static_cast<int>({{LHS}}_elem > {{RHS}}_elem) - " + "static_cast<int>({{LHS}}_elem < {{RHS}}_elem);"; + code_ += space + "}"; + + } else if (IsStruct(elem_type)) { + if (curr_field->key) { + code_ += space + + "const auto {{CURR_FIELD_NAME}}_compare_result = " + "{{LHS_PREFIX}}.KeyCompareWithValue({{RHS}});"; + code_ += space + "if ({{CURR_FIELD_NAME}}_compare_result != 0)"; + code_ += space + " return {{CURR_FIELD_NAME}}_compare_result;"; + continue; + } + GenComparatorForStruct( + *curr_field->value.type.struct_def, space_size + 2, + code_.GetValue("LHS") + "_elem", code_.GetValue("RHS") + "_elem"); + + code_ += space + "}"; + } + + } else if (is_struct) { + GenComparatorForStruct(*curr_field->value.type.struct_def, space_size, + code_.GetValue("LHS"), code_.GetValue("RHS")); + } + } + } + // Generate CompareWithValue method for a key field. void GenKeyFieldMethods(const FieldDef &field) { FLATBUFFERS_ASSERT(field.key); - const bool is_string = (IsString(field.value.type)); - - code_ += " bool KeyCompareLessThan(const {{STRUCT_NAME}} *o) const {"; + const bool is_string = IsString(field.value.type); + const bool is_array = IsArray(field.value.type); + const bool is_struct = IsStruct(field.value.type); + // Generate KeyCompareLessThan function + code_ += + " bool KeyCompareLessThan(const {{STRUCT_NAME}} * const o) const {"; if (is_string) { - // use operator< of flatbuffers::String + // use operator< of ::flatbuffers::String code_ += " return *{{FIELD_NAME}}() < *o->{{FIELD_NAME}}();"; + } else if (is_array || is_struct) { + code_ += " return KeyCompareWithValue(o->{{FIELD_NAME}}()) < 0;"; } else { code_ += " return {{FIELD_NAME}}() < o->{{FIELD_NAME}}();"; } code_ += " }"; + // Generate KeyCompareWithValue function if (is_string) { - code_ += " int KeyCompareWithValue(const char *val) const {"; - code_ += " return strcmp({{FIELD_NAME}}()->c_str(), val);"; - code_ += " }"; + code_ += " int KeyCompareWithValue(const char *_{{FIELD_NAME}}) const {"; + code_ += " return strcmp({{FIELD_NAME}}()->c_str(), _{{FIELD_NAME}});"; + } else if (is_array) { + const auto &elem_type = field.value.type.VectorType(); + std::string input_type = "::flatbuffers::Array<" + + GenTypeGet(elem_type, "", "", "", false) + ", " + + NumToString(elem_type.fixed_length) + ">"; + code_.SetValue("INPUT_TYPE", input_type); + code_ += + " int KeyCompareWithValue(const {{INPUT_TYPE}} *_{{FIELD_NAME}}" + ") const {"; + code_ += + " const {{INPUT_TYPE}} *curr_{{FIELD_NAME}} = {{FIELD_NAME}}();"; + code_ += + " for (::flatbuffers::uoffset_t i = 0; i < " + "curr_{{FIELD_NAME}}->size(); i++) {"; + + if (IsScalar(elem_type.base_type)) { + code_ += " const auto lhs = curr_{{FIELD_NAME}}->Get(i);"; + code_ += " const auto rhs = _{{FIELD_NAME}}->Get(i);"; + code_ += " if (lhs != rhs)"; + code_ += + " return static_cast<int>(lhs > rhs)" + " - static_cast<int>(lhs < rhs);"; + } else if (IsStruct(elem_type)) { + code_ += + " const auto &lhs_{{FIELD_NAME}} = " + "*(curr_{{FIELD_NAME}}->Get(i));"; + code_ += + " const auto &rhs_{{FIELD_NAME}} = " + "*(_{{FIELD_NAME}}->Get(i));"; + GenComparatorForStruct(*elem_type.struct_def, 6, + "lhs_" + code_.GetValue("FIELD_NAME"), + "rhs_" + code_.GetValue("FIELD_NAME")); + } + code_ += " }"; + code_ += " return 0;"; + } else if (is_struct) { + const auto *struct_def = field.value.type.struct_def; + code_.SetValue("INPUT_TYPE", + GenTypeGet(field.value.type, "", "", "", false)); + code_ += + " int KeyCompareWithValue(const {{INPUT_TYPE}} &_{{FIELD_NAME}}) " + "const {"; + code_ += " const auto &lhs_{{FIELD_NAME}} = {{FIELD_NAME}}();"; + code_ += " const auto &rhs_{{FIELD_NAME}} = _{{FIELD_NAME}};"; + GenComparatorForStruct(*struct_def, 4, + "lhs_" + code_.GetValue("FIELD_NAME"), + "rhs_" + code_.GetValue("FIELD_NAME")); + code_ += " return 0;"; + } else { FLATBUFFERS_ASSERT(IsScalar(field.value.type.base_type)); auto type = GenTypeBasic(field.value.type, false); @@ -1958,12 +2472,13 @@ class CppGenerator : public BaseGenerator { } // Returns {field<val: -1, field==val: 0, field>val: +1}. code_.SetValue("KEY_TYPE", type); - code_ += " int KeyCompareWithValue({{KEY_TYPE}} val) const {"; code_ += - " return static_cast<int>({{FIELD_NAME}}() > val) - " - "static_cast<int>({{FIELD_NAME}}() < val);"; - code_ += " }"; + " int KeyCompareWithValue({{KEY_TYPE}} _{{FIELD_NAME}}) const {"; + code_ += + " return static_cast<int>({{FIELD_NAME}}() > _{{FIELD_NAME}}) - " + "static_cast<int>({{FIELD_NAME}}() < _{{FIELD_NAME}});"; } + code_ += " }"; } void GenTableUnionAsGetters(const FieldDef &field) { @@ -1982,7 +2497,7 @@ class CppGenerator : public BaseGenerator { // @TODO: Mby make this decisions more universal? How? code_.SetValue("U_GET_TYPE", - EscapeKeyword(field.name + UnionTypeFieldSuffix())); + EscapeKeyword(Name(field) + UnionTypeFieldSuffix())); code_.SetValue("U_ELEMENT_TYPE", WrapInNameSpace(u->defined_namespace, GetEnumValUse(*u, ev))); code_.SetValue("U_FIELD_TYPE", "const " + full_struct_name + " *"); @@ -2005,15 +2520,21 @@ class CppGenerator : public BaseGenerator { GenComment(field.doc_comment, " "); // Call a different accessor for pointers, that indirects. - if (false == field.IsScalarOptional()) { + if (!field.IsScalarOptional()) { const bool is_scalar = IsScalar(type.base_type); std::string accessor; - if (is_scalar) + std::string offset_size = ""; + if (is_scalar) { accessor = "GetField<"; - else if (IsStruct(type)) + } else if (IsStruct(type)) { accessor = "GetStruct<"; - else - accessor = "GetPointer<"; + } else { + if (field.offset64) { + accessor = "GetPointer64<"; + } else { + accessor = "GetPointer<"; + } + } auto offset_type = GenTypeGet(type, "", "const ", " *", false); auto call = accessor + offset_type + ">(" + offset_str; // Default value as second arg for non-pointer types. @@ -2080,14 +2601,12 @@ class CppGenerator : public BaseGenerator { size_t index = 0; bool need_else = false; // Generate one index-based getter for each field. - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - const auto &field = **it; - if (field.deprecated) { + for (const auto &field : struct_def.fields.vec) { + if (field->deprecated) { // Deprecated fields won't be accessible. continue; } - code_.SetValue("FIELD_NAME", Name(field)); + code_.SetValue("FIELD_NAME", Name(*field)); code_.SetValue("FIELD_INDEX", std::to_string(static_cast<long long>(index++))); if (need_else) { @@ -2112,14 +2631,8 @@ class CppGenerator : public BaseGenerator { // }; // void GenFieldNames(const StructDef &struct_def) { - auto non_deprecated_field_count = std::count_if( - struct_def.fields.vec.begin(), struct_def.fields.vec.end(), - [](const FieldDef *field) { return !field->deprecated; }); code_ += " static constexpr std::array<\\"; - code_.SetValue( - "FIELD_COUNT", - std::to_string(static_cast<long long>(non_deprecated_field_count))); - code_ += "const char *, {{FIELD_COUNT}}> field_names = {\\"; + code_ += "const char *, fields_number> field_names = {\\"; if (struct_def.fields.vec.empty()) { code_ += "};"; return; @@ -2134,14 +2647,14 @@ class CppGenerator : public BaseGenerator { continue; } code_.SetValue("FIELD_NAME", Name(field)); - code_ += " \"{{FIELD_NAME}}\"\\"; + code_ += R"( "{{FIELD_NAME}}"\)"; if (it + 1 != struct_def.fields.vec.end()) { code_ += ","; } } code_ += "\n };"; } void GenFieldsNumber(const StructDef &struct_def) { - auto non_deprecated_field_count = std::count_if( + const auto non_deprecated_field_count = std::count_if( struct_def.fields.vec.begin(), struct_def.fields.vec.end(), [](const FieldDef *field) { return !field->deprecated; }); code_.SetValue( @@ -2165,9 +2678,9 @@ class CppGenerator : public BaseGenerator { code_ += " static constexpr auto fully_qualified_name = " "\"{{FULLY_QUALIFIED_NAME}}\";"; + GenFieldsNumber(struct_def); GenFieldNames(struct_def); GenFieldTypeHelper(struct_def); - GenFieldsNumber(struct_def); } code_ += "};"; code_ += ""; @@ -2181,29 +2694,43 @@ class CppGenerator : public BaseGenerator { auto offset_str = GenFieldOffsetName(field); if (is_scalar) { - const auto wire_type = GenTypeWire(type, "", false); + const auto wire_type = GenTypeWire(type, "", false, field.offset64); code_.SetValue("SET_FN", "SetField<" + wire_type + ">"); code_.SetValue("OFFSET_NAME", offset_str); code_.SetValue("FIELD_TYPE", GenTypeBasic(type, true)); code_.SetValue("FIELD_VALUE", GenUnderlyingCast(field, false, "_" + Name(field))); - code_ += - " bool mutate_{{FIELD_NAME}}({{FIELD_TYPE}} " - "_{{FIELD_NAME}}) {"; - if (false == field.IsScalarOptional()) { + code_ += " bool mutate_{{FIELD_NAME}}({{FIELD_TYPE}} _{{FIELD_NAME}}\\"; + if (!field.IsScalarOptional()) { code_.SetValue("DEFAULT_VALUE", GenDefaultConstant(field)); + code_.SetValue( + "INTERFACE_DEFAULT_VALUE", + GenUnderlyingCast(field, true, GenDefaultConstant(field))); + + // GenUnderlyingCast for a bool field generates 0 != 0 + // So the type has to be checked and the appropriate default chosen + if (IsBool(field.value.type.base_type)) { + code_ += " = {{DEFAULT_VALUE}}) {"; + } else { + code_ += " = {{INTERFACE_DEFAULT_VALUE}}) {"; + } code_ += " return {{SET_FN}}({{OFFSET_NAME}}, {{FIELD_VALUE}}, " "{{DEFAULT_VALUE}});"; } else { + code_ += ") {"; code_ += " return {{SET_FN}}({{OFFSET_NAME}}, {{FIELD_VALUE}});"; } code_ += " }"; } else { auto postptr = " *" + NullableExtension(); auto wire_type = GenTypeGet(type, " ", "", postptr.c_str(), true); - std::string accessor = IsStruct(type) ? "GetStruct<" : "GetPointer<"; + const std::string accessor = [&]() { + if (IsStruct(type)) { return "GetStruct<"; } + if (field.offset64) { return "GetPointer64<"; } + return "GetPointer<"; + }(); auto underlying = accessor + wire_type + ">(" + offset_str + ")"; code_.SetValue("FIELD_TYPE", wire_type); code_.SetValue("FIELD_VALUE", GenUnderlyingCast(field, true, underlying)); @@ -2214,6 +2741,21 @@ class CppGenerator : public BaseGenerator { } } + std::string GetNestedFlatBufferName(const FieldDef &field) { + auto nested = field.attributes.Lookup("nested_flatbuffer"); + if (!nested) return ""; + std::string qualified_name = nested->constant; + auto nested_root = parser_.LookupStruct(nested->constant); + if (nested_root == nullptr) { + qualified_name = + parser_.current_namespace_->GetFullyQualifiedName(nested->constant); + nested_root = parser_.LookupStruct(qualified_name); + } + FLATBUFFERS_ASSERT(nested_root); // Guaranteed to exist by parser. + (void)nested_root; + return TranslateNameSpace(qualified_name); + } + // Generate an accessor struct, builder structs & function for a table. void GenTable(const StructDef &struct_def) { if (opts_.generate_object_based_api) { GenNativeTable(struct_def); } @@ -2225,15 +2767,17 @@ class CppGenerator : public BaseGenerator { code_.SetValue("STRUCT_NAME", Name(struct_def)); code_ += "struct {{STRUCT_NAME}} FLATBUFFERS_FINAL_CLASS" - " : private flatbuffers::Table {"; + " : private ::flatbuffers::Table {"; if (opts_.generate_object_based_api) { code_ += " typedef {{NATIVE_NAME}} NativeTableType;"; } code_ += " typedef {{STRUCT_NAME}}Builder Builder;"; + GenBinarySchemaTypeDef(parser_.root_struct_def_); + if (opts_.g_cpp_std >= cpp::CPP_STD_17) { code_ += " struct Traits;"; } if (opts_.mini_reflect != IDLOptions::kNone) { code_ += - " static const flatbuffers::TypeTable *MiniReflectTypeTable() {"; + " static const ::flatbuffers::TypeTable *MiniReflectTypeTable() {"; code_ += " return {{STRUCT_NAME}}TypeTable();"; code_ += " }"; } @@ -2241,22 +2785,20 @@ class CppGenerator : public BaseGenerator { GenFullyQualifiedNameGetter(struct_def, Name(struct_def)); // Generate field id constants. - if (struct_def.fields.vec.size() > 0) { + if (!struct_def.fields.vec.empty()) { // We need to add a trailing comma to all elements except the last one as // older versions of gcc complain about this. code_.SetValue("SEP", ""); code_ += " enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE {"; - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - const auto &field = **it; - if (field.deprecated) { + for (const auto &field : struct_def.fields.vec) { + if (field->deprecated) { // Deprecated fields won't be accessible. continue; } - code_.SetValue("OFFSET_NAME", GenFieldOffsetName(field)); - code_.SetValue("OFFSET_VALUE", NumToString(field.value.offset)); + code_.SetValue("OFFSET_NAME", GenFieldOffsetName(*field)); + code_.SetValue("OFFSET_VALUE", NumToString(field->value.offset)); code_ += "{{SEP}} {{OFFSET_NAME}} = {{OFFSET_VALUE}}\\"; code_.SetValue("SEP", ",\n"); } @@ -2265,65 +2807,52 @@ class CppGenerator : public BaseGenerator { } // Generate the accessors. - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - const auto &field = **it; - if (field.deprecated) { + for (const auto &field : struct_def.fields.vec) { + if (field->deprecated) { // Deprecated fields won't be accessible. continue; } - code_.SetValue("FIELD_NAME", Name(field)); - GenTableFieldGetter(field); - if (opts_.mutable_buffer) { GenTableFieldSetter(field); } - - auto nested = field.attributes.Lookup("nested_flatbuffer"); - if (nested) { - std::string qualified_name = nested->constant; - auto nested_root = parser_.LookupStruct(nested->constant); - if (nested_root == nullptr) { - qualified_name = parser_.current_namespace_->GetFullyQualifiedName( - nested->constant); - nested_root = parser_.LookupStruct(qualified_name); - } - FLATBUFFERS_ASSERT(nested_root); // Guaranteed to exist by parser. - (void)nested_root; - code_.SetValue("CPP_NAME", TranslateNameSpace(qualified_name)); + code_.SetValue("FIELD_NAME", Name(*field)); + GenTableFieldGetter(*field); + if (opts_.mutable_buffer) { GenTableFieldSetter(*field); } + auto nfn = GetNestedFlatBufferName(*field); + if (!nfn.empty()) { + code_.SetValue("CPP_NAME", nfn); code_ += " const {{CPP_NAME}} *{{FIELD_NAME}}_nested_root() const {"; + code_ += " const auto _f = {{FIELD_NAME}}();"; code_ += - " return " - "flatbuffers::GetRoot<{{CPP_NAME}}>({{FIELD_NAME}}()->Data());"; + " return _f ? ::flatbuffers::GetRoot<{{CPP_NAME}}>(_f->Data())"; + code_ += " : nullptr;"; code_ += " }"; } - if (field.flexbuffer) { + if (field->flexbuffer) { code_ += " flexbuffers::Reference {{FIELD_NAME}}_flexbuffer_root()" " const {"; // Both Data() and size() are const-methods, therefore call order // doesn't matter. - code_ += - " return flexbuffers::GetRoot({{FIELD_NAME}}()->Data(), " - "{{FIELD_NAME}}()->size());"; + code_ += " const auto _f = {{FIELD_NAME}}();"; + code_ += " return _f ? flexbuffers::GetRoot(_f->Data(), _f->size())"; + code_ += " : flexbuffers::Reference();"; code_ += " }"; } // Generate a comparison function for this field if it is a key. - if (field.key) { GenKeyFieldMethods(field); } + if (field->key) { GenKeyFieldMethods(*field); } } if (opts_.cpp_static_reflection) { GenIndexBasedFieldGetter(struct_def); } // Generate a verifier function that can check a buffer from an untrusted // source will never cause reads outside the buffer. - code_ += " bool Verify(flatbuffers::Verifier &verifier) const {"; + code_ += " bool Verify(::flatbuffers::Verifier &verifier) const {"; code_ += " return VerifyTableStart(verifier)\\"; - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - const auto &field = **it; - if (field.deprecated) { continue; } - GenVerifyCall(field, " &&\n "); + for (const auto &field : struct_def.fields.vec) { + if (field->deprecated) { continue; } + GenVerifyCall(*field, " &&\n "); } code_ += " &&\n verifier.EndTable();"; @@ -2340,17 +2869,15 @@ class CppGenerator : public BaseGenerator { code_ += ""; // Explicit specializations for union accessors - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - const auto &field = **it; - if (field.deprecated || field.value.type.base_type != BASE_TYPE_UNION) { + for (const auto &field : struct_def.fields.vec) { + if (field->deprecated || field->value.type.base_type != BASE_TYPE_UNION) { continue; } - auto u = field.value.type.enum_def; + auto u = field->value.type.enum_def; if (u->uses_multiple_type_instances) continue; - code_.SetValue("FIELD_NAME", Name(field)); + code_.SetValue("FIELD_NAME", Name(*field)); for (auto u_it = u->Vals().begin(); u_it != u->Vals().end(); ++u_it) { auto &ev = **u_it; @@ -2363,7 +2890,7 @@ class CppGenerator : public BaseGenerator { WrapInNameSpace(u->defined_namespace, GetEnumValUse(*u, ev))); code_.SetValue("U_FIELD_TYPE", "const " + full_struct_name + " *"); code_.SetValue("U_ELEMENT_NAME", full_struct_name); - code_.SetValue("U_FIELD_NAME", Name(field) + "_as_" + Name(ev)); + code_.SetValue("U_FIELD_NAME", Name(*field) + "_as_" + Name(ev)); // `template<> const T *union_name_as<T>() const` accessor. code_ += @@ -2397,8 +2924,9 @@ class CppGenerator : public BaseGenerator { // Generate code to do force_align for the vector. if (align > 1) { const auto vtype = field.value.type.VectorType(); - const auto type = IsStruct(vtype) ? WrapInNameSpace(*vtype.struct_def) - : GenTypeWire(vtype, "", false); + const std::string &type = + IsStruct(vtype) ? WrapInNameSpace(*vtype.struct_def) + : GenTypeWire(vtype, "", false, field.offset64); return "_fbb.ForceVectorAlignment(" + field_size + ", sizeof(" + type + "), " + std::to_string(static_cast<long long>(align)) + ");"; } @@ -2411,8 +2939,8 @@ class CppGenerator : public BaseGenerator { // Generate a builder struct: code_ += "struct {{STRUCT_NAME}}Builder {"; code_ += " typedef {{STRUCT_NAME}} Table;"; - code_ += " flatbuffers::FlatBufferBuilder &fbb_;"; - code_ += " flatbuffers::uoffset_t start_;"; + code_ += " " + GetBuilder() + " &fbb_;"; + code_ += " ::flatbuffers::uoffset_t start_;"; bool has_string_or_vector_fields = false; for (auto it = struct_def.fields.vec.begin(); @@ -2434,12 +2962,14 @@ class CppGenerator : public BaseGenerator { // fbb_.AddElement<type>(offset, name, default); // } code_.SetValue("FIELD_NAME", Name(field)); - code_.SetValue("FIELD_TYPE", GenTypeWire(field.value.type, " ", true)); + code_.SetValue("FIELD_TYPE", + GenTypeWire(field.value.type, " ", true, field.offset64)); code_.SetValue("ADD_OFFSET", Name(struct_def) + "::" + offset); code_.SetValue("ADD_NAME", name); code_.SetValue("ADD_VALUE", value); if (is_scalar) { - const auto type = GenTypeWire(field.value.type, "", false); + const auto type = + GenTypeWire(field.value.type, "", false, field.offset64); code_.SetValue("ADD_FN", "AddElement<" + type + ">"); } else if (IsStruct(field.value.type)) { code_.SetValue("ADD_FN", "AddStruct"); @@ -2458,24 +2988,22 @@ class CppGenerator : public BaseGenerator { } // Builder constructor - code_ += - " explicit {{STRUCT_NAME}}Builder(flatbuffers::FlatBufferBuilder " - "&_fbb)"; + code_ += " explicit {{STRUCT_NAME}}Builder(" + GetBuilder() + + " " + "&_fbb)"; code_ += " : fbb_(_fbb) {"; code_ += " start_ = fbb_.StartTable();"; code_ += " }"; // Finish() function. - code_ += " flatbuffers::Offset<{{STRUCT_NAME}}> Finish() {"; + code_ += " ::flatbuffers::Offset<{{STRUCT_NAME}}> Finish() {"; code_ += " const auto end = fbb_.EndTable(start_);"; - code_ += " auto o = flatbuffers::Offset<{{STRUCT_NAME}}>(end);"; + code_ += " auto o = ::flatbuffers::Offset<{{STRUCT_NAME}}>(end);"; - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - const auto &field = **it; - if (!field.deprecated && field.IsRequired()) { - code_.SetValue("FIELD_NAME", Name(field)); - code_.SetValue("OFFSET_NAME", GenFieldOffsetName(field)); + for (const auto &field : struct_def.fields.vec) { + if (!field->deprecated && field->IsRequired()) { + code_.SetValue("FIELD_NAME", Name(*field)); + code_.SetValue("OFFSET_NAME", GenFieldOffsetName(*field)); code_ += " fbb_.Required(o, {{STRUCT_NAME}}::{{OFFSET_NAME}});"; } } @@ -2487,13 +3015,11 @@ class CppGenerator : public BaseGenerator { // Generate a convenient CreateX function that uses the above builder // to create a table in one go. code_ += - "inline flatbuffers::Offset<{{STRUCT_NAME}}> " + "inline ::flatbuffers::Offset<{{STRUCT_NAME}}> " "Create{{STRUCT_NAME}}("; - code_ += " flatbuffers::FlatBufferBuilder &_fbb\\"; - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - const auto &field = **it; - if (!field.deprecated) { GenParam(field, false, ",\n "); } + code_ += " " + GetBuilder() + " &_fbb\\"; + for (const auto &field : struct_def.fields.vec) { + if (!field->deprecated) { GenParam(*field, false, ",\n "); } } code_ += ") {"; @@ -2527,71 +3053,96 @@ class CppGenerator : public BaseGenerator { // Generate a CreateXDirect function with vector types as parameters if (opts_.cpp_direct_copy && has_string_or_vector_fields) { code_ += - "inline flatbuffers::Offset<{{STRUCT_NAME}}> " + "inline ::flatbuffers::Offset<{{STRUCT_NAME}}> " "Create{{STRUCT_NAME}}Direct("; - code_ += " flatbuffers::FlatBufferBuilder &_fbb\\"; - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - const auto &field = **it; - if (!field.deprecated) { GenParam(field, true, ",\n "); } + code_ += " " + GetBuilder() + " &_fbb\\"; + for (const auto &field : struct_def.fields.vec) { + if (!field->deprecated) { GenParam(*field, true, ",\n "); } } // Need to call "Create" with the struct namespace. const auto qualified_create_name = struct_def.defined_namespace->GetFullyQualifiedName("Create"); code_.SetValue("CREATE_NAME", TranslateNameSpace(qualified_create_name)); code_ += ") {"; - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - const auto &field = **it; - if (!field.deprecated) { - code_.SetValue("FIELD_NAME", Name(field)); - if (IsString(field.value.type)) { - if (!field.shared) { - code_.SetValue("CREATE_STRING", "CreateString"); + // Offset64 bit fields need to be added to the buffer first, so here we + // loop over the fields in order of their offset size, followed by their + // definition order. Otherwise the emitted code might add a Offset + // followed by an Offset64 which would trigger an assertion. + + // TODO(derekbailey): maybe optimize for the case where there is no + // 64offsets in the whole schema? + ForAllFieldsOrderedByOffset(struct_def, [&](const FieldDef *field) { + if (field->deprecated) { return; } + code_.SetValue("FIELD_NAME", Name(*field)); + if (IsString(field->value.type)) { + if (!field->shared) { + code_.SetValue( + "CREATE_STRING", + "CreateString" + std::string(field->offset64 + ? "<::flatbuffers::Offset64>" + : "")); + } else { + code_.SetValue("CREATE_STRING", "CreateSharedString"); + } + code_ += + " auto {{FIELD_NAME}}__ = {{FIELD_NAME}} ? " + "_fbb.{{CREATE_STRING}}({{FIELD_NAME}}) : 0;"; + } else if (IsVector(field->value.type)) { + const std::string force_align_code = + GenVectorForceAlign(*field, Name(*field) + "->size()"); + if (!force_align_code.empty()) { + code_ += " if ({{FIELD_NAME}}) { " + force_align_code + " }"; + } + code_ += " auto {{FIELD_NAME}}__ = {{FIELD_NAME}} ? \\"; + const auto vtype = field->value.type.VectorType(); + const auto has_key = TypeHasKey(vtype); + if (IsStruct(vtype)) { + const std::string type = WrapInNameSpace(*vtype.struct_def); + if (has_key) { + code_ += "_fbb.CreateVectorOfSortedStructs<" + type + ">\\"; } else { - code_.SetValue("CREATE_STRING", "CreateSharedString"); - } - code_ += - " auto {{FIELD_NAME}}__ = {{FIELD_NAME}} ? " - "_fbb.{{CREATE_STRING}}({{FIELD_NAME}}) : 0;"; - } else if (IsVector(field.value.type)) { - const std::string force_align_code = - GenVectorForceAlign(field, Name(field) + "->size()"); - if (!force_align_code.empty()) { - code_ += " if ({{FIELD_NAME}}) { " + force_align_code + " }"; + // If the field uses 64-bit addressing, create a 64-bit vector. + if (field->value.type.base_type == BASE_TYPE_VECTOR64) { + code_ += "_fbb.CreateVectorOfStructs64\\"; + } else { + code_ += "_fbb.CreateVectorOfStructs\\"; + if (field->offset64) { + // This is normal 32-bit vector, with 64-bit addressing. + code_ += "64<::flatbuffers::Vector>\\"; + } else { + code_ += "<" + type + ">\\"; + } + } } - code_ += " auto {{FIELD_NAME}}__ = {{FIELD_NAME}} ? \\"; - const auto vtype = field.value.type.VectorType(); - const auto has_key = TypeHasKey(vtype); - if (IsStruct(vtype)) { - const auto type = WrapInNameSpace(*vtype.struct_def); - code_ += (has_key ? "_fbb.CreateVectorOfSortedStructs<" - : "_fbb.CreateVectorOfStructs<") + - type + ">\\"; - } else if (has_key) { - const auto type = WrapInNameSpace(*vtype.struct_def); - code_ += "_fbb.CreateVectorOfSortedTables<" + type + ">\\"; + } else if (has_key) { + const auto type = WrapInNameSpace(*vtype.struct_def); + code_ += "_fbb.CreateVectorOfSortedTables<" + type + ">\\"; + } else { + const auto type = GenTypeWire( + vtype, "", VectorElementUserFacing(vtype), field->offset64); + + if (field->value.type.base_type == BASE_TYPE_VECTOR64) { + code_ += "_fbb.CreateVector64\\"; } else { - const auto type = - GenTypeWire(vtype, "", VectorElementUserFacing(vtype)); - code_ += "_fbb.CreateVector<" + type + ">\\"; + // If the field uses 64-bit addressing, create a 64-bit vector. + code_.SetValue("64OFFSET", field->offset64 ? "64" : ""); + code_.SetValue("TYPE", + field->offset64 ? "::flatbuffers::Vector" : type); + + code_ += "_fbb.CreateVector{{64OFFSET}}<{{TYPE}}>\\"; } - code_ += - has_key ? "({{FIELD_NAME}}) : 0;" : "(*{{FIELD_NAME}}) : 0;"; } + code_ += has_key ? "({{FIELD_NAME}}) : 0;" : "(*{{FIELD_NAME}}) : 0;"; } - } + }); code_ += " return {{CREATE_NAME}}{{STRUCT_NAME}}("; code_ += " _fbb\\"; - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - const auto &field = **it; - if (!field.deprecated) { - code_.SetValue("FIELD_NAME", Name(field)); - code_ += ",\n {{FIELD_NAME}}\\"; - if (IsString(field.value.type) || IsVector(field.value.type)) { - code_ += "__\\"; - } + for (const auto &field : struct_def.fields.vec) { + if (field->deprecated) { continue; } + code_.SetValue("FIELD_NAME", Name(*field)); + code_ += ",\n {{FIELD_NAME}}\\"; + if (IsString(field->value.type) || IsVector(field->value.type)) { + code_ += "__\\"; } } code_ += ");"; @@ -2625,7 +3176,7 @@ class CppGenerator : public BaseGenerator { const auto &struct_attrs = type.struct_def->attributes; const auto native_type = struct_attrs.Lookup("native_type"); if (native_type) { - std::string unpack_call = "flatbuffers::UnPack"; + std::string unpack_call = "::flatbuffers::UnPack"; const auto pack_name = struct_attrs.Lookup("native_type_pack_name"); if (pack_name) { unpack_call += pack_name->constant; } unpack_call += "(*" + val + ")"; @@ -2638,7 +3189,8 @@ class CppGenerator : public BaseGenerator { return ptype + "(new " + name + "(*" + val + "))"; } } else { - const auto ptype = GenTypeNativePtr( + std::string ptype = afield.native_inline ? "*" : ""; + ptype += GenTypeNativePtr( WrapNativeNameInNameSpace(*type.struct_def, opts_), &afield, true); return ptype + "(" + val + "->UnPack(_resolver))"; @@ -2661,12 +3213,14 @@ class CppGenerator : public BaseGenerator { const FieldDef *union_field) { std::string code; switch (field.value.type.base_type) { + case BASE_TYPE_VECTOR64: case BASE_TYPE_VECTOR: { auto name = Name(field); if (field.value.type.element == BASE_TYPE_UTYPE) { name = StripUnionType(Name(field)); } - code += "{ _o->" + name + ".resize(_e->size()); "; + const std::string vector_field = "_o->" + name; + code += "{ " + vector_field + ".resize(_e->size()); "; if (!field.value.type.enum_def && !IsBool(field.value.type.element) && IsOneByte(field.value.type.element)) { // For vectors of bytes, std::copy is used to improve performance. @@ -2696,8 +3250,11 @@ class CppGenerator : public BaseGenerator { ? ".type" : (field.value.type.element == BASE_TYPE_UNION ? ".value" : ""); - - code += "for (flatbuffers::uoffset_t _i = 0;"; + if (field.value.type.base_type == BASE_TYPE_VECTOR64) { + code += "for (::flatbuffers::uoffset64_t _i = 0;"; + } else { + code += "for (::flatbuffers::uoffset_t _i = 0;"; + } code += " _i < _e->size(); _i++) { "; auto cpp_type = field.attributes.Lookup("cpp_type"); if (cpp_type) { @@ -2706,13 +3263,13 @@ class CppGenerator : public BaseGenerator { // (*resolver)(&_o->field, (hash_value_t)(_e)); // else // _o->field = nullptr; - code += "//vector resolver, " + PtrType(&field) + "\n"; + code += "/*vector resolver, " + PtrType(&field) + "*/ "; code += "if (_resolver) "; code += "(*_resolver)"; code += "(reinterpret_cast<void **>(&_o->" + name + "[_i]" + access + "), "; code += - "static_cast<flatbuffers::hash_value_t>(" + indexing + "));"; + "static_cast<::flatbuffers::hash_value_t>(" + indexing + "));"; if (PtrType(&field) == "naked") { code += " else "; code += "_o->" + name + "[_i]" + access + " = nullptr"; @@ -2723,11 +3280,19 @@ class CppGenerator : public BaseGenerator { code += "/* else do nothing */"; } } else { + const bool is_pointer = IsVectorOfPointers(field); + if (is_pointer) { + code += "if(_o->" + name + "[_i]" + ") { "; + code += indexing + "->UnPackTo(_o->" + name + + "[_i].get(), _resolver);"; + code += " } else { "; + } code += "_o->" + name + "[_i]" + access + " = "; code += GenUnpackVal(field.value.type.VectorType(), indexing, true, field); + if (is_pointer) { code += "; }"; } } - code += "; } }"; + code += "; } } else { " + vector_field + ".resize(0); }"; } break; } @@ -2755,11 +3320,11 @@ class CppGenerator : public BaseGenerator { // (*resolver)(&_o->field, (hash_value_t)(_e)); // else // _o->field = nullptr; - code += "//scalar resolver, " + PtrType(&field) + " \n"; + code += "/*scalar resolver, " + PtrType(&field) + "*/ "; code += "if (_resolver) "; code += "(*_resolver)"; code += "(reinterpret_cast<void **>(&_o->" + Name(field) + "), "; - code += "static_cast<flatbuffers::hash_value_t>(_e));"; + code += "static_cast<::flatbuffers::hash_value_t>(_e));"; if (PtrType(&field) == "naked") { code += " else "; code += "_o->" + Name(field) + " = nullptr;"; @@ -2772,8 +3337,21 @@ class CppGenerator : public BaseGenerator { } else { // Generate code for assigning the value, of the form: // _o->field = value; - code += "_o->" + Name(field) + " = "; + const bool is_pointer = IsPointer(field); + + const std::string out_field = "_o->" + Name(field); + + if (is_pointer) { + code += "{ if(" + out_field + ") { "; + code += "_e->UnPackTo(" + out_field + ".get(), _resolver);"; + code += " } else { "; + } + code += out_field + " = "; code += GenUnpackVal(field.value.type, "_e", false, field) + ";"; + if (is_pointer) { + code += " } } else if (" + out_field + ") { " + out_field + + ".reset(); }"; + } } break; } @@ -2789,8 +3367,7 @@ class CppGenerator : public BaseGenerator { } else { value += Name(field); } - if (field.value.type.base_type != BASE_TYPE_VECTOR && - field.attributes.Lookup("cpp_type")) { + if (!IsVector(field.value.type) && field.attributes.Lookup("cpp_type")) { auto type = GenTypeBasic(field.value.type, false); value = "_rehasher ? " @@ -2806,7 +3383,10 @@ class CppGenerator : public BaseGenerator { // _fbb.CreateSharedString(_o->field) case BASE_TYPE_STRING: { if (!field.shared) { - code += "_fbb.CreateString("; + code += + "_fbb.CreateString" + + std::string(field.offset64 ? "<::flatbuffers::Offset64>" : "") + + "("; } else { code += "_fbb.CreateSharedString("; } @@ -2833,6 +3413,7 @@ class CppGenerator : public BaseGenerator { // _fbb.CreateVector<Offset<T>>(_o->field.size() [&](size_t i) { // return CreateT(_fbb, _o->Get(i), rehasher); // }); + case BASE_TYPE_VECTOR64: case BASE_TYPE_VECTOR: { auto vector_type = field.value.type.VectorType(); switch (vector_type.base_type) { @@ -2843,7 +3424,8 @@ class CppGenerator : public BaseGenerator { // Use by-function serialization to emulate // CreateVectorOfStrings(); this works also with non-std strings. code += - "_fbb.CreateVector<flatbuffers::Offset<flatbuffers::String>>" + "_fbb.CreateVector<::flatbuffers::Offset<::flatbuffers::" + "String>>" " "; code += "(" + value + ".size(), "; code += "[](size_t i, _VectorArgs *__va) { "; @@ -2866,22 +3448,35 @@ class CppGenerator : public BaseGenerator { const auto pack_name = struct_attrs.Lookup("native_type_pack_name"); if (pack_name) { - code += ", flatbuffers::Pack" + pack_name->constant; + code += ", ::flatbuffers::Pack" + pack_name->constant; } code += ")"; } else { - code += "_fbb.CreateVectorOfStructs"; + // If the field uses 64-bit addressing, create a 64-bit vector. + if (field.value.type.base_type == BASE_TYPE_VECTOR64) { + code += "_fbb.CreateVectorOfStructs64"; + } else { + code += "_fbb.CreateVectorOfStructs"; + if (field.offset64) { + // This is normal 32-bit vector, with 64-bit addressing. + code += "64<::flatbuffers::Vector>"; + } + } code += "(" + value + ")"; } } else { - code += "_fbb.CreateVector<flatbuffers::Offset<"; + code += "_fbb.CreateVector<::flatbuffers::Offset<"; code += WrapInNameSpace(*vector_type.struct_def) + ">> "; code += "(" + value + ".size(), "; code += "[](size_t i, _VectorArgs *__va) { "; code += "return Create" + vector_type.struct_def->name; - code += "(*__va->__fbb, __va->_" + value + "[i]" + - GenPtrGet(field) + ", "; - code += "__va->__rehasher); }, &_va )"; + code += "(*__va->__fbb, "; + if (field.native_inline) { + code += "&(__va->_" + value + "[i])"; + } else { + code += "__va->_" + value + "[i]" + GenPtrGet(field); + } + code += ", __va->__rehasher); }, &_va )"; } break; } @@ -2891,7 +3486,7 @@ class CppGenerator : public BaseGenerator { } case BASE_TYPE_UNION: { code += - "_fbb.CreateVector<flatbuffers::" + "_fbb.CreateVector<::flatbuffers::" "Offset<void>>(" + value + ".size(), [](size_t i, _VectorArgs *__va) { " @@ -2901,10 +3496,16 @@ class CppGenerator : public BaseGenerator { } case BASE_TYPE_UTYPE: { value = StripUnionType(value); - code += "_fbb.CreateVector<uint8_t>(" + value + - ".size(), [](size_t i, _VectorArgs *__va) { " - "return static_cast<uint8_t>(__va->_" + - value + "[i].type); }, &_va)"; + const std::string &type = opts_.scoped_enums + ? Name(*field.value.type.enum_def) + : "uint8_t"; + auto enum_value = "__va->_" + value + "[i].type"; + if (!opts_.scoped_enums) + enum_value = "static_cast<uint8_t>(" + enum_value + ")"; + + code += "_fbb.CreateVector<" + type + ">(" + value + + ".size(), [](size_t i, _VectorArgs *__va) { return " + + enum_value + "; }, &_va)"; break; } default: { @@ -2915,7 +3516,7 @@ class CppGenerator : public BaseGenerator { const auto basetype = GenTypeBasic( field.value.type.enum_def->underlying_type, false); code += "_fbb.CreateVectorScalarCast<" + basetype + - ">(flatbuffers::data(" + value + "), " + value + + ">(::flatbuffers::data(" + value + "), " + value + ".size())"; } else if (field.attributes.Lookup("cpp_type")) { auto type = GenTypeBasic(vector_type, false); @@ -2926,7 +3527,17 @@ class CppGenerator : public BaseGenerator { code += "(__va->_" + value + "[i]" + GenPtrGet(field) + ")) : 0"; code += "; }, &_va )"; } else { - code += "_fbb.CreateVector(" + value + ")"; + // If the field uses 64-bit addressing, create a 64-bit vector. + if (field.value.type.base_type == BASE_TYPE_VECTOR64) { + code += "_fbb.CreateVector64(" + value + ")"; + } else { + code += "_fbb.CreateVector"; + if (field.offset64) { + // This is normal 32-bit vector, with 64-bit addressing. + code += "64<::flatbuffers::Vector>"; + } + code += "(" + value + ")"; + } } break; } @@ -2935,7 +3546,8 @@ class CppGenerator : public BaseGenerator { // If set_empty_vectors_to_null option is enabled, for optional fields, // check to see if there actually is any data in _o->field before // attempting to access it. - if (opts_.set_empty_vectors_to_null && !field.IsRequired()) { + if (field.attributes.Lookup("nested_flatbuffer") || + (opts_.set_empty_vectors_to_null && !field.IsRequired())) { code = value + ".size() ? " + code + " : 0"; } break; @@ -2950,7 +3562,7 @@ class CppGenerator : public BaseGenerator { const auto &struct_attribs = field.value.type.struct_def->attributes; const auto native_type = struct_attribs.Lookup("native_type"); if (native_type) { - code += "flatbuffers::Pack"; + code += "::flatbuffers::Pack"; const auto pack_name = struct_attribs.Lookup("native_type_pack_name"); if (pack_name) { code += pack_name->constant; } @@ -2958,14 +3570,15 @@ class CppGenerator : public BaseGenerator { } else if (field.native_inline) { code += "&" + value; } else { - code += value + " ? " + value + GenPtrGet(field) + " : 0"; + code += value + " ? " + value + GenPtrGet(field) + " : nullptr"; } } else { // _o->field ? CreateT(_fbb, _o->field.get(), _rehasher); - const auto type = field.value.type.struct_def->name; + const std::string &type = field.value.type.struct_def->name; code += value + " ? Create" + type; - code += "(_fbb, " + value + GenPtrGet(field) + ", _rehasher)"; - code += " : 0"; + code += "(_fbb, " + value; + if (!field.native_inline) code += GenPtrGet(field); + code += ", _rehasher) : 0"; } break; } @@ -2979,11 +3592,16 @@ class CppGenerator : public BaseGenerator { // Generate code for tables that needs to come after the regular definition. void GenTablePost(const StructDef &struct_def) { + if (opts_.generate_object_based_api) { GenNativeTablePost(struct_def); } + code_.SetValue("STRUCT_NAME", Name(struct_def)); code_.SetValue("NATIVE_NAME", NativeName(Name(struct_def), &struct_def, opts_)); if (opts_.generate_object_based_api) { + // Generate the >= C++11 copy ctor and assignment operator definitions. + GenCopyCtorAssignOpDefs(struct_def); + // Generate the X::UnPack() method. code_ += "inline " + TableUnPackSignature(struct_def, false, opts_) + " {"; @@ -3046,11 +3664,13 @@ class CppGenerator : public BaseGenerator { code_ += " struct _VectorArgs " - "{ flatbuffers::FlatBufferBuilder *__fbb; " + "{ " + + GetBuilder() + + " *__fbb; " "const " + NativeName(Name(struct_def), &struct_def, opts_) + "* __o; " - "const flatbuffers::rehasher_function_t *__rehasher; } _va = { " + "const ::flatbuffers::rehasher_function_t *__rehasher; } _va = { " "&_fbb, _o, _rehasher}; (void)_va;"; for (auto it = struct_def.fields.vec.begin(); @@ -3071,25 +3691,23 @@ class CppGenerator : public BaseGenerator { code_ += " return {{CREATE_NAME}}{{STRUCT_NAME}}("; code_ += " _fbb\\"; - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - auto &field = **it; - if (field.deprecated) { continue; } + for (const auto &field : struct_def.fields.vec) { + if (field->deprecated) { continue; } bool pass_by_address = false; - if (field.value.type.base_type == BASE_TYPE_STRUCT) { - if (IsStruct(field.value.type)) { + if (field->value.type.base_type == BASE_TYPE_STRUCT) { + if (IsStruct(field->value.type)) { auto native_type = - field.value.type.struct_def->attributes.Lookup("native_type"); + field->value.type.struct_def->attributes.Lookup("native_type"); if (native_type) { pass_by_address = true; } } } // Call the CreateX function using values from |_o|. if (pass_by_address) { - code_ += ",\n &_" + Name(field) + "\\"; + code_ += ",\n &_" + Name(*field) + "\\"; } else { - code_ += ",\n _" + Name(field) + "\\"; + code_ += ",\n _" + Name(*field) + "\\"; } } code_ += ");"; @@ -3134,10 +3752,8 @@ class CppGenerator : public BaseGenerator { bool first_in_init_list = true; int padding_initializer_id = 0; int padding_body_id = 0; - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - const auto field = *it; - const auto field_name = field->name + "_"; + for (const auto &field : struct_def.fields.vec) { + const auto field_name = Name(*field) + "_"; if (first_in_init_list) { first_in_init_list = false; @@ -3203,7 +3819,7 @@ class CppGenerator : public BaseGenerator { init_list += Name(field) + "_"; if (IsScalar(type.base_type)) { auto scalar_type = GenUnderlyingCast(field, false, arg_name); - init_list += "(flatbuffers::EndianScalar(" + scalar_type + "))"; + init_list += "(::flatbuffers::EndianScalar(" + scalar_type + "))"; } else { FLATBUFFERS_ASSERT((is_array && !init_arrays) || IsStruct(type)); if (!is_array) @@ -3226,10 +3842,8 @@ class CppGenerator : public BaseGenerator { code_ += " {{STRUCT_NAME}}({{ARG_LIST}}) {"; } padding_id = 0; - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - const auto &field = **it; - const auto &type = field.value.type; + for (const auto &field : struct_def.fields.vec) { + const auto &type = field->value.type; if (IsArray(type) && init_arrays) { const auto &element_type = type.VectorType(); const auto is_enum = IsEnum(element_type); @@ -3239,14 +3853,14 @@ class CppGenerator : public BaseGenerator { const auto face_type = GenTypeGet(type, " ", "", "", is_enum); std::string get_array = is_enum ? "CastToArrayOfEnum<" + face_type + ">" : "CastToArray"; - const auto field_name = Name(field) + "_"; - const auto arg_name = "_" + Name(field); - code_ += " flatbuffers::" + get_array + "(" + field_name + + const auto field_name = Name(*field) + "_"; + const auto arg_name = "_" + Name(*field); + code_ += " ::flatbuffers::" + get_array + "(" + field_name + ").CopyFromSpan(" + arg_name + ");"; } - if (field.padding) { + if (field->padding) { std::string padding; - GenPadding(field, &padding, &padding_id, PaddingNoop); + GenPadding(*field, &padding, &padding_id, PaddingNoop); code_ += padding; } } @@ -3261,7 +3875,7 @@ class CppGenerator : public BaseGenerator { // It requires a specialization of Array class. // Generate Array<uint8_t> for Array<bool>. const auto face_type = GenTypeGet(type, " ", "", "", is_enum); - std::string ret_type = "flatbuffers::Array<" + face_type + ", " + + std::string ret_type = "::flatbuffers::Array<" + face_type + ", " + NumToString(type.fixed_length) + ">"; if (mutable_accessor) code_ += " " + ret_type + " *mutable_{{FIELD_NAME}}() {"; @@ -3270,7 +3884,7 @@ class CppGenerator : public BaseGenerator { std::string get_array = is_enum ? "CastToArrayOfEnum<" + face_type + ">" : "CastToArray"; - code_ += " return &flatbuffers::" + get_array + "({{FIELD_VALUE}});"; + code_ += " return &::flatbuffers::" + get_array + "({{FIELD_VALUE}});"; code_ += " }"; } @@ -3291,21 +3905,19 @@ class CppGenerator : public BaseGenerator { code_ += " private:"; int padding_id = 0; - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - const auto &field = **it; - const auto &field_type = field.value.type; + for (const auto &field : struct_def.fields.vec) { + const auto &field_type = field->value.type; code_.SetValue("FIELD_TYPE", GenTypeGet(field_type, " ", "", " ", false)); - code_.SetValue("FIELD_NAME", Name(field)); + code_.SetValue("FIELD_NAME", Name(*field)); code_.SetValue("ARRAY", IsArray(field_type) ? "[" + NumToString(field_type.fixed_length) + "]" : ""); code_ += (" {{FIELD_TYPE}}{{FIELD_NAME}}_{{ARRAY}};"); - if (field.padding) { + if (field->padding) { std::string padding; - GenPadding(field, &padding, &padding_id, PaddingDefinition); + GenPadding(*field, &padding, &padding_id, PaddingDefinition); code_ += padding; } } @@ -3319,7 +3931,7 @@ class CppGenerator : public BaseGenerator { // Make TypeTable accessible via the generated struct. if (opts_.mini_reflect != IDLOptions::kNone) { code_ += - " static const flatbuffers::TypeTable *MiniReflectTypeTable() {"; + " static const ::flatbuffers::TypeTable *MiniReflectTypeTable() {"; code_ += " return {{STRUCT_NAME}}TypeTable();"; code_ += " }"; } @@ -3333,35 +3945,31 @@ class CppGenerator : public BaseGenerator { // excluding arrays. GenStructConstructor(struct_def, kArrayArgModeNone); - auto arrays_num = std::count_if(struct_def.fields.vec.begin(), - struct_def.fields.vec.end(), - [](const flatbuffers::FieldDef *fd) { - return IsArray(fd->value.type); - }); + auto arrays_num = std::count_if( + struct_def.fields.vec.begin(), struct_def.fields.vec.end(), + [](const FieldDef *fd) { return IsArray(fd->value.type); }); if (arrays_num > 0) { GenStructConstructor(struct_def, kArrayArgModeSpanStatic); } // Generate accessor methods of the form: - // type name() const { return flatbuffers::EndianScalar(name_); } - for (auto it = struct_def.fields.vec.begin(); - it != struct_def.fields.vec.end(); ++it) { - const auto &field = **it; - const auto &type = field.value.type; + // type name() const { return ::flatbuffers::EndianScalar(name_); } + for (const auto &field : struct_def.fields.vec) { + const auto &type = field->value.type; const auto is_scalar = IsScalar(type.base_type); const auto is_array = IsArray(type); const auto field_type = GenTypeGet(type, " ", is_array ? "" : "const ", is_array ? "" : " &", true); - auto member = Name(field) + "_"; - auto value = - is_scalar ? "flatbuffers::EndianScalar(" + member + ")" : member; + auto member = Name(*field) + "_"; + const std::string &value = + is_scalar ? "::flatbuffers::EndianScalar(" + member + ")" : member; - code_.SetValue("FIELD_NAME", Name(field)); + code_.SetValue("FIELD_NAME", Name(*field)); code_.SetValue("FIELD_TYPE", field_type); - code_.SetValue("FIELD_VALUE", GenUnderlyingCast(field, true, value)); + code_.SetValue("FIELD_VALUE", GenUnderlyingCast(*field, true, value)); - GenComment(field.doc_comment, " "); + GenComment(field->doc_comment, " "); // Generate a const accessor function. if (is_array) { @@ -3380,11 +3988,11 @@ class CppGenerator : public BaseGenerator { if (is_scalar) { code_.SetValue("ARG", GenTypeBasic(type, true)); code_.SetValue("FIELD_VALUE", - GenUnderlyingCast(field, false, "_" + Name(field))); + GenUnderlyingCast(*field, false, "_" + Name(*field))); code_ += " void mutate_{{FIELD_NAME}}({{ARG}} _{{FIELD_NAME}}) {"; code_ += - " flatbuffers::WriteScalar(&{{FIELD_NAME}}_, " + " ::flatbuffers::WriteScalar(&{{FIELD_NAME}}_, " "{{FIELD_VALUE}});"; code_ += " }"; } else if (is_array) { @@ -3397,7 +4005,7 @@ class CppGenerator : public BaseGenerator { } // Generate a comparison function for this field if it is a key. - if (field.key) { GenKeyFieldMethods(field); } + if (field->key) { GenKeyFieldMethods(*field); } } code_.SetValue("NATIVE_NAME", Name(struct_def)); GenOperatorNewDelete(struct_def); @@ -3463,7 +4071,7 @@ bool GenerateCPP(const Parser &parser, const std::string &path, cpp::IDLOptionsCpp opts(parser.opts); // The '--cpp_std' argument could be extended (like ASAN): // Example: "flatc --cpp_std c++17:option1:option2". - auto cpp_std = !opts.cpp_std.empty() ? opts.cpp_std : "C++11"; + std::string cpp_std = !opts.cpp_std.empty() ? opts.cpp_std : "C++11"; std::transform(cpp_std.begin(), cpp_std.end(), cpp_std.begin(), CharToUpper); if (cpp_std == "C++0X") { opts.g_cpp_std = cpp::CPP_STD_X0; @@ -3499,16 +4107,70 @@ bool GenerateCPP(const Parser &parser, const std::string &path, std::string CPPMakeRule(const Parser &parser, const std::string &path, const std::string &file_name) { - const auto filebase = - flatbuffers::StripPath(flatbuffers::StripExtension(file_name)); + const auto filebase = StripPath(StripExtension(file_name)); cpp::CppGenerator geneartor(parser, path, file_name, parser.opts); const auto included_files = parser.GetIncludedFilesRecursive(file_name); std::string make_rule = geneartor.GeneratedFileName(path, filebase, parser.opts) + ": "; - for (auto it = included_files.begin(); it != included_files.end(); ++it) { - make_rule += " " + *it; + for (const std::string &included_file : included_files) { + make_rule += " " + included_file; } return make_rule; } +namespace { + +class CppCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateCPP(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + // Generate code from the provided `buffer` of given `length`. The buffer is a + // serialized reflection.fbs. + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + output = CPPMakeRule(parser, path, filename); + return Status::OK; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateCppGRPC(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kCpp; } + + std::string LanguageName() const override { return "C++"; } +}; + +} // namespace + +std::unique_ptr<CodeGenerator> NewCppCodeGenerator() { + return std::unique_ptr<CppCodeGenerator>(new CppCodeGenerator()); +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_cpp.h b/contrib/libs/flatbuffers/src/idl_gen_cpp.h new file mode 100644 index 00000000000..fcca063a8f6 --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_cpp.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_CPP_H_ +#define FLATBUFFERS_IDL_GEN_CPP_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Cpp code generator. +std::unique_ptr<CodeGenerator> NewCppCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_CPP_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_cpp_yandex_maps_iter.cpp b/contrib/libs/flatbuffers/src/idl_gen_cpp_yandex_maps_iter.cpp index 1ba2ef3e4a3..c368428950d 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_cpp_yandex_maps_iter.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_cpp_yandex_maps_iter.cpp @@ -181,7 +181,7 @@ class CppIterGenerator : public BaseGenerator { auto basename = flatbuffers::StripPath(noext); code_ += "#include \"" + parser_.opts.include_prefix + - (parser_.opts.keep_include_path ? noext : basename) + + (parser_.opts.keep_prefix ? noext : basename) + ".iter.fbs.h\""; num_includes++; } @@ -488,7 +488,17 @@ class CppIterGenerator : public BaseGenerator { code_.SetValue("REQUIRED", field.IsRequired() ? "Required" : ""); code_.SetValue("SIZE", GenTypeSize(field.value.type)); code_.SetValue("OFFSET", GenFieldOffsetName(field)); - code_ += "{{PRE}}this->template VerifyField{{REQUIRED}}<{{SIZE}}>(verifier, {{OFFSET}})\\"; + if (IsScalar(field.value.type.base_type) || IsStruct(field.value.type)) { + code_.SetValue("ALIGN", NumToString(InlineAlignment(field.value.type))); + code_ += + "{{PRE}}this->template VerifyField{{REQUIRED}}<{{SIZE}}>(verifier, " + "{{OFFSET}}, {{ALIGN}})\\"; + } else { + code_.SetValue("OFFSET_SIZE", field.offset64 ? "64" : ""); + code_ += + "{{PRE}}this->template VerifyOffset{{REQUIRED}}<{{SIZE}}>(verifier, " + "{{OFFSET}})\\"; + } switch (field.value.type.base_type) { case BASE_TYPE_UNION: { @@ -728,4 +738,55 @@ bool GenerateCPPYandexMapsIter(const Parser &parser, const std::string &path, return generator.generate(); } +namespace cpp_yandex_maps_iter { + +class CppIterCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateCPPYandexMapsIter(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + return Status::NOT_IMPLEMENTED; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kCppYandexMapsIter; } + + std::string LanguageName() const override { return "C++Iter"; } +}; + +} // namespace + +std::unique_ptr<CodeGenerator> NewCppYandexMapsIterCodeGenerator() { + return std::unique_ptr<cpp_yandex_maps_iter::CppIterCodeGenerator>(new cpp_yandex_maps_iter::CppIterCodeGenerator()); +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_cpp_yandex_maps_iter.h b/contrib/libs/flatbuffers/src/idl_gen_cpp_yandex_maps_iter.h new file mode 100644 index 00000000000..87b2449367c --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_cpp_yandex_maps_iter.h @@ -0,0 +1,13 @@ +#ifndef FLATBUFFERS_IDL_GEN_CPP_YANDEX_MAPS_ITER_H_ +#define FLATBUFFERS_IDL_GEN_CPP_YANDEX_MAPS_ITER_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Cpp Yandex Maps Iter code generator. +std::unique_ptr<CodeGenerator> NewCppYandexMapsIterCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_CPP_YANDEX_MAPS_ITER_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_csharp.cpp b/contrib/libs/flatbuffers/src/idl_gen_csharp.cpp index 681ab6d642f..0dbcd4b927f 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_csharp.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_csharp.cpp @@ -16,15 +16,15 @@ // independent from idl_parser, since this code is not needed for most clients +#include "idl_gen_csharp.h" + +#include <unordered_set> + #include "flatbuffers/code_generators.h" #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" -#if defined(FLATBUFFERS_CPP98_STL) -# include <cctype> -#endif // defined(FLATBUFFERS_CPP98_STL) - namespace flatbuffers { static TypedFloatConstantGenerator CSharpFloatGen("Double.", "Single.", "NaN", @@ -46,8 +46,100 @@ class CSharpGenerator : public BaseGenerator { public: CSharpGenerator(const Parser &parser, const std::string &path, const std::string &file_name) - : BaseGenerator(parser, path, file_name, "", ".", "cs"), - cur_name_space_(nullptr) {} + : BaseGenerator(parser, path, file_name, + parser.opts.cs_global_alias ? "global::" : "", ".", "cs"), + cur_name_space_(nullptr) { + // clang-format off + + // List of keywords retrieved from here: + // https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/keywords/ + + // One per line to ease comparisons to that list are easier + + static const char *const keywords[] = { + "abstract", + "as", + "base", + "bool", + "break", + "byte", + "case", + "catch", + "char", + "checked", + "class", + "const", + "continue", + "decimal", + "default", + "delegate", + "do", + "double", + "else", + "enum", + "event", + "explicit", + "extern", + "false", + "finally", + "fixed", + "float", + "for", + "foreach", + "goto", + "if", + "implicit", + "in", + "int", + "interface", + "internal", + "is", + "lock", + "long", + "namespace", + "new", + "null", + "object", + "operator", + "out", + "override", + "params", + "private", + "protected", + "public", + "readonly", + "ref", + "return", + "sbyte", + "sealed", + "short", + "sizeof", + "stackalloc", + "static", + "string", + "struct", + "switch", + "this", + "throw", + "true", + "try", + "typeof", + "uint", + "ulong", + "unchecked", + "unsafe", + "ushort", + "using", + "virtual", + "void", + "volatile", + "while", + nullptr, + // clang-format on + }; + + for (auto kw = keywords; *kw; kw++) keywords_.insert(*kw); + } CSharpGenerator &operator=(const CSharpGenerator &); @@ -65,7 +157,7 @@ class CSharpGenerator : public BaseGenerator { one_file_code += enumcode; } else { if (!SaveType(enum_def.name, *enum_def.defined_namespace, enumcode, - false)) + false, parser_.opts)) return false; } } @@ -77,26 +169,50 @@ class CSharpGenerator : public BaseGenerator { if (!parser_.opts.one_file) cur_name_space_ = struct_def.defined_namespace; GenStruct(struct_def, &declcode, parser_.opts); + GenStructVerifier(struct_def, &declcode); if (parser_.opts.one_file) { one_file_code += declcode; } else { if (!SaveType(struct_def.name, *struct_def.defined_namespace, declcode, - true)) + true, parser_.opts)) return false; } } if (parser_.opts.one_file) { return SaveType(file_name_, *parser_.current_namespace_, one_file_code, - true); + true, parser_.opts); } return true; } + private: + std::unordered_set<std::string> keywords_; + + std::string EscapeKeyword(const std::string &name) const { + return keywords_.find(name) == keywords_.end() ? name : "@" + name; + } + + std::string Name(const FieldDef &field) const { + std::string name = ConvertCase(field.name, Case::kUpperCamel); + return EscapeKeyword(name); + } + + std::string Name(const Definition &def) const { + return EscapeKeyword(def.name); + } + + std::string NamespacedName(const Definition &def) const { + return WrapInNameSpace(def.defined_namespace, Name(def)); + } + + std::string Name(const EnumVal &ev) const { return EscapeKeyword(ev.name); } + // Save out the generated code for a single class while adding // declaration boilerplate. bool SaveType(const std::string &defname, const Namespace &ns, - const std::string &classcode, bool needs_includes) const { + const std::string &classcode, bool needs_includes, + const IDLOptions &options) const { if (!classcode.length()) return true; std::string code = @@ -113,11 +229,14 @@ class CSharpGenerator : public BaseGenerator { if (needs_includes) { code += "using global::System;\n"; code += "using global::System.Collections.Generic;\n"; - code += "using global::FlatBuffers;\n\n"; + code += "using global::Google.FlatBuffers;\n\n"; } code += classcode; if (!namespace_name.empty()) { code += "\n}\n"; } - auto filename = NamespaceDir(ns) + defname + ".cs"; + auto filename = NamespaceDir(ns) + defname; + if (options.one_file) { filename += options.filename_suffix; } + filename += + options.filename_extension.empty() ? ".cs" : options.filename_extension; return SaveFile(filename.c_str(), code, false); } @@ -134,9 +253,9 @@ class CSharpGenerator : public BaseGenerator { // clang-format on if (enableLangOverrides) { - if (IsEnum(type)) return WrapInNameSpace(*type.enum_def); + if (IsEnum(type)) return NamespacedName(*type.enum_def); if (type.base_type == BASE_TYPE_STRUCT) { - return "Offset<" + WrapInNameSpace(*type.struct_def) + ">"; + return "Offset<" + NamespacedName(*type.struct_def) + ">"; } } @@ -151,7 +270,7 @@ class CSharpGenerator : public BaseGenerator { switch (type.base_type) { case BASE_TYPE_STRING: return "string"; case BASE_TYPE_VECTOR: return GenTypeGet(type.VectorType()); - case BASE_TYPE_STRUCT: return WrapInNameSpace(*type.struct_def); + case BASE_TYPE_STRUCT: return NamespacedName(*type.struct_def); case BASE_TYPE_UNION: return "TTable"; default: return "Table"; } @@ -165,12 +284,12 @@ class CSharpGenerator : public BaseGenerator { } std::string GenOffsetType(const StructDef &struct_def) const { - return "Offset<" + WrapInNameSpace(struct_def) + ">"; + return "Offset<" + NamespacedName(struct_def) + ">"; } std::string GenOffsetConstruct(const StructDef &struct_def, const std::string &variable_name) const { - return "new Offset<" + WrapInNameSpace(struct_def) + ">(" + variable_name + + return "new Offset<" + NamespacedName(struct_def) + ">(" + variable_name + ")"; } @@ -179,7 +298,7 @@ class CSharpGenerator : public BaseGenerator { if (IsSeries(type)) { return DestinationCast(type.VectorType()); } else { - if (IsEnum(type)) return "(" + WrapInNameSpace(*type.enum_def) + ")"; + if (IsEnum(type)) return "(" + NamespacedName(*type.enum_def) + ")"; } return ""; } @@ -190,17 +309,19 @@ class CSharpGenerator : public BaseGenerator { // would be cast down to int before being put onto the buffer. In C#, one cast // directly cast an Enum to its underlying type, which is essential before // putting it onto the buffer. - std::string SourceCast(const Type &type) const { + std::string SourceCast(const Type &type, + const bool isOptional = false) const { if (IsSeries(type)) { return SourceCast(type.VectorType()); } else { - if (IsEnum(type)) return "(" + GenTypeBasic(type, false) + ")"; + if (IsEnum(type)) + return "(" + GenTypeBasic(type, false) + (isOptional ? "?" : "") + ")"; } return ""; } - std::string SourceCastBasic(const Type &type) const { - return IsScalar(type.base_type) ? SourceCast(type) : ""; + std::string SourceCastBasic(const Type &type, const bool isOptional) const { + return IsScalar(type.base_type) ? SourceCast(type, isOptional) : ""; } std::string GenEnumDefaultValue(const FieldDef &field) const { @@ -208,7 +329,7 @@ class CSharpGenerator : public BaseGenerator { FLATBUFFERS_ASSERT(value.type.enum_def); auto &enum_def = *value.type.enum_def; auto enum_val = enum_def.FindByValue(value.constant); - return enum_val ? (WrapInNameSpace(enum_def) + "." + enum_val->name) + return enum_val ? (NamespacedName(enum_def) + "." + Name(*enum_val)) : value.constant; } @@ -252,7 +373,7 @@ class CSharpGenerator : public BaseGenerator { switch (value.type.base_type) { case BASE_TYPE_STRING: return "default(StringOffset)"; case BASE_TYPE_STRUCT: - return "default(Offset<" + WrapInNameSpace(*value.type.struct_def) + + return "default(Offset<" + NamespacedName(*value.type.struct_def) + ">)"; case BASE_TYPE_VECTOR: return "default(VectorOffset)"; default: break; @@ -293,14 +414,14 @@ class CSharpGenerator : public BaseGenerator { } else { code += "public "; } - code += "enum " + enum_def.name; + code += "enum " + Name(enum_def); code += " : " + GenTypeBasic(enum_def.underlying_type, false); code += "\n{\n"; for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { auto &ev = **it; GenComment(ev.doc_comment, code_ptr, &comment_config, " "); code += " "; - code += ev.name + " = "; + code += Name(ev) + " = "; code += enum_def.ToString(ev); code += ",\n"; } @@ -334,27 +455,32 @@ class CSharpGenerator : public BaseGenerator { if (type.base_type == BASE_TYPE_BOOL) { getter = "0!=" + getter; } else if (GenTypeBasic(type, false) != "byte") { - getter += MakeCamel(GenTypeBasic(type, false)); + getter += ConvertCase(GenTypeBasic(type, false), Case::kUpperCamel); } return getter; } } } + std::string GetObjectConstructor(flatbuffers::StructDef &struct_def, + const std::string &data_buffer, + const std::string &offset) const { + // Use the generated type directly, to properly handle default values that + // might not be written to the buffer. + return "new " + Name(struct_def) + "().__assign(" + offset + ", " + + data_buffer + ")"; + } + // Returns the function name that is able to read a value of the given type. - std::string GenGetterForLookupByKey(flatbuffers::FieldDef *key_field, + std::string GenGetterForLookupByKey(flatbuffers::StructDef &struct_def, + flatbuffers::FieldDef *key_field, const std::string &data_buffer, - const char *num = nullptr) const { - auto type = key_field->value.type; - auto dest_mask = ""; - auto dest_cast = DestinationCast(type); - auto getter = data_buffer + ".Get"; - if (GenTypeBasic(type, false) != "byte") { - getter += MakeCamel(GenTypeBasic(type, false)); - } - getter = dest_cast + getter + "(" + GenOffsetGetter(key_field, num) + ")" + - dest_mask; - return getter; + const std::string &offset) const { + // Use the generated type directly, to properly handle default values that + // might not be written to the buffer. + auto name = Name(*key_field); + if (name == struct_def.name) { name += "_"; } + return GetObjectConstructor(struct_def, data_buffer, offset) + "." + name; } // Direct mutation is only allowed for scalar fields. @@ -364,7 +490,7 @@ class CSharpGenerator : public BaseGenerator { std::string setter = "__p.bb.Put"; if (GenTypeBasic(type, false) != "byte" && type.base_type != BASE_TYPE_BOOL) { - setter += MakeCamel(GenTypeBasic(type, false)); + setter += ConvertCase(GenTypeBasic(type, false), Case::kUpperCamel); } return setter; } else { @@ -374,8 +500,9 @@ class CSharpGenerator : public BaseGenerator { // Returns the method name for use with add/put calls. std::string GenMethod(const Type &type) const { - return IsScalar(type.base_type) ? MakeCamel(GenTypeBasic(type, false)) - : (IsStruct(type) ? "Struct" : "Offset"); + return IsScalar(type.base_type) + ? ConvertCase(GenTypeBasic(type, false), Case::kUpperCamel) + : (IsStruct(type) ? "Struct" : "Offset"); } // Recursively generate arguments for a constructor, to deal with nested @@ -395,7 +522,8 @@ class CSharpGenerator : public BaseGenerator { // don't clash, and to make it obvious these arguments are constructing // a nested struct, prefix the name with the field name. GenStructArgs(*field_type.struct_def, code_ptr, - (nameprefix + (field.name + "_")).c_str(), array_cnt); + (nameprefix + (EscapeKeyword(field.name) + "_")).c_str(), + array_cnt); } else { code += ", "; code += GenTypeBasic(type); @@ -407,7 +535,7 @@ class CSharpGenerator : public BaseGenerator { } code += " "; code += nameprefix; - code += MakeCamel(field.name, true); + code += Name(field); } } } @@ -454,7 +582,7 @@ class CSharpGenerator : public BaseGenerator { code += indent + " builder.Put"; code += GenMethod(type) + "("; code += SourceCast(type); - auto argname = nameprefix + MakeCamel(field.name, true); + auto argname = nameprefix + Name(field); code += argname; size_t array_cnt = index + (IsArray(field_type) ? 1 : 0); if (array_cnt > 0) { @@ -485,37 +613,182 @@ class CSharpGenerator : public BaseGenerator { return key_offset; } - std::string GenLookupKeyGetter(flatbuffers::FieldDef *key_field) const { - std::string key_getter = " "; - key_getter += "int tableOffset = Table."; - key_getter += "__indirect(vectorLocation + 4 * (start + middle)"; - key_getter += ", bb);\n "; - if (IsString(key_field->value.type)) { - key_getter += "int comp = Table."; - key_getter += "CompareStrings("; - key_getter += GenOffsetGetter(key_field); - key_getter += ", byteKey, bb);\n"; + std::string GenKeyGetter(flatbuffers::StructDef &struct_def, + flatbuffers::FieldDef *key_field) const { + // Get the getter for the key of the struct. + return GenGetterForLookupByKey(struct_def, key_field, "builder.DataBuffer", + "builder.DataBuffer.Length - o1.Value") + + ".CompareTo(" + + GenGetterForLookupByKey(struct_def, key_field, "builder.DataBuffer", + "builder.DataBuffer.Length - o2.Value") + + ")"; + } + + // Get the value of a table verification function start + void GetStartOfTableVerifier(const StructDef &struct_def, std::string *code_ptr) { + std::string &code = *code_ptr; + code += "\n"; + code += "static public class " + struct_def.name + "Verify\n"; + code += "{\n"; + code += " static public bool Verify"; + code += "(Google.FlatBuffers.Verifier verifier, uint tablePos)\n"; + code += " {\n"; + code += " return verifier.VerifyTableStart(tablePos)\n"; + } + + // Get the value of a table verification function end + void GetEndOfTableVerifier(std::string *code_ptr) { + std::string &code = *code_ptr; + code += " && verifier.VerifyTableEnd(tablePos);\n"; + code += " }\n"; + code += "}\n"; + } + + std::string GetNestedFlatBufferName(const FieldDef &field) { + std::string name; + if (field.nested_flatbuffer) { + name = NamespacedName(*field.nested_flatbuffer); + } else { + name = ""; + } + return name ; + } + + // Generate the code to call the appropriate Verify function(s) for a field. + void GenVerifyCall(CodeWriter &code_, const FieldDef &field, const char *prefix) { + code_.SetValue("PRE", prefix); + code_.SetValue("NAME", ConvertCase(field.name, Case::kUpperCamel)); + code_.SetValue("REQUIRED", field.IsRequired() ? "Required" : ""); + code_.SetValue("REQUIRED_FLAG", field.IsRequired() ? "true" : "false"); + code_.SetValue("TYPE", GenTypeGet(field.value.type)); + code_.SetValue("INLINESIZE", NumToString(InlineSize(field.value.type))); + code_.SetValue("OFFSET", NumToString(field.value.offset)); + + if (IsScalar(field.value.type.base_type) || IsStruct(field.value.type)) { + code_.SetValue("ALIGN", NumToString(InlineAlignment(field.value.type))); + code_ += + "{{PRE}} && verifier.VerifyField(tablePos, " + "{{OFFSET}} /*{{NAME}}*/, {{INLINESIZE}} /*{{TYPE}}*/, {{ALIGN}}, {{REQUIRED_FLAG}})"; } else { - auto get_val = GenGetterForLookupByKey(key_field, "bb"); - key_getter += "int comp = " + get_val + ".CompareTo(key);\n"; + // TODO - probably code below should go to this 'else' - code_ += "{{PRE}}VerifyOffset{{REQUIRED}}(verifier, {{OFFSET}})\\"; + } + + switch (field.value.type.base_type) { + case BASE_TYPE_UNION: { + auto union_name = NamespacedName(*field.value.type.enum_def); + code_.SetValue("ENUM_NAME1", field.value.type.enum_def->name); + code_.SetValue("ENUM_NAME", union_name); + code_.SetValue("SUFFIX", UnionTypeFieldSuffix()); + // Caution: This construction assumes, that UNION type id element has been created just before union data and + // its offset precedes union. Such assumption is common in flatbuffer implementation + code_.SetValue("TYPE_ID_OFFSET", NumToString(field.value.offset - sizeof(voffset_t))); + code_ += "{{PRE}} && verifier.VerifyUnion(tablePos, {{TYPE_ID_OFFSET}}, " + "{{OFFSET}} /*{{NAME}}*/, {{ENUM_NAME}}Verify.Verify, {{REQUIRED_FLAG}})"; + break; + } + case BASE_TYPE_STRUCT: { + if (!field.value.type.struct_def->fixed) { + code_ += "{{PRE}} && verifier.VerifyTable(tablePos, " + "{{OFFSET}} /*{{NAME}}*/, {{TYPE}}Verify.Verify, {{REQUIRED_FLAG}})"; + } + break; + } + case BASE_TYPE_STRING: { + code_ += "{{PRE}} && verifier.VerifyString(tablePos, " + "{{OFFSET}} /*{{NAME}}*/, {{REQUIRED_FLAG}})"; + break; + } + case BASE_TYPE_VECTOR: { + + switch (field.value.type.element) { + case BASE_TYPE_STRING: { + code_ += "{{PRE}} && verifier.VerifyVectorOfStrings(tablePos, " + "{{OFFSET}} /*{{NAME}}*/, {{REQUIRED_FLAG}})"; + break; + } + case BASE_TYPE_STRUCT: { + if (!field.value.type.struct_def->fixed) { + code_ += "{{PRE}} && verifier.VerifyVectorOfTables(tablePos, " + "{{OFFSET}} /*{{NAME}}*/, {{TYPE}}Verify.Verify, {{REQUIRED_FLAG}})"; + } else { + code_.SetValue( + "VECTOR_ELEM_INLINESIZE", + NumToString(InlineSize(field.value.type.VectorType()))); + code_ += + "{{PRE}} && " + "verifier.VerifyVectorOfData(tablePos, " + "{{OFFSET}} /*{{NAME}}*/, {{VECTOR_ELEM_INLINESIZE}} " + "/*{{TYPE}}*/, {{REQUIRED_FLAG}})"; + } + break; + } + case BASE_TYPE_UNION: { + // Vectors of unions are not yet supported for go + break; + } + default: + // Generate verifier for vector of data. + // It may be either nested flatbuffer of just vector of bytes + auto nfn = GetNestedFlatBufferName(field); + if (!nfn.empty()) { + code_.SetValue("CPP_NAME", nfn); + // FIXME: file_identifier. + code_ += "{{PRE}} && verifier.VerifyNestedBuffer(tablePos, " + "{{OFFSET}} /*{{NAME}}*/, {{CPP_NAME}}Verify.Verify, {{REQUIRED_FLAG}})"; + } else if (field.flexbuffer) { + code_ += "{{PRE}} && verifier.VerifyNestedBuffer(tablePos, " + "{{OFFSET}} /*{{NAME}}*/, null, {{REQUIRED_FLAG}})"; + } else { + code_.SetValue("VECTOR_ELEM_INLINESIZE", NumToString(InlineSize(field.value.type.VectorType()))); + code_ += + "{{PRE}} && verifier.VerifyVectorOfData(tablePos, " + "{{OFFSET}} /*{{NAME}}*/, {{VECTOR_ELEM_INLINESIZE}} /*{{TYPE}}*/, {{REQUIRED_FLAG}})"; + } + break; + } + + break; + } + default: { + break; + } + } + } + + // Generate table constructors, conditioned on its members' types. + void GenTableVerifier(const StructDef &struct_def, std::string *code_ptr) { + CodeWriter code_; + + GetStartOfTableVerifier(struct_def, code_ptr); + + // Generate struct fields accessors + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + auto &field = **it; + if (field.deprecated) continue; + + GenVerifyCall(code_, field, ""); } - return key_getter; + + *code_ptr += code_.ToString(); + + GetEndOfTableVerifier(code_ptr); } - std::string GenKeyGetter(flatbuffers::FieldDef *key_field) const { - std::string key_getter = ""; - auto data_buffer = "builder.DataBuffer"; - if (IsString(key_field->value.type)) { - key_getter += "Table.CompareStrings("; - key_getter += GenOffsetGetter(key_field, "o1") + ", "; - key_getter += GenOffsetGetter(key_field, "o2") + ", " + data_buffer + ")"; + // Generate struct or table methods. + void GenStructVerifier(const StructDef &struct_def, std::string *code_ptr) { + if (struct_def.generated) return; + + // cur_name_space_ = struct_def.defined_namespace; + + // Generate verifiers + if (struct_def.fixed) { + // Fixed size structures do not require table members + // verification - instead structure size is verified using VerifyField } else { - auto field_getter = GenGetterForLookupByKey(key_field, data_buffer, "o1"); - key_getter += field_getter; - field_getter = GenGetterForLookupByKey(key_field, data_buffer, "o2"); - key_getter += ".CompareTo(" + field_getter + ")"; + // Create table verification function + GenTableVerifier(struct_def, code_ptr); } - return key_getter; } void GenStruct(StructDef &struct_def, std::string *code_ptr, @@ -549,11 +822,11 @@ class CSharpGenerator : public BaseGenerator { code += " public ByteBuffer ByteBuffer { get { return __p.bb; } }\n"; if (!struct_def.fixed) { - // Generate verson check method. + // Generate version check method. // Force compile time error if not using the same version runtime. code += " public static void ValidateVersion() {"; code += " FlatBufferConstants."; - code += "FLATBUFFERS_2_0_0(); "; + code += "FLATBUFFERS_23_5_9(); "; code += "}\n"; // Generate a special accessor for the table that when used as the root @@ -583,8 +856,20 @@ class CSharpGenerator : public BaseGenerator { code += parser_.file_identifier_; code += "\"); }\n"; } + + // Generate the Verify method that checks if a ByteBuffer is save to + // access + code += " public static "; + code += "bool Verify" + struct_def.name + "(ByteBuffer _bb) {"; + code += "Google.FlatBuffers.Verifier verifier = new "; + code += "Google.FlatBuffers.Verifier(_bb); "; + code += "return verifier.VerifyBuffer(\""; + code += parser_.file_identifier_; + code += "\", false, " + struct_def.name + "Verify.Verify);"; + code += " }\n"; } } + // Generate the __init method that sets the field in a pre-existing // accessor object. This is to allow object reuse. code += " public void __init(int _i, ByteBuffer _bb) "; @@ -618,7 +903,8 @@ class CSharpGenerator : public BaseGenerator { std::string dest_mask = ""; std::string dest_cast = DestinationCast(field.value.type); std::string src_cast = SourceCast(field.value.type); - std::string field_name_camel = MakeCamel(field.name, true); + std::string field_name_camel = Name(field); + if (field_name_camel == struct_def.name) { field_name_camel += "_"; } std::string method_start = " public " + type_name_dest + optional + " " + field_name_camel; std::string obj = "(new " + type_name + "())"; @@ -745,8 +1031,7 @@ class CSharpGenerator : public BaseGenerator { HasUnionStringValue(*vectortype.enum_def)) { code += member_suffix; code += "}\n"; - code += " public string " + MakeCamel(field.name, true) + - "AsString(int j)"; + code += " public string " + Name(field) + "AsString(int j)"; code += offset_prefix + GenGetter(Type(BASE_TYPE_STRING)); code += "(" + index + ") : null"; } @@ -759,8 +1044,7 @@ class CSharpGenerator : public BaseGenerator { if (HasUnionStringValue(*field.value.type.enum_def)) { code += member_suffix; code += "}\n"; - code += " public string " + MakeCamel(field.name, true) + - "AsString()"; + code += " public string " + Name(field) + "AsString()"; code += offset_prefix + GenGetter(Type(BASE_TYPE_STRING)); code += "(o + __p.bb_pos) : null"; } @@ -795,7 +1079,9 @@ class CSharpGenerator : public BaseGenerator { code += member_suffix; code += "}\n"; if (IsVector(field.value.type)) { - code += " public int " + MakeCamel(field.name, true); + auto camel_name = Name(field); + if (camel_name == struct_def.name) { camel_name += "_"; } + code += " public int " + camel_name; code += "Length"; code += " { get"; code += offset_prefix; @@ -810,9 +1096,9 @@ class CSharpGenerator : public BaseGenerator { for (auto kit = fields.begin(); kit != fields.end(); ++kit) { auto &key_field = **kit; if (key_field.key) { - auto qualified_name = WrapInNameSpace(sd); + auto qualified_name = NamespacedName(sd); code += " public " + qualified_name + "? "; - code += MakeCamel(field.name, true) + "ByKey("; + code += Name(field) + "ByKey("; code += GenTypeGet(key_field.value.type) + " key)"; code += offset_prefix; code += qualified_name + ".__lookup_by_key("; @@ -831,7 +1117,7 @@ class CSharpGenerator : public BaseGenerator { code += "#if ENABLE_SPAN_T\n"; code += " public Span<" + GenTypeBasic(field.value.type.VectorType()) + "> Get"; - code += MakeCamel(field.name, true); + code += Name(field); code += "Bytes() { return "; code += "__p.__vector_as_span<" + GenTypeBasic(field.value.type.VectorType()) + ">("; @@ -841,7 +1127,7 @@ class CSharpGenerator : public BaseGenerator { code += "); }\n"; code += "#else\n"; code += " public ArraySegment<byte>? Get"; - code += MakeCamel(field.name, true); + code += Name(field); code += "Bytes() { return "; code += "__p.__vector_as_arraysegment("; code += NumToString(field.value.offset); @@ -852,7 +1138,7 @@ class CSharpGenerator : public BaseGenerator { code += " public "; code += GenTypeBasic(field.value.type.VectorType()); code += "[] Get"; - code += MakeCamel(field.name, true); + code += Name(field); code += "Array() { "; if (IsEnum(field.value.type.VectorType())) { // Since __vector_as_array does not work for enum types, @@ -881,9 +1167,9 @@ class CSharpGenerator : public BaseGenerator { } // generate object accessors if is nested_flatbuffer if (field.nested_flatbuffer) { - auto nested_type_name = WrapInNameSpace(*field.nested_flatbuffer); + auto nested_type_name = NamespacedName(*field.nested_flatbuffer); auto nested_method_name = - MakeCamel(field.name, true) + "As" + field.nested_flatbuffer->name; + Name(field) + "As" + field.nested_flatbuffer->name; auto get_nested_method_name = nested_method_name; get_nested_method_name = "Get" + nested_method_name; conditional_cast = "(" + nested_type_name + "?)"; @@ -904,15 +1190,16 @@ class CSharpGenerator : public BaseGenerator { is_series ? field.value.type.VectorType() : field.value.type; // Boolean parameters have to be explicitly converted to byte // representation. - auto setter_parameter = underlying_type.base_type == BASE_TYPE_BOOL - ? "(byte)(" + field.name + " ? 1 : 0)" - : field.name; - auto mutator_prefix = MakeCamel("mutate", true); + auto setter_parameter = + underlying_type.base_type == BASE_TYPE_BOOL + ? "(byte)(" + EscapeKeyword(field.name) + " ? 1 : 0)" + : EscapeKeyword(field.name); + auto mutator_prefix = "Mutate"; // A vector mutator also needs the index of the vector element it should // mutate. auto mutator_params = (is_series ? "(int j, " : "(") + - GenTypeGet(underlying_type) + " " + field.name + - ") { "; + GenTypeGet(underlying_type) + " " + + EscapeKeyword(field.name) + ") { "; auto setter_index = is_series ? "__p." + @@ -926,7 +1213,7 @@ class CSharpGenerator : public BaseGenerator { if (IsScalar(underlying_type.base_type) && !IsUnion(field.value.type)) { code += " public "; code += struct_def.fixed ? "void " : "bool "; - code += mutator_prefix + MakeCamel(field.name, true); + code += mutator_prefix + Name(field); code += mutator_params; if (struct_def.fixed) { code += GenSetter(underlying_type) + "(" + setter_index + ", "; @@ -942,9 +1229,10 @@ class CSharpGenerator : public BaseGenerator { } if (parser_.opts.java_primitive_has_method && IsScalar(field.value.type.base_type) && !struct_def.fixed) { - auto vt_offset_constant = " public static final int VT_" + - MakeScreamingCamel(field.name) + " = " + - NumToString(field.value.offset) + ";"; + auto vt_offset_constant = + " public static final int VT_" + + ConvertCase(field.name, Case::kScreamingSnake) + " = " + + NumToString(field.value.offset) + ";"; code += vt_offset_constant; code += "\n"; @@ -1002,13 +1290,13 @@ class CSharpGenerator : public BaseGenerator { field.value.type.struct_def->defined_namespace, GenTypeName_ObjectAPI(field.value.type.struct_def->name, opts)); code += " "; - code += field.name; + code += EscapeKeyword(field.name); code += " = null"; } else { code += GenTypeBasic(field.value.type); if (field.IsScalarOptional()) { code += "?"; } code += " "; - code += field.name; + code += EscapeKeyword(field.name); if (!IsScalar(field.value.type.base_type)) code += "Offset"; code += " = "; @@ -1028,13 +1316,13 @@ class CSharpGenerator : public BaseGenerator { size == SizeOf(field.value.type.base_type))) { code += " " + struct_def.name + "."; code += "Add"; - code += MakeCamel(field.name) + "(builder, "; + code += Name(field) + "(builder, "; if (IsStruct(field.value.type) && opts.generate_object_based_api) { code += GenTypePointer(field.value.type) + ".Pack(builder, " + - field.name + ")"; + EscapeKeyword(field.name) + ")"; } else { - code += field.name; + code += EscapeKeyword(field.name); if (!IsScalar(field.value.type.base_type)) code += "Offset"; } @@ -1062,17 +1350,17 @@ class CSharpGenerator : public BaseGenerator { if (field.deprecated) continue; if (field.key) key_field = &field; code += " public static void Add"; - code += MakeCamel(field.name); + code += Name(field); code += "(FlatBufferBuilder builder, "; code += GenTypeBasic(field.value.type); - auto argname = MakeCamel(field.name, false); + auto argname = ConvertCase(field.name, Case::kLowerCamel); if (!IsScalar(field.value.type.base_type)) argname += "Offset"; if (field.IsScalarOptional()) { code += "?"; } - code += " " + argname + ") { builder.Add"; + code += " " + EscapeKeyword(argname) + ") { builder.Add"; code += GenMethod(field.value.type) + "("; code += NumToString(it - struct_def.fields.vec.begin()) + ", "; - code += SourceCastBasic(field.value.type); - code += argname; + code += SourceCastBasic(field.value.type, field.IsScalarOptional()); + code += EscapeKeyword(argname); if (!IsScalar(field.value.type.base_type) && field.value.type.base_type != BASE_TYPE_UNION) { code += ".Value"; @@ -1093,7 +1381,7 @@ class CSharpGenerator : public BaseGenerator { field_has_create_set.insert(&field); code += " public static VectorOffset "; code += "Create"; - code += MakeCamel(field.name); + code += Name(field); code += "Vector(FlatBufferBuilder builder, "; code += GenTypeBasic(vector_type) + "[] data) "; code += "{ builder.StartVector("; @@ -1105,7 +1393,11 @@ class CSharpGenerator : public BaseGenerator { code += "Add"; code += GenMethod(vector_type); code += "("; - code += SourceCastBasic(vector_type); + // At the moment there is no support of the type Vector with + // optional enum, e.g. if we have enum type SomeEnum there is no way + // to define `SomeEmum?[] enums` in FlatBuffer schema, so isOptional + // = false + code += SourceCastBasic(vector_type, false); code += "data[i]"; if (vector_type.base_type == BASE_TYPE_STRUCT || IsString(vector_type)) @@ -1113,9 +1405,11 @@ class CSharpGenerator : public BaseGenerator { code += "); return "; code += "builder.EndVector(); }\n"; + // add Create...VectorBlock() overloads for T[], ArraySegment<T> and + // IntPtr code += " public static VectorOffset "; code += "Create"; - code += MakeCamel(field.name); + code += Name(field); code += "VectorBlock(FlatBufferBuilder builder, "; code += GenTypeBasic(vector_type) + "[] data) "; code += "{ builder.StartVector("; @@ -1123,11 +1417,31 @@ class CSharpGenerator : public BaseGenerator { code += ", data.Length, "; code += NumToString(alignment); code += "); builder.Add(data); return builder.EndVector(); }\n"; + + code += " public static VectorOffset "; + code += "Create"; + code += Name(field); + code += "VectorBlock(FlatBufferBuilder builder, "; + code += "ArraySegment<" + GenTypeBasic(vector_type) + "> data) "; + code += "{ builder.StartVector("; + code += NumToString(elem_size); + code += ", data.Count, "; + code += NumToString(alignment); + code += "); builder.Add(data); return builder.EndVector(); }\n"; + + code += " public static VectorOffset "; + code += "Create"; + code += Name(field); + code += "VectorBlock(FlatBufferBuilder builder, "; + code += "IntPtr dataPtr, int sizeInBytes) "; + code += "{ builder.StartVector(1, sizeInBytes, 1); "; + code += "builder.Add<" + GenTypeBasic(vector_type) + + ">(dataPtr, sizeInBytes); return builder.EndVector(); }\n"; } // Generate a method to start a vector, data to be added manually // after. code += " public static void Start"; - code += MakeCamel(field.name); + code += Name(field); code += "Vector(FlatBufferBuilder builder, int numElems) "; code += "{ builder.StartVector("; code += NumToString(elem_size); @@ -1170,14 +1484,17 @@ class CSharpGenerator : public BaseGenerator { // because `key_field` is not set for struct if (struct_def.has_key && !struct_def.fixed) { FLATBUFFERS_ASSERT(key_field); + auto name = Name(*key_field); + if (name == struct_def.name) { name += "_"; } code += "\n public static VectorOffset "; code += "CreateSortedVectorOf" + struct_def.name; code += "(FlatBufferBuilder builder, "; code += "Offset<" + struct_def.name + ">"; code += "[] offsets) {\n"; - code += " Array.Sort(offsets, (Offset<" + struct_def.name + - "> o1, Offset<" + struct_def.name + "> o2) => " + - GenKeyGetter(key_field); + code += " Array.Sort(offsets,\n"; + code += " (Offset<" + struct_def.name + "> o1, Offset<" + + struct_def.name + "> o2) =>\n"; + code += " " + GenKeyGetter(struct_def, key_field); code += ");\n"; code += " return builder.CreateVectorOfTables(offsets);\n }\n"; @@ -1186,16 +1503,19 @@ class CSharpGenerator : public BaseGenerator { code += "int vectorLocation, "; code += GenTypeGet(key_field->value.type); code += " key, ByteBuffer bb) {\n"; - if (IsString(key_field->value.type)) { - code += " byte[] byteKey = "; - code += "System.Text.Encoding.UTF8.GetBytes(key);\n"; - } + code += + " " + struct_def.name + " obj_ = new " + struct_def.name + "();\n"; code += " int span = "; code += "bb.GetInt(vectorLocation - 4);\n"; code += " int start = 0;\n"; code += " while (span != 0) {\n"; code += " int middle = span / 2;\n"; - code += GenLookupKeyGetter(key_field); + code += + " int tableOffset = Table.__indirect(vectorLocation + 4 * " + "(start + middle), bb);\n"; + + code += " obj_.__assign(tableOffset, bb);\n"; + code += " int comp = obj_." + name + ".CompareTo(key);\n"; code += " if (comp > 0) {\n"; code += " span = middle;\n"; code += " } else if (comp < 0) {\n"; @@ -1203,9 +1523,7 @@ class CSharpGenerator : public BaseGenerator { code += " start += middle;\n"; code += " span -= middle;\n"; code += " } else {\n"; - code += " return "; - code += "new " + struct_def.name + "()"; - code += ".__assign(tableOffset, bb);\n"; + code += " return obj_;\n"; code += " }\n }\n"; code += " return null;\n"; code += " }\n"; @@ -1215,7 +1533,7 @@ class CSharpGenerator : public BaseGenerator { GenPackUnPack_ObjectAPI(struct_def, code_ptr, opts, struct_has_create, field_has_create_set); } - code += "};\n\n"; + code += "}\n\n"; if (opts.generate_object_based_api) { GenStruct_ObjectAPI(struct_def, code_ptr, opts); @@ -1280,6 +1598,67 @@ class CSharpGenerator : public BaseGenerator { code += " }\n"; } + std::string GenUnionVerify(const Type &union_type) const { + if (union_type.enum_def) { + const auto &enum_def = *union_type.enum_def; + + auto ret = + "\n\nstatic public class " + enum_def.name + "Verify\n"; + ret += "{\n"; + ret += + " static public bool Verify(Google.FlatBuffers.Verifier verifier, " + "byte typeId, uint tablePos)\n"; + ret += " {\n"; + ret += " bool result = true;\n"; + + const auto union_enum_loop = [&]() { + ret += " switch((" + enum_def.name + ")typeId)\n"; + ret += " {\n"; + + for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { + const auto &ev = **it; + if (ev.IsZero()) { continue; } + + ret += " case " + Name(enum_def) + "." + Name(ev) + ":\n"; + + if (IsString(ev.union_type)) { + ret += + " result = verifier.VerifyUnionString(tablePos);\n"; + ret += " break;"; + } else if (ev.union_type.base_type == BASE_TYPE_STRUCT) { + if (! ev.union_type.struct_def->fixed) { + auto type = GenTypeGet(ev.union_type); + ret += " result = " + type + "Verify.Verify(verifier, tablePos);\n"; + } else { + ret += " result = verifier.VerifyUnionData(tablePos, " + + NumToString(InlineSize(ev.union_type)) + ", " + + NumToString(InlineAlignment(ev.union_type)) + + ");\n";; + } + ret += " break;"; + } else { + FLATBUFFERS_ASSERT(false); + } + ret += "\n"; + } + + ret += " default: result = true;\n"; + ret += " break;\n"; + ret += " }\n"; + ret += " return result;\n"; + }; + + union_enum_loop(); + ret += " }\n"; + ret += "}\n"; + ret += "\n"; + + return ret; + } + FLATBUFFERS_ASSERT(0); + return ""; + } + void GenEnum_ObjectAPI(EnumDef &enum_def, std::string *code_ptr, const IDLOptions &opts) const { auto &code = *code_ptr; @@ -1291,45 +1670,59 @@ class CSharpGenerator : public BaseGenerator { code += "public "; } auto union_name = enum_def.name + "Union"; + auto class_member = std::string("Value"); + if (class_member == enum_def.name) { class_member += "_"; }; code += "class " + union_name + " {\n"; // Type code += " public " + enum_def.name + " Type { get; set; }\n"; // Value - code += " public object Value { get; set; }\n"; + code += " public object " + class_member + " { get; set; }\n"; code += "\n"; // Constructor code += " public " + union_name + "() {\n"; code += " this.Type = " + enum_def.name + "." + enum_def.Vals()[0]->name + ";\n"; - code += " this.Value = null;\n"; + code += " this." + class_member + " = null;\n"; code += " }\n\n"; // As<T> - code += " public T As<T>() where T : class { return this.Value as T; }\n"; - // As + code += " public T As<T>() where T : class { return this." + class_member + + " as T; }\n"; + // As, From for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { auto &ev = **it; if (ev.union_type.base_type == BASE_TYPE_NONE) continue; auto type_name = GenTypeGet_ObjectAPI(ev.union_type, opts); - if (ev.union_type.base_type == BASE_TYPE_STRUCT && - ev.union_type.struct_def->attributes.Lookup("private")) { - code += " internal "; - } else { - code += " public "; - } - code += type_name + " As" + ev.name + "() { return this.As<" + type_name + - ">(); }\n"; + std::string accessibility = + (ev.union_type.base_type == BASE_TYPE_STRUCT && + ev.union_type.struct_def->attributes.Lookup("private")) + ? "internal" + : "public"; + // As + code += " " + accessibility + " " + type_name + " As" + ev.name + + "() { return this.As<" + type_name + ">(); }\n"; + // From + auto lower_ev_name = ev.name; + std::transform(lower_ev_name.begin(), lower_ev_name.end(), + lower_ev_name.begin(), CharToLower); + code += " " + accessibility + " static " + union_name + " From" + + ev.name + "(" + type_name + " _" + lower_ev_name + + ") { return new " + union_name + "{ Type = " + Name(enum_def) + + "." + Name(ev) + ", " + class_member + " = _" + lower_ev_name + + " }; }\n"; } code += "\n"; // Pack() - code += " public static int Pack(FlatBuffers.FlatBufferBuilder builder, " + - union_name + " _o) {\n"; + code += + " public static int Pack(Google.FlatBuffers.FlatBufferBuilder " + "builder, " + + union_name + " _o) {\n"; code += " switch (_o.Type) {\n"; for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { auto &ev = **it; if (ev.union_type.base_type == BASE_TYPE_NONE) { code += " default: return 0;\n"; } else { - code += " case " + enum_def.name + "." + ev.name + ": return "; + code += " case " + Name(enum_def) + "." + Name(ev) + ": return "; if (IsString(ev.union_type)) { code += "builder.CreateString(_o.As" + ev.name + "()).Value;\n"; } else { @@ -1341,6 +1734,9 @@ class CSharpGenerator : public BaseGenerator { code += " }\n"; code += " }\n"; code += "}\n\n"; + + code += GenUnionVerify(enum_def.underlying_type); + // JsonConverter if (opts.cs_gen_json_serializer) { if (enum_def.attributes.Lookup("private")) { @@ -1377,7 +1773,7 @@ class CSharpGenerator : public BaseGenerator { " _o, " "Newtonsoft.Json.JsonSerializer serializer) {\n"; code += " if (_o == null) return;\n"; - code += " serializer.Serialize(writer, _o.Value);\n"; + code += " serializer.Serialize(writer, _o." + class_member + ");\n"; code += " }\n"; code += " public override object ReadJson(Newtonsoft.Json.JsonReader " @@ -1414,8 +1810,8 @@ class CSharpGenerator : public BaseGenerator { code += " default: break;\n"; } else { auto type_name = GenTypeGet_ObjectAPI(ev.union_type, opts); - code += " case " + enum_def.name + "." + ev.name + - ": _o.Value = serializer.Deserialize<" + type_name + + code += " case " + Name(enum_def) + "." + Name(ev) + ": _o." + + class_member + " = serializer.Deserialize<" + type_name + ">(reader); break;\n"; } } @@ -1433,9 +1829,12 @@ class CSharpGenerator : public BaseGenerator { void GenUnionUnPack_ObjectAPI(const EnumDef &enum_def, std::string *code_ptr, const std::string &camel_name, + const std::string &camel_name_short, bool is_vector) const { auto &code = *code_ptr; std::string varialbe_name = "_o." + camel_name; + std::string class_member = "Value"; + if (class_member == enum_def.name) class_member += "_"; std::string type_suffix = ""; std::string func_suffix = "()"; std::string indent = " "; @@ -1450,20 +1849,21 @@ class CSharpGenerator : public BaseGenerator { } else { code += indent + varialbe_name + " = new "; } - code += WrapInNameSpace(enum_def) + "Union();\n"; - code += indent + varialbe_name + ".Type = this." + camel_name + "Type" + - type_suffix + ";\n"; - code += - indent + "switch (this." + camel_name + "Type" + type_suffix + ") {\n"; + code += NamespacedName(enum_def) + "Union();\n"; + code += indent + varialbe_name + ".Type = this." + camel_name_short + + "Type" + type_suffix + ";\n"; + code += indent + "switch (this." + camel_name_short + "Type" + type_suffix + + ") {\n"; for (auto eit = enum_def.Vals().begin(); eit != enum_def.Vals().end(); ++eit) { auto &ev = **eit; if (ev.union_type.base_type == BASE_TYPE_NONE) { code += indent + " default: break;\n"; } else { - code += indent + " case " + WrapInNameSpace(enum_def) + "." + ev.name + + code += indent + " case " + NamespacedName(enum_def) + "." + ev.name + ":\n"; - code += indent + " " + varialbe_name + ".Value = this." + camel_name; + code += indent + " " + varialbe_name + "." + class_member + + " = this." + camel_name; if (IsString(ev.union_type)) { code += "AsString" + func_suffix + ";\n"; } else { @@ -1499,7 +1899,9 @@ class CSharpGenerator : public BaseGenerator { it != struct_def.fields.vec.end(); ++it) { auto &field = **it; if (field.deprecated) continue; - auto camel_name = MakeCamel(field.name); + auto camel_name = Name(field); + if (camel_name == struct_def.name) { camel_name += "_"; } + auto camel_name_short = Name(field); auto start = " _o." + camel_name + " = "; switch (field.value.type.base_type) { case BASE_TYPE_STRUCT: { @@ -1515,11 +1917,10 @@ class CSharpGenerator : public BaseGenerator { case BASE_TYPE_ARRAY: { auto type_name = GenTypeGet_ObjectAPI(field.value.type, opts); auto length_str = NumToString(field.value.type.fixed_length); - auto unpack_method = field.value.type.struct_def == nullptr - ? "" - : field.value.type.struct_def->fixed - ? ".UnPack()" - : "?.UnPack()"; + auto unpack_method = field.value.type.struct_def == nullptr ? "" + : field.value.type.struct_def->fixed + ? ".UnPack()" + : "?.UnPack()"; code += start + "new " + type_name.substr(0, type_name.length() - 1) + length_str + "];\n"; code += " for (var _j = 0; _j < " + length_str + "; ++_j) { _o." + @@ -1534,7 +1935,7 @@ class CSharpGenerator : public BaseGenerator { code += " for (var _j = 0; _j < this." + camel_name + "Length; ++_j) {\n"; GenUnionUnPack_ObjectAPI(*field.value.type.enum_def, code_ptr, - camel_name, true); + camel_name, camel_name_short, true); code += " }\n"; } else if (field.value.type.element != BASE_TYPE_UTYPE) { auto fixed = field.value.type.struct_def == nullptr; @@ -1555,7 +1956,7 @@ class CSharpGenerator : public BaseGenerator { case BASE_TYPE_UTYPE: break; case BASE_TYPE_UNION: { GenUnionUnPack_ObjectAPI(*field.value.type.enum_def, code_ptr, - camel_name, false); + camel_name, camel_name_short, false); break; } default: { @@ -1574,7 +1975,9 @@ class CSharpGenerator : public BaseGenerator { it != struct_def.fields.vec.end(); ++it) { auto &field = **it; if (field.deprecated) continue; - auto camel_name = MakeCamel(field.name); + auto camel_name = Name(field); + if (camel_name == struct_def.name) { camel_name += "_"; } + auto camel_name_short = Name(field); // pre switch (field.value.type.base_type) { case BASE_TYPE_STRUCT: { @@ -1628,12 +2031,12 @@ class CSharpGenerator : public BaseGenerator { break; case BASE_TYPE_UTYPE: property_name = camel_name.substr(0, camel_name.size() - 4); - array_type = WrapInNameSpace(*field.value.type.enum_def); + array_type = NamespacedName(*field.value.type.enum_def); to_array = "_o." + property_name + "[_j].Type"; break; case BASE_TYPE_UNION: array_type = "int"; - to_array = WrapInNameSpace(*field.value.type.enum_def) + + to_array = NamespacedName(*field.value.type.enum_def) + "Union.Pack(builder, _o." + property_name + "[_j])"; break; default: gen_for_loop = false; break; @@ -1650,7 +2053,7 @@ class CSharpGenerator : public BaseGenerator { code += " var " + array_name + " = _o." + property_name + ".ToArray();\n"; } - code += " _" + field.name + " = Create" + camel_name + + code += " _" + field.name + " = Create" + camel_name_short + "Vector(builder, " + array_name + ");\n"; code += " }\n"; } else { @@ -1662,7 +2065,7 @@ class CSharpGenerator : public BaseGenerator { camel_name + "[_j]);"; code += " var _" + field.name + " = default(VectorOffset);\n"; code += " if (_o." + camel_name + " != null) {\n"; - code += " Start" + camel_name + "Vector(builder, _o." + + code += " Start" + camel_name_short + "Vector(builder, _o." + camel_name + ".Count);\n"; code += " for (var _j = _o." + camel_name + ".Count - 1; _j >= 0; --_j) { " + pack_method + " }\n"; @@ -1688,7 +2091,7 @@ class CSharpGenerator : public BaseGenerator { } case BASE_TYPE_UNION: { code += " var _" + field.name + "_type = _o." + camel_name + - " == null ? " + WrapInNameSpace(*field.value.type.enum_def) + + " == null ? " + NamespacedName(*field.value.type.enum_def) + ".NONE : " + "_o." + camel_name + ".Type;\n"; code += " var _" + field.name + " = _o." + camel_name + @@ -1707,7 +2110,8 @@ class CSharpGenerator : public BaseGenerator { it != struct_def.fields.vec.end(); ++it) { auto &field = **it; if (field.deprecated) continue; - auto camel_name = MakeCamel(field.name); + auto camel_name = Name(field); + if (camel_name == struct_def.name) { camel_name += "_"; } switch (field.value.type.base_type) { case BASE_TYPE_STRUCT: { if (struct_def.fixed) { @@ -1761,7 +2165,7 @@ class CSharpGenerator : public BaseGenerator { it != struct_def.fields.vec.end(); ++it) { auto &field = **it; if (field.deprecated) continue; - auto camel_name = MakeCamel(field.name); + auto camel_name = Name(field); switch (field.value.type.base_type) { case BASE_TYPE_STRUCT: { if (field.value.type.struct_def->fixed) { @@ -1857,7 +2261,7 @@ class CSharpGenerator : public BaseGenerator { } code += "] = _o"; for (size_t i = 0, j = 0; i < array_lengths.size(); ++i) { - code += "." + MakeCamel(array_lengths[i].name); + code += "." + ConvertCase(array_lengths[i].name, Case::kUpperCamel); if (array_lengths[i].length <= 0) continue; code += "[idx" + NumToString(j++) + "]"; } @@ -1868,7 +2272,7 @@ class CSharpGenerator : public BaseGenerator { } else { code += "_o"; for (size_t i = 0; i < array_lengths.size(); ++i) { - code += "." + MakeCamel(array_lengths[i].name); + code += "." + ConvertCase(array_lengths[i].name, Case::kUpperCamel); } code += ";"; } @@ -1911,13 +2315,13 @@ class CSharpGenerator : public BaseGenerator { type_name.replace(type_name.length() - type_name_length, type_name_length, new_type_name); } else if (type.element == BASE_TYPE_UNION) { - type_name = WrapInNameSpace(*type.enum_def) + "Union"; + type_name = NamespacedName(*type.enum_def) + "Union"; } break; } case BASE_TYPE_UNION: { - type_name = WrapInNameSpace(*type.enum_def) + "Union"; + type_name = NamespacedName(*type.enum_def) + "Union"; break; } default: break; @@ -1961,10 +2365,11 @@ class CSharpGenerator : public BaseGenerator { if (field.value.type.element == BASE_TYPE_UTYPE) continue; auto type_name = GenTypeGet_ObjectAPI(field.value.type, opts); if (field.IsScalarOptional()) type_name += "?"; - auto camel_name = MakeCamel(field.name, true); + auto camel_name = Name(field); + if (camel_name == struct_def.name) { camel_name += "_"; } if (opts.cs_gen_json_serializer) { if (IsUnion(field.value.type)) { - auto utype_name = WrapInNameSpace(*field.value.type.enum_def); + auto utype_name = NamespacedName(*field.value.type.enum_def); code += " [Newtonsoft.Json.JsonProperty(\"" + field.name + "_type\")]\n"; if (IsVector(field.value.type)) { @@ -2027,7 +2432,9 @@ class CSharpGenerator : public BaseGenerator { if (field.deprecated) continue; if (field.value.type.base_type == BASE_TYPE_UTYPE) continue; if (field.value.type.element == BASE_TYPE_UTYPE) continue; - code += " this." + MakeCamel(field.name) + " = "; + auto camel_name = Name(field); + if (camel_name == struct_def.name) { camel_name += "_"; } + code += " this." + camel_name + " = "; auto type_name = GenTypeGet_ObjectAPI(field.value.type, opts); if (IsScalar(field.value.type.base_type)) { code += GenDefaultValue(field) + ";\n"; @@ -2097,4 +2504,58 @@ bool GenerateCSharp(const Parser &parser, const std::string &path, return generator.generate(); } +namespace { + +class CSharpCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateCSharp(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + output = CSharpMakeRule(parser, path, filename); + return Status::OK; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kCSharp; } + + std::string LanguageName() const override { return "CSharp"; } +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewCSharpCodeGenerator() { + return std::unique_ptr<CSharpCodeGenerator>(new CSharpCodeGenerator()); +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_csharp.h b/contrib/libs/flatbuffers/src/idl_gen_csharp.h new file mode 100644 index 00000000000..f5895a9369c --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_csharp.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_CSHARP_H_ +#define FLATBUFFERS_IDL_GEN_CSHARP_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new CSharp code generator. +std::unique_ptr<CodeGenerator> NewCSharpCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_CSHARP_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_dart.cpp b/contrib/libs/flatbuffers/src/idl_gen_dart.cpp index 56c4a825558..299409bac2d 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_dart.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_dart.cpp @@ -15,32 +15,66 @@ */ // independent from idl_parser, since this code is not needed for most clients +#include "idl_gen_dart.h" + #include <cassert> +#include <cmath> #include "flatbuffers/code_generators.h" #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" +#include "idl_namer.h" namespace flatbuffers { namespace dart { +namespace { + +static Namer::Config DartDefaultConfig() { + return { /*types=*/Case::kUpperCamel, + /*constants=*/Case::kScreamingSnake, + /*methods=*/Case::kLowerCamel, + /*functions=*/Case::kUnknown, // unused. + /*fields=*/Case::kLowerCamel, + /*variables=*/Case::kLowerCamel, + /*variants=*/Case::kKeep, + /*enum_variant_seperator=*/".", + /*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase, + /*namespaces=*/Case::kSnake2, + /*namespace_seperator=*/".", + /*object_prefix=*/"", + /*object_suffix=*/"T", + /*keyword_prefix=*/"$", + /*keyword_suffix=*/"", + /*filenames=*/Case::kKeep, + /*directories=*/Case::kKeep, + /*output_path=*/"", + /*filename_suffix=*/"_generated", + /*filename_extension=*/".dart" }; +} + +static std::set<std::string> DartKeywords() { + // see https://www.dartlang.org/guides/language/language-tour#keywords + // yield*, async*, and sync* shouldn't be proble + return { + "abstract", "else", "import", "show", "as", "enum", + "in", "static", "assert", "export", "interface", "super", + "async", "extends", "is", "switch", "await", "extension", + "late", "sync", "break", "external", "library", "this", + "case", "factory", "mixin", "throw", "catch", "false", + "new", "true", "class", "final", "null", "try", + "const", "finally", "on", "typedef", "continue", "for", + "operator", "var", "covariant", "Function", "part", "void", + "default", "get", "required", "while", "deferred", "hide", + "rethrow", "with", "do", "if", "return", "yield", + "dynamic", "implements", "set", + }; +} +} // namespace + const std::string _kFb = "fb"; -// see https://www.dartlang.org/guides/language/language-tour#keywords -// yeild*, async*, and sync* shouldn't be problems anyway but keeping them in -static const char *keywords[] = { - "abstract", "deferred", "if", "super", "as", "do", - "implements", "switch", "assert", "dynamic", "import", "sync*", - "async", "else", "in", "this", "async*", "enum", - "is", "throw", "await", "export", "library", "true", - "break", "external", "new", "try", "case", "extends", - "null", "typedef", "catch", "factory", "operator", "var", - "class", "false", "part", "void", "const", "final", - "rethrow", "while", "continue", "finally", "return", "with", - "covariant", "for", "set", "yield", "default", "get", - "static", "yield*" -}; // Iterate through all definitions we haven't generate code for (enums, structs, // and tables) and output them to a single file. @@ -50,20 +84,57 @@ class DartGenerator : public BaseGenerator { DartGenerator(const Parser &parser, const std::string &path, const std::string &file_name) - : BaseGenerator(parser, path, file_name, "", ".", "dart") {} + : BaseGenerator(parser, path, file_name, "", ".", "dart"), + namer_(WithFlagOptions(DartDefaultConfig(), parser.opts, path), + DartKeywords()) {} + + template<typename T> + void import_generator(const std::vector<T *> &definitions, + const std::string &included, + std::set<std::string> &imports) { + for (const auto &item : definitions) { + if (item->file == included) { + std::string component = namer_.Namespace(*item->defined_namespace); + std::string filebase = + flatbuffers::StripPath(flatbuffers::StripExtension(item->file)); + std::string filename = + namer_.File(filebase + (component.empty() ? "" : "_" + component)); + + imports.emplace("import './" + filename + "'" + + (component.empty() + ? ";\n" + : " as " + ImportAliasName(component) + ";\n")); + } + } + } + // Iterate through all definitions we haven't generate code for (enums, // structs, and tables) and output them to a single file. bool generate() { std::string code; namespace_code_map namespace_code; - GenerateEnums(&namespace_code); - GenerateStructs(&namespace_code); + GenerateEnums(namespace_code); + GenerateStructs(namespace_code); + + std::set<std::string> imports; + + for (const auto &included_file : parser_.GetIncludedFiles()) { + if (included_file.filename == parser_.file_being_parsed_) continue; + + import_generator(parser_.structs_.vec, included_file.filename, imports); + import_generator(parser_.enums_.vec, included_file.filename, imports); + } + + std::string import_code = ""; + for (const auto &file : imports) { import_code += file; } + + import_code += import_code.empty() ? "" : "\n"; for (auto kv = namespace_code.begin(); kv != namespace_code.end(); ++kv) { code.clear(); code = code + "// " + FlatBuffersGeneratedWarning() + "\n"; code = code + - "// ignore_for_file: unused_import, unused_field, " + "// ignore_for_file: unused_import, unused_field, unused_element, " "unused_local_variable\n\n"; if (!kv->first.empty()) { code += "library " + kv->first + ";\n\n"; } @@ -75,31 +146,26 @@ class DartGenerator : public BaseGenerator { for (auto kv2 = namespace_code.begin(); kv2 != namespace_code.end(); ++kv2) { if (kv2->first != kv->first) { - code += - "import '" + - GeneratedFileName( - "./", - file_name_ + (!kv2->first.empty() ? "_" + kv2->first : ""), - parser_.opts) + - "' as " + ImportAliasName(kv2->first) + ";\n"; + code += "import './" + Filename(kv2->first, /*path=*/false) + + "' as " + ImportAliasName(kv2->first) + ";\n"; } } + code += "\n"; + code += import_code; + code += kv->second; - if (!SaveFile( - GeneratedFileName( - path_, - file_name_ + (!kv->first.empty() ? "_" + kv->first : ""), - parser_.opts) - .c_str(), - code, false)) { - return false; - } + if (!SaveFile(Filename(kv->first).c_str(), code, false)) { return false; } } return true; } + std::string Filename(const std::string &suffix, bool path = true) const { + return (path ? path_ : "") + + namer_.File(file_name_ + (suffix.empty() ? "" : "_" + suffix)); + } + private: static std::string ImportAliasName(const std::string &ns) { std::string ret; @@ -113,62 +179,15 @@ class DartGenerator : public BaseGenerator { return ret; } - static std::string BuildNamespaceName(const Namespace &ns) { - if (ns.components.empty()) { return ""; } - std::stringstream sstream; - std::copy(ns.components.begin(), ns.components.end() - 1, - std::ostream_iterator<std::string>(sstream, ".")); - - auto ret = sstream.str() + ns.components.back(); - for (size_t i = 0; i < ret.size(); i++) { - auto lower = CharToLower(ret[i]); - if (lower != ret[i]) { - ret[i] = lower; - if (i != 0 && ret[i - 1] != '.') { - ret.insert(i, "_"); - i++; - } - } - } - // std::transform(ret.begin(), ret.end(), ret.begin(), CharToLower); - return ret; - } - - void GenIncludeDependencies(std::string *code, - const std::string &the_namespace) { - for (auto it = parser_.included_files_.begin(); - it != parser_.included_files_.end(); ++it) { - if (it->second.empty()) continue; - - auto noext = flatbuffers::StripExtension(it->second); - auto basename = flatbuffers::StripPath(noext); - - *code += - "import '" + - GeneratedFileName( - "", basename + (the_namespace == "" ? "" : "_" + the_namespace), - parser_.opts) + - "';\n"; - } - } - - static std::string EscapeKeyword(const std::string &name) { - for (size_t i = 0; i < sizeof(keywords) / sizeof(keywords[0]); i++) { - if (name == keywords[i]) { return MakeCamel(name + "_", false); } - } - - return MakeCamel(name, false); - } - - void GenerateEnums(namespace_code_map *namespace_code) { + void GenerateEnums(namespace_code_map &namespace_code) { for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end(); ++it) { auto &enum_def = **it; - GenEnum(enum_def, namespace_code); // enum_code_ptr); + GenEnum(enum_def, namespace_code); } } - void GenerateStructs(namespace_code_map *namespace_code) { + void GenerateStructs(namespace_code_map &namespace_code) { for (auto it = parser_.structs_.vec.begin(); it != parser_.structs_.vec.end(); ++it) { auto &struct_def = **it; @@ -178,70 +197,51 @@ class DartGenerator : public BaseGenerator { // Generate a documentation comment, if available. static void GenDocComment(const std::vector<std::string> &dc, - std::string *code_ptr, - const std::string &extra_lines, - const char *indent = nullptr) { - if (dc.empty() && extra_lines.empty()) { - // Don't output empty comment blocks with 0 lines of comment content. - return; - } - - auto &code = *code_ptr; - + const char *indent, std::string &code) { for (auto it = dc.begin(); it != dc.end(); ++it) { if (indent) code += indent; code += "/// " + *it + "\n"; } - if (!extra_lines.empty()) { - if (!dc.empty()) { - if (indent) code += indent; - code += "///\n"; - } - if (indent) code += indent; - std::string::size_type start = 0; - for (;;) { - auto end = extra_lines.find('\n', start); - if (end != std::string::npos) { - code += "/// " + extra_lines.substr(start, end - start) + "\n"; - start = end + 1; - } else { - code += "/// " + extra_lines.substr(start) + "\n"; - break; - } - } - } - } - - static void GenDocComment(std::string *code_ptr, - const std::string &extra_lines) { - GenDocComment(std::vector<std::string>(), code_ptr, extra_lines); } // Generate an enum declaration and an enum string lookup table. - void GenEnum(EnumDef &enum_def, namespace_code_map *namespace_code) { + void GenEnum(EnumDef &enum_def, namespace_code_map &namespace_code) { if (enum_def.generated) return; - auto ns = BuildNamespaceName(*enum_def.defined_namespace); - std::string code; - GenDocComment(enum_def.doc_comment, &code, ""); - - auto name = enum_def.is_union ? enum_def.name + "TypeId" : enum_def.name; - auto is_bit_flags = enum_def.attributes.Lookup("bit_flags"); - - code += "class " + name + " {\n"; + std::string &code = + namespace_code[namer_.Namespace(*enum_def.defined_namespace)]; + GenDocComment(enum_def.doc_comment, "", code); + + const std::string enum_type = + namer_.Type(enum_def) + (enum_def.is_union ? "TypeId" : ""); + const bool is_bit_flags = + enum_def.attributes.Lookup("bit_flags") != nullptr; + // The flatbuffer schema language allows bit flag enums to potentially have + // a default value of zero, even if it's not a valid enum value... + const bool permit_zero = is_bit_flags; + + code += "class " + enum_type + " {\n"; code += " final int value;\n"; - code += " const " + name + "._(this.value);\n\n"; - code += " factory " + name + ".fromValue(int value) {\n"; - code += " if (value == null) value = 0;\n"; - - code += " if (!values.containsKey(value)) {\n"; - code += - " throw new StateError('Invalid value $value for bit flag enum "; - code += name + "');\n"; + code += " const " + enum_type + "._(this.value);\n\n"; + code += " factory " + enum_type + ".fromValue(int value) {\n"; + code += " final result = values[value];\n"; + code += " if (result == null) {\n"; + if (permit_zero) { + code += " if (value == 0) {\n"; + code += " return " + enum_type + "._(0);\n"; + code += " } else {\n"; + } + code += " throw StateError('Invalid value $value for bit flag enum "; + code += enum_type + "');\n"; + if (permit_zero) { code += " }\n"; } code += " }\n"; - code += " return values[value];\n"; + code += " return result;\n"; code += " }\n\n"; + code += " static " + enum_type + "? _createOrNull(int? value) => \n"; + code += + " value == null ? null : " + enum_type + ".fromValue(value);\n\n"; + // this is meaningless for bit_flags // however, note that unlike "regular" dart enums this enum can still have // holes. @@ -258,52 +258,52 @@ class DartGenerator : public BaseGenerator { for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { auto &ev = **it; + const auto enum_var = namer_.Variant(ev); if (!ev.doc_comment.empty()) { if (it != enum_def.Vals().begin()) { code += '\n'; } - GenDocComment(ev.doc_comment, &code, "", " "); + GenDocComment(ev.doc_comment, " ", code); } - code += " static const " + name + " " + ev.name + " = "; - code += "const " + name + "._(" + enum_def.ToString(ev) + ");\n"; + code += " static const " + enum_type + " " + enum_var + " = " + + enum_type + "._(" + enum_def.ToString(ev) + ");\n"; } - code += " static const Map<int," + name + "> values = {"; + code += " static const Map<int, " + enum_type + "> values = {\n"; for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { auto &ev = **it; - code += enum_def.ToString(ev) + ": " + ev.name + ","; + const auto enum_var = namer_.Variant(ev); + if (it != enum_def.Vals().begin()) code += ",\n"; + code += " " + enum_def.ToString(ev) + ": " + enum_var; } code += "};\n\n"; - code += " static const " + _kFb + ".Reader<" + name + - "> reader = const _" + name + "Reader();\n\n"; + code += " static const " + _kFb + ".Reader<" + enum_type + "> reader = _" + + enum_type + "Reader();\n\n"; code += " @override\n"; code += " String toString() {\n"; - code += " return '" + name + "{value: $value}';\n"; + code += " return '" + enum_type + "{value: $value}';\n"; code += " }\n"; code += "}\n\n"; - GenEnumReader(enum_def, name, &code); - (*namespace_code)[ns] += code; + GenEnumReader(enum_def, enum_type, code); } - void GenEnumReader(EnumDef &enum_def, const std::string &name, - std::string *code_ptr) { - auto &code = *code_ptr; - - code += "class _" + name + "Reader extends " + _kFb + ".Reader<" + name + - "> {\n"; - code += " const _" + name + "Reader();\n\n"; + void GenEnumReader(EnumDef &enum_def, const std::string &enum_type, + std::string &code) { + code += "class _" + enum_type + "Reader extends " + _kFb + ".Reader<" + + enum_type + "> {\n"; + code += " const _" + enum_type + "Reader();\n\n"; code += " @override\n"; - code += " int get size => 1;\n\n"; + code += " int get size => " + EnumSize(enum_def.underlying_type) + ";\n\n"; code += " @override\n"; - code += - " " + name + " read(" + _kFb + ".BufferContext bc, int offset) =>\n"; - code += " new " + name + ".fromValue(const " + _kFb + "." + + code += " " + enum_type + " read(" + _kFb + + ".BufferContext bc, int offset) =>\n"; + code += " " + enum_type + ".fromValue(const " + _kFb + "." + GenType(enum_def.underlying_type) + "Reader().read(bc, offset));\n"; code += "}\n\n"; } - static std::string GenType(const Type &type) { + std::string GenType(const Type &type) { switch (type.base_type) { case BASE_TYPE_BOOL: return "Bool"; case BASE_TYPE_CHAR: return "Int8"; @@ -319,45 +319,74 @@ class DartGenerator : public BaseGenerator { case BASE_TYPE_DOUBLE: return "Float64"; case BASE_TYPE_STRING: return "String"; case BASE_TYPE_VECTOR: return GenType(type.VectorType()); - case BASE_TYPE_STRUCT: return type.struct_def->name; - case BASE_TYPE_UNION: return type.enum_def->name + "TypeId"; + case BASE_TYPE_STRUCT: return namer_.Type(*type.struct_def); + case BASE_TYPE_UNION: return namer_.Type(*type.enum_def) + "TypeId"; default: return "Table"; } } + static std::string EnumSize(const Type &type) { + switch (type.base_type) { + case BASE_TYPE_BOOL: + case BASE_TYPE_CHAR: + case BASE_TYPE_UTYPE: + case BASE_TYPE_UCHAR: return "1"; + case BASE_TYPE_SHORT: + case BASE_TYPE_USHORT: return "2"; + case BASE_TYPE_INT: + case BASE_TYPE_UINT: + case BASE_TYPE_FLOAT: return "4"; + case BASE_TYPE_LONG: + case BASE_TYPE_ULONG: + case BASE_TYPE_DOUBLE: return "8"; + default: return "1"; + } + } + std::string GenReaderTypeName(const Type &type, Namespace *current_namespace, const FieldDef &def, - bool parent_is_vector = false) { + bool parent_is_vector = false, bool lazy = true, + bool constConstruct = true) { + std::string prefix = (constConstruct ? "const " : "") + _kFb; if (type.base_type == BASE_TYPE_BOOL) { - return "const " + _kFb + ".BoolReader()"; + return prefix + ".BoolReader()"; } else if (IsVector(type)) { - return "const " + _kFb + ".ListReader<" + + if (!type.VectorType().enum_def) { + if (type.VectorType().base_type == BASE_TYPE_CHAR) { + return prefix + ".Int8ListReader(" + (lazy ? ")" : "lazy: false)"); + } + if (type.VectorType().base_type == BASE_TYPE_UCHAR) { + return prefix + ".Uint8ListReader(" + (lazy ? ")" : "lazy: false)"); + } + } + return prefix + ".ListReader<" + GenDartTypeName(type.VectorType(), current_namespace, def) + ">(" + - GenReaderTypeName(type.VectorType(), current_namespace, def, - true) + - ")"; + GenReaderTypeName(type.VectorType(), current_namespace, def, true, + true, false) + + (lazy ? ")" : ", lazy: false)"); } else if (IsString(type)) { - return "const " + _kFb + ".StringReader()"; + return prefix + ".StringReader()"; } if (IsScalar(type.base_type)) { if (type.enum_def && parent_is_vector) { return GenDartTypeName(type, current_namespace, def) + ".reader"; } - return "const " + _kFb + "." + GenType(type) + "Reader()"; + return prefix + "." + GenType(type) + "Reader()"; } else { return GenDartTypeName(type, current_namespace, def) + ".reader"; } } std::string GenDartTypeName(const Type &type, Namespace *current_namespace, - const FieldDef &def, bool addBuilder = false) { + const FieldDef &def, + std::string struct_type_suffix = "") { if (type.enum_def) { if (type.enum_def->is_union && type.base_type != BASE_TYPE_UNION) { - return type.enum_def->name + "TypeId"; + return namer_.Type(*type.enum_def) + "TypeId"; } else if (type.enum_def->is_union) { return "dynamic"; } else if (type.base_type != BASE_TYPE_VECTOR) { - return type.enum_def->name; + return namer_.Type(*type.enum_def); } } @@ -376,32 +405,39 @@ class DartGenerator : public BaseGenerator { case BASE_TYPE_STRING: return "String"; case BASE_TYPE_STRUCT: return MaybeWrapNamespace( - type.struct_def->name + (addBuilder ? "ObjectBuilder" : ""), + namer_.Type(*type.struct_def) + struct_type_suffix, current_namespace, def); case BASE_TYPE_VECTOR: return "List<" + GenDartTypeName(type.VectorType(), current_namespace, def, - addBuilder) + + struct_type_suffix) + ">"; default: assert(0); return "dynamic"; } } - static const std::string MaybeWrapNamespace(const std::string &type_name, - Namespace *current_ns, - const FieldDef &field) { - auto curr_ns_str = BuildNamespaceName(*current_ns); - std::string field_ns_str = ""; - if (field.value.type.struct_def) { - field_ns_str += - BuildNamespaceName(*field.value.type.struct_def->defined_namespace); - } else if (field.value.type.enum_def) { - field_ns_str += - BuildNamespaceName(*field.value.type.enum_def->defined_namespace); - } + std::string GenDartTypeName(const Type &type, Namespace *current_namespace, + const FieldDef &def, bool nullable, + std::string struct_type_suffix) { + std::string typeName = + GenDartTypeName(type, current_namespace, def, struct_type_suffix); + if (nullable && typeName != "dynamic") typeName += "?"; + return typeName; + } - if (field_ns_str != "" && field_ns_str != curr_ns_str) { - return ImportAliasName(field_ns_str) + "." + type_name; + std::string MaybeWrapNamespace(const std::string &type_name, + Namespace *current_ns, + const FieldDef &field) const { + const std::string current_namespace = namer_.Namespace(*current_ns); + const std::string field_namespace = + field.value.type.struct_def + ? namer_.Namespace(*field.value.type.struct_def->defined_namespace) + : field.value.type.enum_def + ? namer_.Namespace(*field.value.type.enum_def->defined_namespace) + : ""; + + if (field_namespace != "" && field_namespace != current_namespace) { + return ImportAliasName(field_namespace) + "." + type_name; } else { return type_name; } @@ -409,38 +445,38 @@ class DartGenerator : public BaseGenerator { // Generate an accessor struct with constructor for a flatbuffers struct. void GenStruct(const StructDef &struct_def, - namespace_code_map *namespace_code) { + namespace_code_map &namespace_code) { if (struct_def.generated) return; - auto object_namespace = BuildNamespaceName(*struct_def.defined_namespace); - std::string code; + std::string &code = + namespace_code[namer_.Namespace(*struct_def.defined_namespace)]; - const auto &object_name = struct_def.name; + const auto &struct_type = namer_.Type(struct_def); // Emit constructor - GenDocComment(struct_def.doc_comment, &code, ""); + GenDocComment(struct_def.doc_comment, "", code); - auto reader_name = "_" + object_name + "Reader"; - auto builder_name = object_name + "Builder"; - auto object_builder_name = object_name + "ObjectBuilder"; + auto reader_name = "_" + struct_type + "Reader"; + auto builder_name = struct_type + "Builder"; + auto object_builder_name = struct_type + "ObjectBuilder"; std::string reader_code, builder_code; - code += "class " + object_name + " {\n"; + code += "class " + struct_type + " {\n"; - code += " " + object_name + "._(this._bc, this._bcOffset);\n"; + code += " " + struct_type + "._(this._bc, this._bcOffset);\n"; if (!struct_def.fixed) { - code += " factory " + object_name + "(List<int> bytes) {\n"; - code += " " + _kFb + ".BufferContext rootRef = new " + _kFb + - ".BufferContext.fromBytes(bytes);\n"; + code += " factory " + struct_type + "(List<int> bytes) {\n"; + code += + " final rootRef = " + _kFb + ".BufferContext.fromBytes(bytes);\n"; code += " return reader.read(rootRef, 0);\n"; code += " }\n"; } code += "\n"; - code += " static const " + _kFb + ".Reader<" + object_name + - "> reader = const " + reader_name + "();\n\n"; + code += " static const " + _kFb + ".Reader<" + struct_type + + "> reader = " + reader_name + "();\n\n"; code += " final " + _kFb + ".BufferContext _bc;\n"; code += " final int _bcOffset;\n\n"; @@ -448,25 +484,154 @@ class DartGenerator : public BaseGenerator { std::vector<std::pair<int, FieldDef *>> non_deprecated_fields; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { - auto &field = **it; + FieldDef &field = **it; if (field.deprecated) continue; auto offset = static_cast<int>(it - struct_def.fields.vec.begin()); non_deprecated_fields.push_back(std::make_pair(offset, &field)); } - GenImplementationGetters(struct_def, non_deprecated_fields, &code); + GenImplementationGetters(struct_def, non_deprecated_fields, code); + + if (parser_.opts.generate_object_based_api) { + code += + "\n" + GenStructObjectAPIUnpack(struct_def, non_deprecated_fields); + + code += "\n static int pack(fb.Builder fbBuilder, " + + namer_.ObjectType(struct_def) + "? object) {\n"; + code += " if (object == null) return 0;\n"; + code += " return object.pack(fbBuilder);\n"; + code += " }\n"; + } code += "}\n\n"; - GenReader(struct_def, &reader_name, &reader_code); - GenBuilder(struct_def, non_deprecated_fields, &builder_name, &builder_code); - GenObjectBuilder(struct_def, non_deprecated_fields, &object_builder_name, - &builder_code); + if (parser_.opts.generate_object_based_api) { + code += GenStructObjectAPI(struct_def, non_deprecated_fields); + } + + GenReader(struct_def, reader_name, reader_code); + GenBuilder(struct_def, non_deprecated_fields, builder_name, builder_code); + GenObjectBuilder(struct_def, non_deprecated_fields, object_builder_name, + builder_code); code += reader_code; code += builder_code; + } + + // Generate an accessor struct with constructor for a flatbuffers struct. + std::string GenStructObjectAPI( + const StructDef &struct_def, + const std::vector<std::pair<int, FieldDef *>> &non_deprecated_fields) { + std::string code; + GenDocComment(struct_def.doc_comment, "", code); + + std::string object_type = namer_.ObjectType(struct_def); + code += "class " + object_type + " implements " + _kFb + ".Packable {\n"; + + std::string constructor_args; + for (auto it = non_deprecated_fields.begin(); + it != non_deprecated_fields.end(); ++it) { + const FieldDef &field = *it->second; + + const std::string field_name = namer_.Field(field); + const std::string defaultValue = getDefaultValue(field.value); + const std::string type_name = + GenDartTypeName(field.value.type, struct_def.defined_namespace, field, + defaultValue.empty() && !struct_def.fixed, "T"); + + GenDocComment(field.doc_comment, " ", code); + code += " " + type_name + " " + field_name + ";\n"; + + if (!constructor_args.empty()) constructor_args += ",\n"; + constructor_args += " "; + constructor_args += (struct_def.fixed ? "required " : ""); + constructor_args += "this." + field_name; + if (!struct_def.fixed && !defaultValue.empty()) { + if (IsEnum(field.value.type)) { + auto &enum_def = *field.value.type.enum_def; + if (auto val = enum_def.FindByValue(defaultValue)) { + constructor_args += " = " + namer_.EnumVariant(enum_def, *val); + } else { + constructor_args += " = const " + namer_.Type(enum_def) + "._(" + + defaultValue + ")"; + } + } else { + constructor_args += " = " + defaultValue; + } + } + } + + if (!constructor_args.empty()) { + code += "\n " + object_type + "({\n" + constructor_args + "});\n\n"; + } + + code += GenStructObjectAPIPack(struct_def, non_deprecated_fields); + code += "\n"; + code += GenToString(object_type, non_deprecated_fields); + + code += "}\n\n"; + return code; + } + + // Generate function `StructNameT unpack()` + std::string GenStructObjectAPIUnpack( + const StructDef &struct_def, + const std::vector<std::pair<int, FieldDef *>> &non_deprecated_fields) { + std::string constructor_args; + for (auto it = non_deprecated_fields.begin(); + it != non_deprecated_fields.end(); ++it) { + const FieldDef &field = *it->second; + + const std::string field_name = namer_.Field(field); + if (!constructor_args.empty()) constructor_args += ",\n"; + constructor_args += " " + field_name + ": "; + + const Type &type = field.value.type; + std::string defaultValue = getDefaultValue(field.value); + bool isNullable = defaultValue.empty() && !struct_def.fixed; + std::string nullableValueAccessOperator = isNullable ? "?" : ""; + if (type.base_type == BASE_TYPE_STRUCT) { + constructor_args += + field_name + nullableValueAccessOperator + ".unpack()"; + } else if (type.base_type == BASE_TYPE_VECTOR) { + if (type.VectorType().base_type == BASE_TYPE_STRUCT) { + constructor_args += field_name + nullableValueAccessOperator + + ".map((e) => e.unpack()).toList()"; + } else { + constructor_args += + GenReaderTypeName(field.value.type, struct_def.defined_namespace, + field, false, false); + constructor_args += ".vTableGet"; + std::string offset = NumToString(field.value.offset); + constructor_args += + isNullable + ? "Nullable(_bc, _bcOffset, " + offset + ")" + : "(_bc, _bcOffset, " + offset + ", " + defaultValue + ")"; + } + } else { + constructor_args += field_name; + } + } - (*namespace_code)[object_namespace] += code; + const std::string object_type = namer_.ObjectType(struct_def); + std::string code = " " + object_type + " unpack() => " + object_type + "("; + if (!constructor_args.empty()) code += "\n" + constructor_args; + code += ");\n"; + return code; + } + + // Generate function `StructNameT pack()` + std::string GenStructObjectAPIPack( + const StructDef &struct_def, + const std::vector<std::pair<int, FieldDef *>> &non_deprecated_fields) { + std::string code; + + code += " @override\n"; + code += " int pack(fb.Builder fbBuilder) {\n"; + code += GenObjectBuilderImplementation(struct_def, non_deprecated_fields, + false, true); + code += " }\n"; + return code; } std::string NamespaceAliasFromUnionType(Namespace *root_namespace, @@ -476,7 +641,7 @@ class DartGenerator : public BaseGenerator { if (std::equal(root_namespace->components.begin(), root_namespace->components.end(), qualified_name_parts.begin())) { - return type.struct_def->name; + return namer_.Type(*type.struct_def); } std::string ns; @@ -496,40 +661,39 @@ class DartGenerator : public BaseGenerator { if (it != qualified_name_parts.end() - 1) { ns += "_"; } } - return ns + "." + type.struct_def->name; + return ns + "." + namer_.Type(*type.struct_def); } void GenImplementationGetters( const StructDef &struct_def, - std::vector<std::pair<int, FieldDef *>> non_deprecated_fields, - std::string *code_ptr) { - auto &code = *code_ptr; - + const std::vector<std::pair<int, FieldDef *>> &non_deprecated_fields, + std::string &code) { for (auto it = non_deprecated_fields.begin(); it != non_deprecated_fields.end(); ++it) { - auto pair = *it; - auto &field = *pair.second; + const FieldDef &field = *it->second; - std::string field_name = MakeCamel(field.name, false); - std::string type_name = GenDartTypeName( - field.value.type, struct_def.defined_namespace, field, false); + const std::string field_name = namer_.Field(field); + const std::string defaultValue = getDefaultValue(field.value); + const bool isNullable = defaultValue.empty() && !struct_def.fixed; + const std::string type_name = + GenDartTypeName(field.value.type, struct_def.defined_namespace, field, + isNullable, ""); - GenDocComment(field.doc_comment, &code, "", " "); + GenDocComment(field.doc_comment, " ", code); code += " " + type_name + " get " + field_name; if (field.value.type.base_type == BASE_TYPE_UNION) { code += " {\n"; code += " switch (" + field_name + "Type?.value) {\n"; - auto &enum_def = *field.value.type.enum_def; + const auto &enum_def = *field.value.type.enum_def; for (auto en_it = enum_def.Vals().begin() + 1; en_it != enum_def.Vals().end(); ++en_it) { - auto &ev = **en_it; - - auto enum_name = NamespaceAliasFromUnionType( + const auto &ev = **en_it; + const auto enum_name = NamespaceAliasFromUnionType( enum_def.defined_namespace, ev.union_type); code += " case " + enum_def.ToString(ev) + ": return " + - enum_name + ".reader.vTableGet(_bc, _bcOffset, " + - NumToString(field.value.offset) + ", null);\n"; + enum_name + ".reader.vTableGetNullable(_bc, _bcOffset, " + + NumToString(field.value.offset) + ");\n"; } code += " default: return null;\n"; code += " }\n"; @@ -538,10 +702,9 @@ class DartGenerator : public BaseGenerator { code += " => "; if (field.value.type.enum_def && field.value.type.base_type != BASE_TYPE_VECTOR) { - code += "new " + - GenDartTypeName(field.value.type, + code += GenDartTypeName(field.value.type, struct_def.defined_namespace, field) + - ".fromValue("; + (isNullable ? "._createOrNull(" : ".fromValue("); } code += GenReaderTypeName(field.value.type, @@ -550,33 +713,13 @@ class DartGenerator : public BaseGenerator { code += ".read(_bc, _bcOffset + " + NumToString(field.value.offset) + ")"; } else { - code += ".vTableGet(_bc, _bcOffset, " + - NumToString(field.value.offset) + ", "; - if (!field.value.constant.empty() && field.value.constant != "0") { - if (IsBool(field.value.type.base_type)) { - code += "true"; - } else if (field.value.constant == "nan" || - field.value.constant == "+nan" || - field.value.constant == "-nan") { - code += "double.nan"; - } else if (field.value.constant == "inf" || - field.value.constant == "+inf") { - code += "double.infinity"; - } else if (field.value.constant == "-inf") { - code += "double.negativeInfinity"; - } else { - code += field.value.constant; - } + code += ".vTableGet"; + std::string offset = NumToString(field.value.offset); + if (isNullable) { + code += "Nullable(_bc, _bcOffset, " + offset + ")"; } else { - if (IsBool(field.value.type.base_type)) { - code += "false"; - } else if (IsScalar(field.value.type.base_type)) { - code += "0"; - } else { - code += "null"; - } + code += "(_bc, _bcOffset, " + offset + ", " + defaultValue + ")"; } - code += ")"; } if (field.value.type.enum_def && field.value.type.base_type != BASE_TYPE_VECTOR) { @@ -587,27 +730,61 @@ class DartGenerator : public BaseGenerator { } code += "\n"; + code += GenToString(namer_.Type(struct_def), non_deprecated_fields); + } + std::string GenToString( + const std::string &object_name, + const std::vector<std::pair<int, FieldDef *>> &non_deprecated_fields) { + std::string code; code += " @override\n"; code += " String toString() {\n"; - code += " return '" + struct_def.name + "{"; + code += " return '" + object_name + "{"; for (auto it = non_deprecated_fields.begin(); it != non_deprecated_fields.end(); ++it) { - auto pair = *it; - auto &field = *pair.second; - code += - MakeCamel(field.name, false) + ": $" + MakeCamel(field.name, false); + const std::string field = namer_.Field(*it->second); + // We need to escape the fact that some fields have $ in the name which is + // also used in symbol/string substitution. + std::string escaped_field; + for (size_t i = 0; i < field.size(); i++) { + if (field[i] == '$') escaped_field.push_back('\\'); + escaped_field.push_back(field[i]); + } + code += escaped_field + ": ${" + field + "}"; if (it != non_deprecated_fields.end() - 1) { code += ", "; } } code += "}';\n"; code += " }\n"; + return code; } - void GenReader(const StructDef &struct_def, std::string *reader_name_ptr, - std::string *code_ptr) { - auto &code = *code_ptr; - auto &reader_name = *reader_name_ptr; - auto &impl_name = struct_def.name; + std::string getDefaultValue(const Value &value) const { + if (!value.constant.empty() && value.constant != "0") { + if (IsBool(value.type.base_type)) { + return "true"; + } + if (IsScalar(value.type.base_type)) { + if (StringIsFlatbufferNan(value.constant)) { + return "double.nan"; + } else if (StringIsFlatbufferPositiveInfinity(value.constant)) { + return "double.infinity"; + } else if (StringIsFlatbufferNegativeInfinity(value.constant)) { + return "double.negativeInfinity"; + } + } + return value.constant; + } else if (IsBool(value.type.base_type)) { + return "false"; + } else if (IsScalar(value.type.base_type) && !IsUnion(value.type)) { + return "0"; + } else { + return ""; + } + } + + void GenReader(const StructDef &struct_def, const std::string &reader_name, + std::string &code) { + const auto struct_type = namer_.Type(struct_def); code += "class " + reader_name + " extends " + _kFb; if (struct_def.fixed) { @@ -615,7 +792,7 @@ class DartGenerator : public BaseGenerator { } else { code += ".TableReader<"; } - code += impl_name + "> {\n"; + code += struct_type + "> {\n"; code += " const " + reader_name + "();\n\n"; if (struct_def.fixed) { @@ -623,29 +800,26 @@ class DartGenerator : public BaseGenerator { code += " int get size => " + NumToString(struct_def.bytesize) + ";\n\n"; } code += " @override\n"; - code += " " + impl_name + - " createObject(fb.BufferContext bc, int offset) => \n new " + - impl_name + "._(bc, offset);\n"; + code += " " + struct_type + + " createObject(fb.BufferContext bc, int offset) => \n " + + struct_type + "._(bc, offset);\n"; code += "}\n\n"; } - void GenBuilder(const StructDef &struct_def, - std::vector<std::pair<int, FieldDef *>> non_deprecated_fields, - std::string *builder_name_ptr, std::string *code_ptr) { + void GenBuilder( + const StructDef &struct_def, + const std::vector<std::pair<int, FieldDef *>> &non_deprecated_fields, + const std::string &builder_name, std::string &code) { if (non_deprecated_fields.size() == 0) { return; } - auto &code = *code_ptr; - auto &builder_name = *builder_name_ptr; code += "class " + builder_name + " {\n"; - code += " " + builder_name + "(this.fbBuilder) {\n"; - code += " assert(fbBuilder != null);\n"; - code += " }\n\n"; + code += " " + builder_name + "(this.fbBuilder);\n\n"; code += " final " + _kFb + ".Builder fbBuilder;\n\n"; if (struct_def.fixed) { - StructBuilderBody(struct_def, non_deprecated_fields, code_ptr); + StructBuilderBody(struct_def, non_deprecated_fields, code); } else { - TableBuilderBody(struct_def, non_deprecated_fields, code_ptr); + TableBuilderBody(struct_def, non_deprecated_fields, code); } code += "}\n\n"; @@ -653,15 +827,13 @@ class DartGenerator : public BaseGenerator { void StructBuilderBody( const StructDef &struct_def, - std::vector<std::pair<int, FieldDef *>> non_deprecated_fields, - std::string *code_ptr) { - auto &code = *code_ptr; - + const std::vector<std::pair<int, FieldDef *>> &non_deprecated_fields, + std::string &code) { code += " int finish("; for (auto it = non_deprecated_fields.begin(); it != non_deprecated_fields.end(); ++it) { - auto pair = *it; - auto &field = *pair.second; + const FieldDef &field = *it->second; + const std::string field_name = namer_.Field(field); if (IsStruct(field.value.type)) { code += "fb.StructBuilder"; @@ -669,26 +841,26 @@ class DartGenerator : public BaseGenerator { code += GenDartTypeName(field.value.type, struct_def.defined_namespace, field); } - code += " " + field.name; + code += " " + field_name; if (it != non_deprecated_fields.end() - 1) { code += ", "; } } code += ") {\n"; for (auto it = non_deprecated_fields.rbegin(); it != non_deprecated_fields.rend(); ++it) { - auto pair = *it; - auto &field = *pair.second; + const FieldDef &field = *it->second; + const std::string field_name = namer_.Field(field); if (field.padding) { code += " fbBuilder.pad(" + NumToString(field.padding) + ");\n"; } if (IsStruct(field.value.type)) { - code += " " + field.name + "();\n"; + code += " " + field_name + "();\n"; } else { code += " fbBuilder.put" + GenType(field.value.type) + "("; - code += field.name; - if (field.value.type.enum_def) { code += "?.value"; } + code += field_name; + if (field.value.type.enum_def) { code += ".value"; } code += ");\n"; } } @@ -698,36 +870,36 @@ class DartGenerator : public BaseGenerator { void TableBuilderBody( const StructDef &struct_def, - std::vector<std::pair<int, FieldDef *>> non_deprecated_fields, - std::string *code_ptr) { - auto &code = *code_ptr; - + const std::vector<std::pair<int, FieldDef *>> &non_deprecated_fields, + std::string &code) { code += " void begin() {\n"; - code += " fbBuilder.startTable();\n"; + code += " fbBuilder.startTable(" + + NumToString(struct_def.fields.vec.size()) + ");\n"; code += " }\n\n"; for (auto it = non_deprecated_fields.begin(); it != non_deprecated_fields.end(); ++it) { - auto pair = *it; - auto &field = *pair.second; - auto offset = pair.first; + const auto &field = *it->second; + const auto offset = it->first; + const std::string add_field = namer_.Method("add", field); + const std::string field_var = namer_.Variable(field); if (IsScalar(field.value.type.base_type)) { - code += " int add" + MakeCamel(field.name) + "("; + code += " int " + add_field + "("; code += GenDartTypeName(field.value.type, struct_def.defined_namespace, field); - code += " " + MakeCamel(field.name, false) + ") {\n"; + code += "? " + field_var + ") {\n"; code += " fbBuilder.add" + GenType(field.value.type) + "(" + NumToString(offset) + ", "; - code += MakeCamel(field.name, false); + code += field_var; if (field.value.type.enum_def) { code += "?.value"; } code += ");\n"; } else if (IsStruct(field.value.type)) { - code += " int add" + MakeCamel(field.name) + "(int offset) {\n"; + code += " int " + add_field + "(int offset) {\n"; code += " fbBuilder.addStruct(" + NumToString(offset) + ", offset);\n"; } else { - code += " int add" + MakeCamel(field.name) + "Offset(int offset) {\n"; + code += " int " + add_field + "Offset(int? offset) {\n"; code += " fbBuilder.addOffset(" + NumToString(offset) + ", offset);\n"; } @@ -743,21 +915,17 @@ class DartGenerator : public BaseGenerator { void GenObjectBuilder( const StructDef &struct_def, - std::vector<std::pair<int, FieldDef *>> non_deprecated_fields, - std::string *builder_name_ptr, std::string *code_ptr) { - auto &code = *code_ptr; - auto &builder_name = *builder_name_ptr; - + const std::vector<std::pair<int, FieldDef *>> &non_deprecated_fields, + const std::string &builder_name, std::string &code) { code += "class " + builder_name + " extends " + _kFb + ".ObjectBuilder {\n"; for (auto it = non_deprecated_fields.begin(); it != non_deprecated_fields.end(); ++it) { - auto pair = *it; - auto &field = *pair.second; + const FieldDef &field = *it->second; code += " final " + GenDartTypeName(field.value.type, struct_def.defined_namespace, - field, true) + - " _" + MakeCamel(field.name, false) + ";\n"; + field, !struct_def.fixed, "ObjectBuilder") + + " _" + namer_.Variable(field) + ";\n"; } code += "\n"; code += " " + builder_name + "("; @@ -766,23 +934,21 @@ class DartGenerator : public BaseGenerator { code += "{\n"; for (auto it = non_deprecated_fields.begin(); it != non_deprecated_fields.end(); ++it) { - auto pair = *it; - auto &field = *pair.second; + const FieldDef &field = *it->second; - code += " " + + code += " "; + code += (struct_def.fixed ? "required " : "") + GenDartTypeName(field.value.type, struct_def.defined_namespace, - field, true) + - " " + MakeCamel(field.name, false) + ",\n"; + field, !struct_def.fixed, "ObjectBuilder") + + " " + namer_.Variable(field) + ",\n"; } code += " })\n"; code += " : "; for (auto it = non_deprecated_fields.begin(); it != non_deprecated_fields.end(); ++it) { - auto pair = *it; - auto &field = *pair.second; + const FieldDef &field = *it->second; - code += "_" + MakeCamel(field.name, false) + " = " + - MakeCamel(field.name, false); + code += "_" + namer_.Variable(field) + " = " + namer_.Variable(field); if (it == non_deprecated_fields.end() - 1) { code += ";\n\n"; } else { @@ -795,80 +961,105 @@ class DartGenerator : public BaseGenerator { code += " /// Finish building, and store into the [fbBuilder].\n"; code += " @override\n"; - code += " int finish(\n"; - code += " " + _kFb + ".Builder fbBuilder) {\n"; - code += " assert(fbBuilder != null);\n"; + code += " int finish(" + _kFb + ".Builder fbBuilder) {\n"; + code += GenObjectBuilderImplementation(struct_def, non_deprecated_fields); + code += " }\n\n"; + + code += " /// Convenience method to serialize to byte list.\n"; + code += " @override\n"; + code += " Uint8List toBytes([String? fileIdentifier]) {\n"; + code += " final fbBuilder = " + _kFb + + ".Builder(deduplicateTables: false);\n"; + code += " fbBuilder.finish(finish(fbBuilder), fileIdentifier);\n"; + code += " return fbBuilder.buffer;\n"; + code += " }\n"; + code += "}\n"; + } + std::string GenObjectBuilderImplementation( + const StructDef &struct_def, + const std::vector<std::pair<int, FieldDef *>> &non_deprecated_fields, + bool prependUnderscore = true, bool pack = false) { + std::string code; for (auto it = non_deprecated_fields.begin(); it != non_deprecated_fields.end(); ++it) { - auto pair = *it; - auto &field = *pair.second; + const FieldDef &field = *it->second; if (IsScalar(field.value.type.base_type) || IsStruct(field.value.type)) continue; - code += " final int " + MakeCamel(field.name, false) + "Offset"; - if (IsVector(field.value.type)) { + std::string offset_name = namer_.Variable(field) + "Offset"; + std::string field_name = + (prependUnderscore ? "_" : "") + namer_.Variable(field); + // custom handling for fixed-sized struct in pack() + if (pack && IsVector(field.value.type) && + field.value.type.VectorType().base_type == BASE_TYPE_STRUCT && + field.value.type.struct_def->fixed) { + code += " int? " + offset_name + ";\n"; + code += " if (" + field_name + " != null) {\n"; code += - " = _" + MakeCamel(field.name, false) + "?.isNotEmpty == true\n"; - code += " ? fbBuilder.writeList"; + " for (var e in " + field_name + "!) { e.pack(fbBuilder); }\n"; + code += " " + namer_.Variable(field) + + "Offset = fbBuilder.endStructVector(" + field_name + + "!.length);\n"; + code += " }\n"; + continue; + } + + code += " final int? " + offset_name; + if (IsVector(field.value.type)) { + code += " = " + field_name + " == null ? null\n"; + code += " : fbBuilder.writeList"; switch (field.value.type.VectorType().base_type) { case BASE_TYPE_STRING: - code += "(_" + MakeCamel(field.name, false) + - ".map((b) => fbBuilder.writeString(b)).toList())"; + code += + "(" + field_name + "!.map(fbBuilder.writeString).toList());\n"; break; case BASE_TYPE_STRUCT: if (field.value.type.struct_def->fixed) { - code += "OfStructs(_" + MakeCamel(field.name, false) + ")"; + code += "OfStructs(" + field_name + "!);\n"; } else { - code += "(_" + MakeCamel(field.name, false) + - ".map((b) => b.getOrCreateOffset(fbBuilder)).toList())"; + code += "(" + field_name + "!.map((b) => b." + + (pack ? "pack" : "getOrCreateOffset") + + "(fbBuilder)).toList());\n"; } break; default: - code += GenType(field.value.type.VectorType()) + "(_" + - MakeCamel(field.name, false); - if (field.value.type.enum_def) { code += ".map((f) => f.value)"; } - code += ")"; + code += + GenType(field.value.type.VectorType()) + "(" + field_name + "!"; + if (field.value.type.enum_def) { + code += ".map((f) => f.value).toList()"; + } + code += ");\n"; } - code += "\n : null;\n"; } else if (IsString(field.value.type)) { - code += " = fbBuilder.writeString(_" + MakeCamel(field.name, false) + - ");\n"; + code += " = " + field_name + " == null ? null\n"; + code += " : fbBuilder.writeString(" + field_name + "!);\n"; } else { - code += " = _" + MakeCamel(field.name, false) + - "?.getOrCreateOffset(fbBuilder);\n"; + code += " = " + field_name + "?." + + (pack ? "pack" : "getOrCreateOffset") + "(fbBuilder);\n"; } } - code += "\n"; if (struct_def.fixed) { - StructObjectBuilderBody(non_deprecated_fields, code_ptr); + code += StructObjectBuilderBody(non_deprecated_fields, prependUnderscore, + pack); } else { - TableObjectBuilderBody(non_deprecated_fields, code_ptr); + code += TableObjectBuilderBody(struct_def, non_deprecated_fields, + prependUnderscore, pack); } - code += " }\n\n"; - - code += " /// Convenience method to serialize to byte list.\n"; - code += " @override\n"; - code += " Uint8List toBytes([String fileIdentifier]) {\n"; - code += " " + _kFb + ".Builder fbBuilder = new "; - code += _kFb + ".Builder();\n"; - code += " int offset = finish(fbBuilder);\n"; - code += " return fbBuilder.finish(offset, fileIdentifier);\n"; - code += " }\n"; - code += "}\n"; + return code; } - void StructObjectBuilderBody( - std::vector<std::pair<int, FieldDef *>> non_deprecated_fields, - std::string *code_ptr, bool prependUnderscore = true) { - auto &code = *code_ptr; + std::string StructObjectBuilderBody( + const std::vector<std::pair<int, FieldDef *>> &non_deprecated_fields, + bool prependUnderscore = true, bool pack = false) { + std::string code; for (auto it = non_deprecated_fields.rbegin(); it != non_deprecated_fields.rend(); ++it) { - auto pair = *it; - auto &field = *pair.second; + const FieldDef &field = *it->second; + const std::string field_name = namer_.Field(field); if (field.padding) { code += " fbBuilder.pad(" + NumToString(field.padding) + ");\n"; @@ -877,55 +1068,59 @@ class DartGenerator : public BaseGenerator { if (IsStruct(field.value.type)) { code += " "; if (prependUnderscore) { code += "_"; } - code += field.name + ".finish(fbBuilder);\n"; + code += field_name + (pack ? ".pack" : ".finish") + "(fbBuilder);\n"; } else { code += " fbBuilder.put" + GenType(field.value.type) + "("; if (prependUnderscore) { code += "_"; } - code += field.name; - if (field.value.type.enum_def) { code += "?.value"; } + code += field_name; + if (field.value.type.enum_def) { code += ".value"; } code += ");\n"; } } code += " return fbBuilder.offset;\n"; + return code; } - void TableObjectBuilderBody( - std::vector<std::pair<int, FieldDef *>> non_deprecated_fields, - std::string *code_ptr, bool prependUnderscore = true) { - std::string &code = *code_ptr; - code += " fbBuilder.startTable();\n"; + std::string TableObjectBuilderBody( + const StructDef &struct_def, + const std::vector<std::pair<int, FieldDef *>> &non_deprecated_fields, + bool prependUnderscore = true, bool pack = false) { + std::string code; + code += " fbBuilder.startTable(" + + NumToString(struct_def.fields.vec.size()) + ");\n"; for (auto it = non_deprecated_fields.begin(); it != non_deprecated_fields.end(); ++it) { - auto pair = *it; - auto &field = *pair.second; - auto offset = pair.first; + const FieldDef &field = *it->second; + auto offset = it->first; + + std::string field_var = + (prependUnderscore ? "_" : "") + namer_.Variable(field); if (IsScalar(field.value.type.base_type)) { code += " fbBuilder.add" + GenType(field.value.type) + "(" + - NumToString(offset) + ", "; - if (prependUnderscore) { code += "_"; } - code += MakeCamel(field.name, false); - if (field.value.type.enum_def) { code += "?.value"; } + NumToString(offset) + ", " + field_var; + if (field.value.type.enum_def) { + bool isNullable = getDefaultValue(field.value).empty(); + code += (isNullable || !pack) ? "?.value" : ".value"; + } code += ");\n"; } else if (IsStruct(field.value.type)) { - code += " if ("; - if (prependUnderscore) { code += "_"; } - code += MakeCamel(field.name, false) + " != null) {\n"; - code += " fbBuilder.addStruct(" + NumToString(offset) + ", "; - code += "_" + MakeCamel(field.name, false) + ".finish(fbBuilder));\n"; + code += " if (" + field_var + " != null) {\n"; + code += " fbBuilder.addStruct(" + NumToString(offset) + ", " + + field_var + (pack ? "!.pack" : "!.finish") + "(fbBuilder));\n"; code += " }\n"; } else { - code += - " if (" + MakeCamel(field.name, false) + "Offset != null) {\n"; - code += " fbBuilder.addOffset(" + NumToString(offset) + ", " + - MakeCamel(field.name, false) + "Offset);\n"; - code += " }\n"; + code += " fbBuilder.addOffset(" + NumToString(offset) + ", " + + namer_.Variable(field) + "Offset);\n"; } } code += " return fbBuilder.endTable();\n"; + return code; } + + const IdlNamer namer_; }; } // namespace dart @@ -937,13 +1132,10 @@ bool GenerateDart(const Parser &parser, const std::string &path, std::string DartMakeRule(const Parser &parser, const std::string &path, const std::string &file_name) { - assert(parser.opts.lang <= IDLOptions::kMAX); - auto filebase = flatbuffers::StripPath(flatbuffers::StripExtension(file_name)); dart::DartGenerator generator(parser, path, file_name); - auto make_rule = - generator.GeneratedFileName(path, file_name, parser.opts) + ": "; + auto make_rule = generator.Filename("") + ": "; auto included_files = parser.GetIncludedFilesRecursive(file_name); for (auto it = included_files.begin(); it != included_files.end(); ++it) { @@ -952,4 +1144,58 @@ std::string DartMakeRule(const Parser &parser, const std::string &path, return make_rule; } +namespace { + +class DartCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateDart(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + output = DartMakeRule(parser, path, filename); + return Status::OK; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kDart; } + + std::string LanguageName() const override { return "Dart"; } +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewDartCodeGenerator() { + return std::unique_ptr<DartCodeGenerator>(new DartCodeGenerator()); +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_dart.h b/contrib/libs/flatbuffers/src/idl_gen_dart.h new file mode 100644 index 00000000000..efaa08e39bc --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_dart.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_DART_H_ +#define FLATBUFFERS_IDL_GEN_DART_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Dart code generator. +std::unique_ptr<CodeGenerator> NewDartCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_DART_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_fbs.cpp b/contrib/libs/flatbuffers/src/idl_gen_fbs.cpp index 35c1a7d4f54..f71c21f97d9 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_fbs.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_fbs.cpp @@ -15,7 +15,13 @@ */ // independent from idl_parser, since this code is not needed for most clients +#include "idl_gen_fbs.h" +#include <unordered_map> +#include <utility> +#include <vector> + +#include "flatbuffers/code_generator.h" #include "flatbuffers/code_generators.h" #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" @@ -34,9 +40,201 @@ static std::string GenType(const Type &type, bool underlying = false) { return type.enum_def->defined_namespace->GetFullyQualifiedName( type.enum_def->name); } else { - return kTypeNames[type.base_type]; + return TypeName(type.base_type); + } + } +} + +static bool HasFieldWithId(const std::vector<FieldDef *> &fields) { + static const std::string ID = "id"; + + for (const auto *field : fields) { + const auto *id_attribute = field->attributes.Lookup(ID); + if (id_attribute != nullptr && !id_attribute->constant.empty()) { + return true; + } + } + return false; +} + +static bool HasNonPositiveFieldId(const std::vector<FieldDef *> &fields) { + static const std::string ID = "id"; + + for (const auto *field : fields) { + const auto *id_attribute = field->attributes.Lookup(ID); + if (id_attribute != nullptr && !id_attribute->constant.empty()) { + voffset_t proto_id = 0; + bool done = StringToNumber(id_attribute->constant.c_str(), &proto_id); + if (!done) { return true; } + } + } + return false; +} + +static bool HasFieldIdFromReservedIds( + const std::vector<FieldDef *> &fields, + const std::vector<voffset_t> &reserved_ids) { + static const std::string ID = "id"; + + for (const auto *field : fields) { + const auto *id_attribute = field->attributes.Lookup(ID); + if (id_attribute != nullptr && !id_attribute->constant.empty()) { + voffset_t proto_id = 0; + bool done = StringToNumber(id_attribute->constant.c_str(), &proto_id); + if (!done) { return true; } + auto id_it = + std::find(std::begin(reserved_ids), std::end(reserved_ids), proto_id); + if (id_it != reserved_ids.end()) { return true; } + } + } + return false; +} + +static std::vector<voffset_t> ExtractProtobufIds( + const std::vector<FieldDef *> &fields) { + static const std::string ID = "id"; + std::vector<voffset_t> used_proto_ids; + for (const auto *field : fields) { + const auto *id_attribute = field->attributes.Lookup(ID); + if (id_attribute != nullptr && !id_attribute->constant.empty()) { + voffset_t proto_id = 0; + bool done = StringToNumber(id_attribute->constant.c_str(), &proto_id); + if (done) { used_proto_ids.push_back(proto_id); } + } + } + + return used_proto_ids; +} + +static bool HasTwiceUsedId(const std::vector<FieldDef *> &fields) { + std::vector<voffset_t> used_proto_ids = ExtractProtobufIds(fields); + std::sort(std::begin(used_proto_ids), std::end(used_proto_ids)); + for (auto it = std::next(std::begin(used_proto_ids)); + it != std::end(used_proto_ids); it++) { + if (*it == *std::prev(it)) { return true; } + } + + return false; +} + +static bool HasGapInProtoId(const std::vector<FieldDef *> &fields) { + std::vector<voffset_t> used_proto_ids = ExtractProtobufIds(fields); + std::sort(std::begin(used_proto_ids), std::end(used_proto_ids)); + for (auto it = std::next(std::begin(used_proto_ids)); + it != std::end(used_proto_ids); it++) { + if (*it != *std::prev(it) + 1) { return true; } + } + + return false; +} + +static bool ProtobufIdSanityCheck(const StructDef &struct_def, + IDLOptions::ProtoIdGapAction gap_action, + bool no_log = false) { + const auto &fields = struct_def.fields.vec; + if (HasNonPositiveFieldId(fields)) { + // TODO: Use LogCompilerWarn + if (!no_log) { + fprintf(stderr, "Field id in struct %s has a non positive number value\n", + struct_def.name.c_str()); + } + return false; + } + + if (HasTwiceUsedId(fields)) { + // TODO: Use LogCompilerWarn + if (!no_log) { + fprintf(stderr, "Fields in struct %s have used an id twice\n", + struct_def.name.c_str()); + } + return false; + } + + if (HasFieldIdFromReservedIds(fields, struct_def.reserved_ids)) { + // TODO: Use LogCompilerWarn + if (!no_log) { + fprintf(stderr, "Fields in struct %s use id from reserved ids\n", + struct_def.name.c_str()); + } + return false; + } + + if (gap_action != IDLOptions::ProtoIdGapAction::NO_OP) { + if (HasGapInProtoId(fields)) { + // TODO: Use LogCompilerWarn + if (!no_log) { + fprintf(stderr, "Fields in struct %s have gap between ids\n", + struct_def.name.c_str()); } + if (gap_action == IDLOptions::ProtoIdGapAction::ERROR) { return false; } + } } + + return true; +} + +struct ProtobufToFbsIdMap { + using FieldName = std::string; + using FieldID = voffset_t; + using FieldNameToIdMap = std::unordered_map<FieldName, FieldID>; + + FieldNameToIdMap field_to_id; + bool successful = false; +}; + +static ProtobufToFbsIdMap MapProtoIdsToFieldsId( + const StructDef &struct_def, IDLOptions::ProtoIdGapAction gap_action, + bool no_log) { + const auto &fields = struct_def.fields.vec; + + if (!HasFieldWithId(fields)) { + ProtobufToFbsIdMap result; + result.successful = true; + return result; + } + + if (!ProtobufIdSanityCheck(struct_def, gap_action, no_log)) { return {}; } + + static constexpr int UNION_ID = -1; + using ProtoIdFieldNamePair = std::pair<int, std::string>; + std::vector<ProtoIdFieldNamePair> proto_ids; + + for (const auto *field : fields) { + const auto *id_attribute = field->attributes.Lookup("id"); + if (id_attribute != nullptr) { + // When we have union but do not use union flag to keep them + if (id_attribute->constant.empty() && + field->value.type.base_type == BASE_TYPE_UNION) { + proto_ids.emplace_back(UNION_ID, field->name); + } else { + voffset_t proto_id = 0; + StringToNumber(id_attribute->constant.c_str(), &proto_id); + proto_ids.emplace_back(proto_id, field->name); + } + } else { + // TODO: Use LogCompilerWarn + if (!no_log) { + fprintf(stderr, "Fields id in struct %s is missing\n", + struct_def.name.c_str()); + } + return {}; + } + } + + std::sort( + std::begin(proto_ids), std::end(proto_ids), + [](const ProtoIdFieldNamePair &rhs, const ProtoIdFieldNamePair &lhs) { + return rhs.first < lhs.first; + }); + struct ProtobufToFbsIdMap proto_to_fbs; + + voffset_t id = 0; + for (const auto &element : proto_ids) { + if (element.first == UNION_ID) { id++; } + proto_to_fbs.field_to_id.emplace(element.second, id++); + } + proto_to_fbs.successful = true; + return proto_to_fbs; } static void GenNameSpace(const Namespace &name_space, std::string *_schema, @@ -54,7 +252,8 @@ static void GenNameSpace(const Namespace &name_space, std::string *_schema, } // Generate a flatbuffer schema from the Parser's internal representation. -std::string GenerateFBS(const Parser &parser, const std::string &file_name) { +std::string GenerateFBS(const Parser &parser, const std::string &file_name, + bool no_log = false) { // Proto namespaces may clash with table names, escape the ones that were // generated from a table: for (auto it = parser.namespaces_.begin(); it != parser.namespaces_.end(); @@ -79,10 +278,11 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name) { int num_includes = 0; for (auto it = parser.included_files_.begin(); it != parser.included_files_.end(); ++it) { - if (it->second.empty()) + if (it->second.empty()) { continue; +} std::string basename; - if(parser.opts.keep_include_path) { + if(parser.opts.keep_prefix) { basename = flatbuffers::StripExtension(it->second); } else { basename = flatbuffers::StripPath( @@ -94,6 +294,7 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name) { if (num_includes) schema += "\n"; // clang-format on } + // Generate code for all the enum declarations. const Namespace *last_namespace = nullptr; for (auto enum_def_it = parser.enums_.vec.begin(); @@ -104,18 +305,22 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name) { } GenNameSpace(*enum_def.defined_namespace, &schema, &last_namespace); GenComment(enum_def.doc_comment, &schema, nullptr); - if (enum_def.is_union) + if (enum_def.is_union) { schema += "union " + enum_def.name; - else + } else { schema += "enum " + enum_def.name + " : "; + } + schema += GenType(enum_def.underlying_type, true) + " {\n"; + for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { auto &ev = **it; GenComment(ev.doc_comment, &schema, nullptr, " "); - if (enum_def.is_union) + if (enum_def.is_union) { schema += " " + GenType(ev.union_type) + ",\n"; - else + } else { schema += " " + ev.name + " = " + enum_def.ToString(ev) + ",\n"; + } } schema += "}\n\n"; } @@ -123,9 +328,14 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name) { for (auto it = parser.structs_.vec.begin(); it != parser.structs_.vec.end(); ++it) { StructDef &struct_def = **it; + const auto proto_fbs_ids = MapProtoIdsToFieldsId( + struct_def, parser.opts.proto_id_gap_action, no_log); + if (!proto_fbs_ids.successful) { return {}; } + if (parser.opts.include_dependence_headers && struct_def.generated) { continue; } + GenNameSpace(*struct_def.defined_namespace, &schema, &last_namespace); GenComment(struct_def.doc_comment, &schema, nullptr); schema += "table " + struct_def.name + " {\n"; @@ -136,7 +346,26 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name) { GenComment(field.doc_comment, &schema, nullptr, " "); schema += " " + field.name + ":" + GenType(field.value.type); if (field.value.constant != "0") schema += " = " + field.value.constant; - if (field.IsRequired()) schema += " (required)"; + std::vector<std::string> attributes; + if (field.IsRequired()) attributes.push_back("required"); + if (field.key) attributes.push_back("key"); + + if (parser.opts.keep_proto_id) { + auto it = proto_fbs_ids.field_to_id.find(field.name); + if (it != proto_fbs_ids.field_to_id.end()) { + attributes.push_back("id: " + NumToString(it->second)); + } // If not found it means we do not have any ids + } + + if (!attributes.empty()) { + schema += " ("; + for (const auto &attribute : attributes) { + schema += attribute + ","; + } + schema.pop_back(); + schema += ")"; + } + schema += ";\n"; } } @@ -146,9 +375,81 @@ std::string GenerateFBS(const Parser &parser, const std::string &file_name) { } bool GenerateFBS(const Parser &parser, const std::string &path, - const std::string &file_name) { - return SaveFile((path + file_name + ".fbs").c_str(), - GenerateFBS(parser, file_name), false); + const std::string &file_name, bool no_log = false) { + const std::string fbs = GenerateFBS(parser, file_name, no_log); + if (fbs.empty()) { return false; } + // TODO: Use LogCompilerWarn + if (!no_log) { + fprintf(stderr, + "When you use --proto, that you should check for conformity " + "yourself, using the existing --conform"); + } + return SaveFile((path + file_name + ".fbs").c_str(), fbs, false); +} + +namespace { + +class FBSCodeGenerator : public CodeGenerator { + public: + explicit FBSCodeGenerator(const bool no_log) : no_log_(no_log) {} + + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateFBS(parser, path, filename, no_log_)) { return Status::ERROR; } + return Status::OK; + } + + // Generate code from the provided `buffer` of given `length`. The buffer is a + // serialized reflection.fbs. + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + (void)parser; + (void)path; + (void)filename; + (void)output; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return false; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kProto; } + + std::string LanguageName() const override { return "proto"; } + + protected: + const bool no_log_; +}; + +} // namespace + +std::unique_ptr<CodeGenerator> NewFBSCodeGenerator(const bool no_log) { + return std::unique_ptr<FBSCodeGenerator>(new FBSCodeGenerator(no_log)); } } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_fbs.h b/contrib/libs/flatbuffers/src/idl_gen_fbs.h new file mode 100644 index 00000000000..403f160b9bd --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_fbs.h @@ -0,0 +1,28 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_FBS_H_ +#define FLATBUFFERS_IDL_GEN_FBS_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +std::unique_ptr<CodeGenerator> NewFBSCodeGenerator(bool no_log = false); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_FBS_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_go.cpp b/contrib/libs/flatbuffers/src/idl_gen_go.cpp index 867f4023221..0f2882b7584 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_go.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_go.cpp @@ -16,13 +16,19 @@ // independent from idl_parser, since this code is not needed for most clients +#include "idl_gen_go.h" + +#include <algorithm> +#include <cmath> #include <sstream> #include <string> +#include "flatbuffers/base.h" #include "flatbuffers/code_generators.h" #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" +#include "idl_namer.h" #ifdef _WIN32 # include <direct.h> @@ -37,30 +43,56 @@ namespace flatbuffers { namespace go { -// see https://golang.org/ref/spec#Keywords -static const char *const g_golang_keywords[] = { - "break", "default", "func", "interface", "select", "case", "defer", - "go", "map", "struct", "chan", "else", "goto", "package", - "switch", "const", "fallthrough", "if", "range", "type", "continue", - "for", "import", "return", "var", -}; +namespace { -static std::string GoIdentity(const std::string &name) { - for (size_t i = 0; - i < sizeof(g_golang_keywords) / sizeof(g_golang_keywords[0]); i++) { - if (name == g_golang_keywords[i]) { return MakeCamel(name + "_", false); } - } +// see https://golang.org/ref/spec#Keywords +static std::set<std::string> GoKeywords() { + return { + "break", "default", "func", "interface", "select", + "case", "defer", "go", "map", "struct", + "chan", "else", "goto", "package", "switch", + "const", "fallthrough", "if", "range", "type", + "continue", "for", "import", "return", "var", + }; +} - return MakeCamel(name, false); +static Namer::Config GoDefaultConfig() { + // Note that the functions with user defined types in the name use + // upper camel case for all but the user defined type itself, which is keep + // cased. Despite being a function, we interpret it as a Type. + return { /*types=*/Case::kKeep, + /*constants=*/Case::kUnknown, + /*methods=*/Case::kUpperCamel, + /*functions=*/Case::kUpperCamel, + /*fields=*/Case::kUpperCamel, + /*variables=*/Case::kLowerCamel, + /*variants=*/Case::kKeep, + /*enum_variant_seperator=*/"", // I.e. Concatenate. + /*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase, + /*namespaces=*/Case::kKeep, + /*namespace_seperator=*/"__", + /*object_prefix=*/"", + /*object_suffix=*/"T", + /*keyword_prefix=*/"", + /*keyword_suffix=*/"_", + /*filenames=*/Case::kKeep, + /*directories=*/Case::kKeep, + /*output_path=*/"", + /*filename_suffix=*/"", + /*filename_extension=*/".go" }; } +} // namespace + class GoGenerator : public BaseGenerator { public: GoGenerator(const Parser &parser, const std::string &path, const std::string &file_name, const std::string &go_namespace) : BaseGenerator(parser, path, file_name, "" /* not used*/, "" /* not used */, "go"), - cur_name_space_(nullptr) { + cur_name_space_(nullptr), + namer_(WithFlagOptions(GoDefaultConfig(), parser.opts, path), + GoKeywords()) { std::istringstream iss(go_namespace); std::string component; while (std::getline(iss, component, '.')) { @@ -73,8 +105,10 @@ class GoGenerator : public BaseGenerator { bool needs_imports = false; for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end(); ++it) { - tracked_imported_namespaces_.clear(); - needs_imports = false; + if (!parser_.opts.one_file) { + needs_imports = false; + ResetImports(); + } std::string enumcode; GenEnum(**it, &enumcode); if ((*it)->is_union && parser_.opts.generate_object_based_api) { @@ -92,7 +126,7 @@ class GoGenerator : public BaseGenerator { for (auto it = parser_.structs_.vec.begin(); it != parser_.structs_.vec.end(); ++it) { - tracked_imported_namespaces_.clear(); + if (!parser_.opts.one_file) { ResetImports(); } std::string declcode; GenStruct(**it, &declcode); if (parser_.opts.one_file) { @@ -118,13 +152,16 @@ class GoGenerator : public BaseGenerator { private: Namespace go_namespace_; Namespace *cur_name_space_; + const IdlNamer namer_; struct NamespacePtrLess { - bool operator()(const Namespace *a, const Namespace *b) const { - return *a < *b; + bool operator()(const Definition *a, const Definition *b) const { + return *a->defined_namespace < *b->defined_namespace; } }; - std::set<const Namespace *, NamespacePtrLess> tracked_imported_namespaces_; + std::set<const Definition *, NamespacePtrLess> tracked_imported_namespaces_; + bool needs_math_import_ = false; + bool needs_bytes_import_ = false; // Most field accessors need to retrieve and test the field offset first, // this is the prefix code for that. @@ -137,7 +174,7 @@ class GoGenerator : public BaseGenerator { void BeginClass(const StructDef &struct_def, std::string *code_ptr) { std::string &code = *code_ptr; - code += "type " + struct_def.name + " struct {\n\t"; + code += "type " + namer_.Type(struct_def) + " struct {\n\t"; // _ is reserved in flatbuffers field names, so no chance of name conflict: code += "_tab "; @@ -147,8 +184,7 @@ class GoGenerator : public BaseGenerator { // Construct the name of the type for this enum. std::string GetEnumTypeName(const EnumDef &enum_def) { - return WrapInNameSpaceAndTrack(enum_def.defined_namespace, - GoIdentity(enum_def.name)); + return WrapInNameSpaceAndTrack(&enum_def, namer_.Type(enum_def)); } // Create a type for the enum values. @@ -169,8 +205,7 @@ class GoGenerator : public BaseGenerator { size_t max_name_length, std::string *code_ptr) { std::string &code = *code_ptr; code += "\t"; - code += enum_def.name; - code += ev.name; + code += namer_.EnumVariant(enum_def, ev); code += " "; code += std::string(max_name_length - ev.name.length(), ' '); code += GetEnumTypeName(enum_def); @@ -197,8 +232,7 @@ class GoGenerator : public BaseGenerator { size_t max_name_length, std::string *code_ptr) { std::string &code = *code_ptr; code += "\t"; - code += enum_def.name; - code += ev.name; + code += namer_.EnumVariant(enum_def, ev); code += ": "; code += std::string(max_name_length - ev.name.length(), ' '); code += "\""; @@ -215,8 +249,9 @@ class GoGenerator : public BaseGenerator { // Generate String() method on enum type. void EnumStringer(const EnumDef &enum_def, std::string *code_ptr) { std::string &code = *code_ptr; - code += "func (v " + enum_def.name + ") String() string {\n"; - code += "\tif s, ok := EnumNames" + enum_def.name + "[v]; ok {\n"; + const std::string enum_type = namer_.Type(enum_def); + code += "func (v " + enum_type + ") String() string {\n"; + code += "\tif s, ok := EnumNames" + enum_type + "[v]; ok {\n"; code += "\t\treturn s\n"; code += "\t}\n"; code += "\treturn \"" + enum_def.name; @@ -228,7 +263,7 @@ class GoGenerator : public BaseGenerator { void BeginEnumValues(const EnumDef &enum_def, std::string *code_ptr) { std::string &code = *code_ptr; code += "var EnumValues"; - code += enum_def.name; + code += namer_.Type(enum_def); code += " = map[string]" + GetEnumTypeName(enum_def) + "{\n"; } @@ -240,8 +275,7 @@ class GoGenerator : public BaseGenerator { code += ev.name; code += "\": "; code += std::string(max_name_length - ev.name.length(), ' '); - code += enum_def.name; - code += ev.name; + code += namer_.EnumVariant(enum_def, ev); code += ",\n"; } @@ -255,13 +289,21 @@ class GoGenerator : public BaseGenerator { void NewRootTypeFromBuffer(const StructDef &struct_def, std::string *code_ptr) { std::string &code = *code_ptr; - std::string size_prefix[] = { "", "SizePrefixed" }; + const std::string size_prefix[] = { "", "SizePrefixed" }; + const std::string struct_type = namer_.Type(struct_def); + + bool has_file_identifier = (parser_.root_struct_def_ == &struct_def) && + parser_.file_identifier_.length(); + + if (has_file_identifier) { + code += "const " + struct_type + "Identifier = \"" + + parser_.file_identifier_ + "\"\n\n"; + } for (int i = 0; i < 2; i++) { - code += "func Get" + size_prefix[i] + "RootAs"; - code += struct_def.name; + code += "func Get" + size_prefix[i] + "RootAs" + struct_type; code += "(buf []byte, offset flatbuffers.UOffsetT) "; - code += "*" + struct_def.name + ""; + code += "*" + struct_type + ""; code += " {\n"; if (i == 0) { code += "\tn := flatbuffers.GetUOffsetT(buf[offset:])\n"; @@ -270,7 +312,7 @@ class GoGenerator : public BaseGenerator { "\tn := " "flatbuffers.GetUOffsetT(buf[offset+flatbuffers.SizeUint32:])\n"; } - code += "\tx := &" + struct_def.name + "{}\n"; + code += "\tx := &" + struct_type + "{}\n"; if (i == 0) { code += "\tx.Init(buf, n+offset)\n"; } else { @@ -278,6 +320,26 @@ class GoGenerator : public BaseGenerator { } code += "\treturn x\n"; code += "}\n\n"; + + code += "func Finish" + size_prefix[i] + struct_type + + "Buffer(builder *flatbuffers.Builder, offset " + "flatbuffers.UOffsetT) {\n"; + if (has_file_identifier) { + code += "\tidentifierBytes := []byte(" + struct_type + "Identifier)\n"; + code += "\tbuilder.Finish" + size_prefix[i] + + "WithFileIdentifier(offset, identifierBytes)\n"; + } else { + code += "\tbuilder.Finish" + size_prefix[i] + "(offset)\n"; + } + code += "}\n\n"; + + if (has_file_identifier) { + code += "func " + size_prefix[i] + struct_type + + "BufferHasIdentifier(buf []byte) bool {\n"; + code += "\treturn flatbuffers." + size_prefix[i] + + "BufferHasIdentifier(buf, " + struct_type + "Identifier)\n"; + code += "}\n\n"; + } } } @@ -315,7 +377,7 @@ class GoGenerator : public BaseGenerator { std::string &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += " " + MakeCamel(field.name) + "Length("; + code += " " + namer_.Function(field) + "Length("; code += ") int " + OffsetPrefix(field); code += "\t\treturn rcv._tab.VectorLen(o)\n\t}\n"; code += "\treturn 0\n}\n\n"; @@ -327,7 +389,7 @@ class GoGenerator : public BaseGenerator { std::string &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += " " + MakeCamel(field.name) + "Bytes("; + code += " " + namer_.Function(field) + "Bytes("; code += ") []byte " + OffsetPrefix(field); code += "\t\treturn rcv._tab.ByteVector(o + rcv._tab.Pos)\n\t}\n"; code += "\treturn nil\n}\n\n"; @@ -339,7 +401,7 @@ class GoGenerator : public BaseGenerator { std::string &code = *code_ptr; std::string getter = GenGetter(field.value.type); GenReceiver(struct_def, code_ptr); - code += " " + MakeCamel(field.name); + code += " " + namer_.Function(field); code += "() " + TypeName(field) + " {\n"; code += "\treturn " + CastToEnum(field.value.type, @@ -354,10 +416,16 @@ class GoGenerator : public BaseGenerator { std::string &code = *code_ptr; std::string getter = GenGetter(field.value.type); GenReceiver(struct_def, code_ptr); - code += " " + MakeCamel(field.name); + code += " " + namer_.Function(field); code += "() " + TypeName(field) + " "; - code += OffsetPrefix(field) + "\t\treturn "; + code += OffsetPrefix(field); + if (field.IsScalarOptional()) { + code += "\t\tv := "; + } else { + code += "\t\treturn "; + } code += CastToEnum(field.value.type, getter + "(o + rcv._tab.Pos)"); + if (field.IsScalarOptional()) { code += "\n\t\treturn &v"; } code += "\n\t}\n"; code += "\treturn " + GenConstant(field) + "\n"; code += "}\n\n"; @@ -369,7 +437,7 @@ class GoGenerator : public BaseGenerator { const FieldDef &field, std::string *code_ptr) { std::string &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += " " + MakeCamel(field.name); + code += " " + namer_.Function(field); code += "(obj *" + TypeName(field); code += ") *" + TypeName(field); code += " {\n"; @@ -388,7 +456,7 @@ class GoGenerator : public BaseGenerator { std::string *code_ptr) { std::string &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += " " + MakeCamel(field.name); + code += " " + namer_.Function(field); code += "(obj *"; code += TypeName(field); code += ") *" + TypeName(field) + " " + OffsetPrefix(field); @@ -410,7 +478,7 @@ class GoGenerator : public BaseGenerator { std::string *code_ptr) { std::string &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += " " + MakeCamel(field.name); + code += " " + namer_.Function(field); code += "() " + TypeName(field) + " "; code += OffsetPrefix(field) + "\t\treturn " + GenGetter(field.value.type); code += "(o + rcv._tab.Pos)\n\t}\n\treturn nil\n"; @@ -422,7 +490,7 @@ class GoGenerator : public BaseGenerator { std::string *code_ptr) { std::string &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += " " + MakeCamel(field.name) + "("; + code += " " + namer_.Function(field) + "("; code += "obj " + GenTypePointer(field.value.type) + ") bool "; code += OffsetPrefix(field); code += "\t\t" + GenGetter(field.value.type); @@ -438,7 +506,7 @@ class GoGenerator : public BaseGenerator { auto vectortype = field.value.type.VectorType(); GenReceiver(struct_def, code_ptr); - code += " " + MakeCamel(field.name); + code += " " + namer_.Function(field); code += "(obj *" + TypeName(field); code += ", j int) bool " + OffsetPrefix(field); code += "\t\tx := rcv._tab.Vector(o)\n"; @@ -453,6 +521,36 @@ class GoGenerator : public BaseGenerator { code += "}\n\n"; } + void GetMemberOfVectorOfStructByKey(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) { + std::string &code = *code_ptr; + auto vectortype = field.value.type.VectorType(); + FLATBUFFERS_ASSERT(vectortype.struct_def->has_key); + + auto &vector_struct_fields = vectortype.struct_def->fields.vec; + auto kit = + std::find_if(vector_struct_fields.begin(), vector_struct_fields.end(), + [&](FieldDef *vector_struct_field) { + return vector_struct_field->key; + }); + + auto &key_field = **kit; + FLATBUFFERS_ASSERT(key_field.key); + + GenReceiver(struct_def, code_ptr); + code += " " + namer_.Field(field) + "ByKey"; + code += "(obj *" + TypeName(field); + code += ", key " + NativeType(key_field.value.type) + ") bool " + + OffsetPrefix(field); + code += "\t\tx := rcv._tab.Vector(o)\n"; + code += "\t\treturn "; + code += "obj.LookupByKey(key, x, rcv._tab.Bytes)\n"; + code += "\t}\n"; + code += "\treturn false\n"; + code += "}\n\n"; + } + // Get the value of a vector's non-struct member. void GetMemberOfVectorOfNonStruct(const StructDef &struct_def, const FieldDef &field, @@ -461,7 +559,7 @@ class GoGenerator : public BaseGenerator { auto vectortype = field.value.type.VectorType(); GenReceiver(struct_def, code_ptr); - code += " " + MakeCamel(field.name); + code += " " + namer_.Function(field); code += "(j int) " + TypeName(field) + " "; code += OffsetPrefix(field); code += "\t\ta := rcv._tab.Vector(o)\n"; @@ -509,7 +607,7 @@ class GoGenerator : public BaseGenerator { } else { std::string &code = *code_ptr; code += std::string(", ") + nameprefix; - code += GoIdentity(field.name); + code += namer_.Variable(field); code += " " + TypeName(field); } } @@ -539,7 +637,7 @@ class GoGenerator : public BaseGenerator { } else { code += "\tbuilder.Prepend" + GenMethod(field) + "("; code += CastToBaseType(field.value.type, - nameprefix + GoIdentity(field.name)) + + nameprefix + namer_.Variable(field)) + ")\n"; } } @@ -554,7 +652,7 @@ class GoGenerator : public BaseGenerator { // Get the value of a table's starting offset. void GetStartOfTable(const StructDef &struct_def, std::string *code_ptr) { std::string &code = *code_ptr; - code += "func " + struct_def.name + "Start"; + code += "func " + namer_.Type(struct_def) + "Start"; code += "(builder *flatbuffers.Builder) {\n"; code += "\tbuilder.StartObject("; code += NumToString(struct_def.fields.vec.size()); @@ -565,35 +663,45 @@ class GoGenerator : public BaseGenerator { void BuildFieldOfTable(const StructDef &struct_def, const FieldDef &field, const size_t offset, std::string *code_ptr) { std::string &code = *code_ptr; - code += "func " + struct_def.name + "Add" + MakeCamel(field.name); + const std::string field_var = namer_.Variable(field); + code += "func " + namer_.Type(struct_def) + "Add" + namer_.Function(field); code += "(builder *flatbuffers.Builder, "; - code += GoIdentity(field.name) + " "; + code += field_var + " "; if (!IsScalar(field.value.type.base_type) && (!struct_def.fixed)) { code += "flatbuffers.UOffsetT"; } else { - code += TypeName(field); + code += GenTypeGet(field.value.type); + } + code += ") {\n\t"; + code += "builder.Prepend"; + code += GenMethod(field); + if (field.IsScalarOptional()) { + code += "("; + } else { + code += "Slot(" + NumToString(offset) + ", "; } - code += ") {\n"; - code += "\tbuilder.Prepend"; - code += GenMethod(field) + "Slot("; - code += NumToString(offset) + ", "; if (!IsScalar(field.value.type.base_type) && (!struct_def.fixed)) { code += "flatbuffers.UOffsetT"; - code += "("; - code += GoIdentity(field.name) + ")"; + code += "(" + field_var + ")"; } else { - code += CastToBaseType(field.value.type, GoIdentity(field.name)); + code += CastToBaseType(field.value.type, field_var); } - code += ", " + GenConstant(field); - code += ")\n}\n"; + if (field.IsScalarOptional()) { + code += ")\n"; + code += "\tbuilder.Slot(" + NumToString(offset); + } else { + code += ", " + GenConstant(field); + } + code += ")\n"; + code += "}\n"; } // Set the value of one of the members of a table's vector. void BuildVectorOfTable(const StructDef &struct_def, const FieldDef &field, std::string *code_ptr) { std::string &code = *code_ptr; - code += "func " + struct_def.name + "Start"; - code += MakeCamel(field.name); + code += "func " + namer_.Type(struct_def) + "Start"; + code += namer_.Function(field); code += "Vector(builder *flatbuffers.Builder, numElems int) "; code += "flatbuffers.UOffsetT {\n\treturn builder.StartVector("; auto vector_type = field.value.type.VectorType(); @@ -607,7 +715,7 @@ class GoGenerator : public BaseGenerator { // Get the offset of the end of a table. void GetEndOffsetOnTable(const StructDef &struct_def, std::string *code_ptr) { std::string &code = *code_ptr; - code += "func " + struct_def.name + "End"; + code += "func " + namer_.Type(struct_def) + "End"; code += "(builder *flatbuffers.Builder) flatbuffers.UOffsetT "; code += "{\n\treturn builder.EndObject()\n}\n"; } @@ -615,7 +723,7 @@ class GoGenerator : public BaseGenerator { // Generate the receiver for function signatures. void GenReceiver(const StructDef &struct_def, std::string *code_ptr) { std::string &code = *code_ptr; - code += "func (rcv *" + struct_def.name + ")"; + code += "func (rcv *" + namer_.Type(struct_def) + ")"; } // Generate a struct field getter, conditioned on its child type(s). @@ -644,6 +752,12 @@ class GoGenerator : public BaseGenerator { auto vectortype = field.value.type.VectorType(); if (vectortype.base_type == BASE_TYPE_STRUCT) { GetMemberOfVectorOfStruct(struct_def, field, code_ptr); + // TODO(michaeltle): Support querying fixed struct by key. + // Currently, we only support keyed tables. + if (!vectortype.struct_def->fixed && + vectortype.struct_def->has_key) { + GetMemberOfVectorOfStructByKey(struct_def, field, code_ptr); + } } else { GetMemberOfVectorOfNonStruct(struct_def, field, code_ptr); } @@ -665,11 +779,12 @@ class GoGenerator : public BaseGenerator { void MutateScalarFieldOfStruct(const StructDef &struct_def, const FieldDef &field, std::string *code_ptr) { std::string &code = *code_ptr; - std::string type = MakeCamel(GenTypeBasic(field.value.type)); - std::string setter = "rcv._tab.Mutate" + type; + std::string setter = + "rcv._tab.Mutate" + namer_.Method(GenTypeBasic(field.value.type)); GenReceiver(struct_def, code_ptr); - code += " Mutate" + MakeCamel(field.name); - code += "(n " + TypeName(field) + ") bool {\n\treturn " + setter; + code += " Mutate" + namer_.Function(field); + code += + "(n " + GenTypeGet(field.value.type) + ") bool {\n\treturn " + setter; code += "(rcv._tab.Pos+flatbuffers.UOffsetT("; code += NumToString(field.value.offset) + "), "; code += CastToBaseType(field.value.type, "n") + ")\n}\n\n"; @@ -679,11 +794,11 @@ class GoGenerator : public BaseGenerator { void MutateScalarFieldOfTable(const StructDef &struct_def, const FieldDef &field, std::string *code_ptr) { std::string &code = *code_ptr; - std::string type = MakeCamel(GenTypeBasic(field.value.type)); - std::string setter = "rcv._tab.Mutate" + type + "Slot"; + std::string setter = "rcv._tab.Mutate" + + namer_.Method(GenTypeBasic(field.value.type)) + "Slot"; GenReceiver(struct_def, code_ptr); - code += " Mutate" + MakeCamel(field.name); - code += "(n " + TypeName(field) + ") bool {\n\treturn "; + code += " Mutate" + namer_.Function(field); + code += "(n " + GenTypeGet(field.value.type) + ") bool {\n\treturn "; code += setter + "(" + NumToString(field.value.offset) + ", "; code += CastToBaseType(field.value.type, "n") + ")\n"; code += "}\n\n"; @@ -695,10 +810,10 @@ class GoGenerator : public BaseGenerator { std::string *code_ptr) { std::string &code = *code_ptr; auto vectortype = field.value.type.VectorType(); - std::string type = MakeCamel(GenTypeBasic(vectortype)); - std::string setter = "rcv._tab.Mutate" + type; + std::string setter = + "rcv._tab.Mutate" + namer_.Method(GenTypeBasic(vectortype)); GenReceiver(struct_def, code_ptr); - code += " Mutate" + MakeCamel(field.name); + code += " Mutate" + namer_.Function(field); code += "(j int, n " + TypeName(field) + ") bool "; code += OffsetPrefix(field); code += "\t\ta := rcv._tab.Vector(o)\n"; @@ -777,6 +892,12 @@ class GoGenerator : public BaseGenerator { GenStructAccessor(struct_def, field, code_ptr); GenStructMutator(struct_def, field, code_ptr); + // TODO(michaeltle): Support querying fixed struct by key. Currently, + // we only support keyed tables. + if (!struct_def.fixed && field.key) { + GenKeyCompare(struct_def, field, code_ptr); + GenLookupByKey(struct_def, field, code_ptr); + } } // Generate builders @@ -789,6 +910,79 @@ class GoGenerator : public BaseGenerator { } } + void GenKeyCompare(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr) { + FLATBUFFERS_ASSERT(struct_def.has_key); + FLATBUFFERS_ASSERT(field.key); + std::string &code = *code_ptr; + + code += "func " + namer_.Type(struct_def) + "KeyCompare("; + code += "o1, o2 flatbuffers.UOffsetT, buf []byte) bool {\n"; + code += "\tobj1 := &" + namer_.Type(struct_def) + "{}\n"; + code += "\tobj2 := &" + namer_.Type(struct_def) + "{}\n"; + code += "\tobj1.Init(buf, flatbuffers.UOffsetT(len(buf))-o1)\n"; + code += "\tobj2.Init(buf, flatbuffers.UOffsetT(len(buf))-o2)\n"; + if (IsString(field.value.type)) { + code += "\treturn string(obj1." + namer_.Function(field.name) + "()) < "; + code += "string(obj2." + namer_.Function(field.name) + "())\n"; + } else { + code += "\treturn obj1." + namer_.Function(field.name) + "() < "; + code += "obj2." + namer_.Function(field.name) + "()\n"; + } + code += "}\n\n"; + } + + void GenLookupByKey(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr) { + FLATBUFFERS_ASSERT(struct_def.has_key); + FLATBUFFERS_ASSERT(field.key); + std::string &code = *code_ptr; + + GenReceiver(struct_def, code_ptr); + code += " LookupByKey("; + code += "key " + NativeType(field.value.type) + ", "; + code += "vectorLocation flatbuffers.UOffsetT, "; + code += "buf []byte) bool {\n"; + code += "\tspan := flatbuffers.GetUOffsetT(buf[vectorLocation-4:])\n"; + code += "\tstart := flatbuffers.UOffsetT(0)\n"; + if (IsString(field.value.type)) { code += "\tbKey := []byte(key)\n"; } + code += "\tfor span != 0 {\n"; + code += "\t\tmiddle := span / 2\n"; + code += "\t\ttableOffset := flatbuffers.GetIndirectOffset(buf, "; + code += "vectorLocation+4*(start+middle))\n"; + + code += "\t\tobj := &" + namer_.Type(struct_def) + "{}\n"; + code += "\t\tobj.Init(buf, tableOffset)\n"; + + if (IsString(field.value.type)) { + needs_bytes_import_ = true; + code += + "\t\tcomp := bytes.Compare(obj." + namer_.Function(field.name) + "()"; + code += ", bKey)\n"; + } else { + code += "\t\tval := obj." + namer_.Function(field.name) + "()\n"; + code += "\t\tcomp := 0\n"; + code += "\t\tif val > key {\n"; + code += "\t\t\tcomp = 1\n"; + code += "\t\t} else if val < key {\n"; + code += "\t\t\tcomp = -1\n"; + code += "\t\t}\n"; + } + code += "\t\tif comp > 0 {\n"; + code += "\t\t\tspan = middle\n"; + code += "\t\t} else if comp < 0 {\n"; + code += "\t\t\tmiddle += 1\n"; + code += "\t\t\tstart += middle\n"; + code += "\t\t\tspan -= middle\n"; + code += "\t\t} else {\n"; + code += "\t\t\trcv.Init(buf, tableOffset)\n"; + code += "\t\t\treturn true\n"; + code += "\t\t}\n"; + code += "\t}\n"; + code += "\treturn false\n"; + code += "}\n\n"; + } + void GenNativeStruct(const StructDef &struct_def, std::string *code_ptr) { std::string &code = *code_ptr; @@ -801,8 +995,10 @@ class GoGenerator : public BaseGenerator { field.value.type.enum_def != nullptr && field.value.type.enum_def->is_union) continue; - code += "\t" + MakeCamel(field.name) + " " + - NativeType(field.value.type) + "\n"; + code += "\t" + namer_.Field(field) + " "; + if (field.IsScalarOptional()) { code += "*"; } + code += NativeType(field.value.type) + " `json:\"" + field.name + "\"`" + + "\n"; } code += "}\n\n"; @@ -818,7 +1014,7 @@ class GoGenerator : public BaseGenerator { void GenNativeUnion(const EnumDef &enum_def, std::string *code_ptr) { std::string &code = *code_ptr; code += "type " + NativeName(enum_def) + " struct {\n"; - code += "\tType " + enum_def.name + "\n"; + code += "\tType " + namer_.Type(enum_def) + "\n"; code += "\tValue interface{}\n"; code += "}\n\n"; } @@ -834,7 +1030,7 @@ class GoGenerator : public BaseGenerator { ++it2) { const EnumVal &ev = **it2; if (ev.IsZero()) continue; - code += "\tcase " + enum_def.name + ev.name + ":\n"; + code += "\tcase " + namer_.EnumVariant(enum_def, ev) + ":\n"; code += "\t\treturn t.Value.(" + NativeType(ev.union_type) + ").Pack(builder)\n"; } @@ -846,7 +1042,7 @@ class GoGenerator : public BaseGenerator { void GenNativeUnionUnPack(const EnumDef &enum_def, std::string *code_ptr) { std::string &code = *code_ptr; - code += "func (rcv " + enum_def.name + + code += "func (rcv " + namer_.Type(enum_def) + ") UnPack(table flatbuffers.Table) *" + NativeName(enum_def) + " {\n"; code += "\tswitch rcv {\n"; @@ -855,13 +1051,17 @@ class GoGenerator : public BaseGenerator { ++it2) { const EnumVal &ev = **it2; if (ev.IsZero()) continue; - code += "\tcase " + enum_def.name + ev.name + ":\n"; - code += "\t\tx := " + ev.union_type.struct_def->name + "{_tab: table}\n"; + code += "\tcase " + namer_.EnumVariant(enum_def, ev) + ":\n"; + code += "\t\tvar x " + + WrapInNameSpaceAndTrack(ev.union_type.struct_def, + ev.union_type.struct_def->name) + + "\n"; + code += "\t\tx.Init(table.Bytes, table.Pos)\n"; code += "\t\treturn &" + - WrapInNameSpaceAndTrack(enum_def.defined_namespace, - NativeName(enum_def)) + - "{ Type: " + enum_def.name + ev.name + ", Value: x.UnPack() }\n"; + WrapInNameSpaceAndTrack(&enum_def, NativeName(enum_def)) + + "{Type: " + namer_.EnumVariant(enum_def, ev) + + ", Value: x.UnPack()}\n"; } code += "\t}\n"; code += "\treturn nil\n"; @@ -870,63 +1070,70 @@ class GoGenerator : public BaseGenerator { void GenNativeTablePack(const StructDef &struct_def, std::string *code_ptr) { std::string &code = *code_ptr; + const std::string struct_type = namer_.Type(struct_def); code += "func (t *" + NativeName(struct_def) + ") Pack(builder *flatbuffers.Builder) flatbuffers.UOffsetT {\n"; - code += "\tif t == nil { return 0 }\n"; + code += "\tif t == nil {\n\t\treturn 0\n\t}\n"; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { const FieldDef &field = **it; if (field.deprecated) continue; if (IsScalar(field.value.type.base_type)) continue; - std::string offset = MakeCamel(field.name, false) + "Offset"; + const std::string field_field = namer_.Field(field); + const std::string field_var = namer_.Variable(field); + const std::string offset = field_var + "Offset"; if (IsString(field.value.type)) { - code += "\t" + offset + " := builder.CreateString(t." + - MakeCamel(field.name) + ")\n"; + code += "\t" + offset + " := flatbuffers.UOffsetT(0)\n"; + code += "\tif t." + field_field + " != \"\" {\n"; + code += "\t\t" + offset + " = builder.CreateString(t." + field_field + + ")\n"; + code += "\t}\n"; } else if (IsVector(field.value.type) && field.value.type.element == BASE_TYPE_UCHAR && field.value.type.enum_def == nullptr) { code += "\t" + offset + " := flatbuffers.UOffsetT(0)\n"; - code += "\tif t." + MakeCamel(field.name) + " != nil {\n"; + code += "\tif t." + field_field + " != nil {\n"; code += "\t\t" + offset + " = builder.CreateByteString(t." + - MakeCamel(field.name) + ")\n"; + field_field + ")\n"; code += "\t}\n"; } else if (IsVector(field.value.type)) { code += "\t" + offset + " := flatbuffers.UOffsetT(0)\n"; - code += "\tif t." + MakeCamel(field.name) + " != nil {\n"; - std::string length = MakeCamel(field.name, false) + "Length"; - std::string offsets = MakeCamel(field.name, false) + "Offsets"; - code += "\t\t" + length + " := len(t." + MakeCamel(field.name) + ")\n"; + code += "\tif t." + field_field + " != nil {\n"; + std::string length = field_var + "Length"; + std::string offsets = field_var + "Offsets"; + code += "\t\t" + length + " := len(t." + field_field + ")\n"; if (field.value.type.element == BASE_TYPE_STRING) { code += "\t\t" + offsets + " := make([]flatbuffers.UOffsetT, " + length + ")\n"; code += "\t\tfor j := 0; j < " + length + "; j++ {\n"; code += "\t\t\t" + offsets + "[j] = builder.CreateString(t." + - MakeCamel(field.name) + "[j])\n"; + field_field + "[j])\n"; code += "\t\t}\n"; } else if (field.value.type.element == BASE_TYPE_STRUCT && !field.value.type.struct_def->fixed) { code += "\t\t" + offsets + " := make([]flatbuffers.UOffsetT, " + length + ")\n"; code += "\t\tfor j := 0; j < " + length + "; j++ {\n"; - code += "\t\t\t" + offsets + "[j] = t." + MakeCamel(field.name) + + code += "\t\t\t" + offsets + "[j] = t." + field_field + "[j].Pack(builder)\n"; code += "\t\t}\n"; } - code += "\t\t" + struct_def.name + "Start" + MakeCamel(field.name) + + code += "\t\t" + struct_type + "Start" + namer_.Function(field) + "Vector(builder, " + length + ")\n"; code += "\t\tfor j := " + length + " - 1; j >= 0; j-- {\n"; if (IsScalar(field.value.type.element)) { code += "\t\t\tbuilder.Prepend" + - MakeCamel(GenTypeBasic(field.value.type.VectorType())) + "(" + + namer_.Method(GenTypeBasic(field.value.type.VectorType())) + + "(" + CastToBaseType(field.value.type.VectorType(), - "t." + MakeCamel(field.name) + "[j]") + + "t." + field_field + "[j]") + ")\n"; } else if (field.value.type.element == BASE_TYPE_STRUCT && field.value.type.struct_def->fixed) { - code += "\t\t\tt." + MakeCamel(field.name) + "[j].Pack(builder)\n"; + code += "\t\t\tt." + field_field + "[j].Pack(builder)\n"; } else { code += "\t\t\tbuilder.PrependUOffsetT(" + offsets + "[j])\n"; } @@ -935,92 +1142,96 @@ class GoGenerator : public BaseGenerator { code += "\t}\n"; } else if (field.value.type.base_type == BASE_TYPE_STRUCT) { if (field.value.type.struct_def->fixed) continue; - code += "\t" + offset + " := t." + MakeCamel(field.name) + - ".Pack(builder)\n"; + code += "\t" + offset + " := t." + field_field + ".Pack(builder)\n"; } else if (field.value.type.base_type == BASE_TYPE_UNION) { - code += "\t" + offset + " := t." + MakeCamel(field.name) + - ".Pack(builder)\n"; - code += "\t\n"; + code += "\t" + offset + " := t." + field_field + ".Pack(builder)\n\n"; } else { FLATBUFFERS_ASSERT(0); } } - code += "\t" + struct_def.name + "Start(builder)\n"; + code += "\t" + struct_type + "Start(builder)\n"; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { const FieldDef &field = **it; if (field.deprecated) continue; + const std::string field_field = namer_.Field(field); + const std::string field_fn = namer_.Function(field); + const std::string offset = namer_.Variable(field) + "Offset"; - std::string offset = MakeCamel(field.name, false) + "Offset"; if (IsScalar(field.value.type.base_type)) { + std::string prefix; + if (field.IsScalarOptional()) { + code += "\tif t." + field_field + " != nil {\n\t"; + prefix = "*"; + } if (field.value.type.enum_def == nullptr || !field.value.type.enum_def->is_union) { - code += "\t" + struct_def.name + "Add" + MakeCamel(field.name) + - "(builder, t." + MakeCamel(field.name) + ")\n"; + code += "\t" + struct_type + "Add" + field_fn + "(builder, " + + prefix + "t." + field_field + ")\n"; } + if (field.IsScalarOptional()) { code += "\t}\n"; } } else { if (field.value.type.base_type == BASE_TYPE_STRUCT && field.value.type.struct_def->fixed) { - code += "\t" + offset + " := t." + MakeCamel(field.name) + - ".Pack(builder)\n"; + code += "\t" + offset + " := t." + field_field + ".Pack(builder)\n"; } else if (field.value.type.enum_def != nullptr && field.value.type.enum_def->is_union) { - code += "\tif t." + MakeCamel(field.name) + " != nil {\n"; - code += "\t\t" + struct_def.name + "Add" + - MakeCamel(field.name + UnionTypeFieldSuffix()) + - "(builder, t." + MakeCamel(field.name) + ".Type)\n"; + code += "\tif t." + field_field + " != nil {\n"; + code += "\t\t" + struct_type + "Add" + + namer_.Method(field.name + UnionTypeFieldSuffix()) + + "(builder, t." + field_field + ".Type)\n"; code += "\t}\n"; } - code += "\t" + struct_def.name + "Add" + MakeCamel(field.name) + - "(builder, " + offset + ")\n"; + code += "\t" + struct_type + "Add" + field_fn + "(builder, " + offset + + ")\n"; } } - code += "\treturn " + struct_def.name + "End(builder)\n"; + code += "\treturn " + struct_type + "End(builder)\n"; code += "}\n\n"; } void GenNativeTableUnPack(const StructDef &struct_def, std::string *code_ptr) { std::string &code = *code_ptr; + const std::string struct_type = namer_.Type(struct_def); - code += "func (rcv *" + struct_def.name + ") UnPackTo(t *" + + code += "func (rcv *" + struct_type + ") UnPackTo(t *" + NativeName(struct_def) + ") {\n"; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { const FieldDef &field = **it; if (field.deprecated) continue; - std::string field_name_camel = MakeCamel(field.name); - std::string length = MakeCamel(field.name, false) + "Length"; + const std::string field_field = namer_.Field(field); + const std::string field_var = namer_.Variable(field); + const std::string length = field_var + "Length"; if (IsScalar(field.value.type.base_type)) { if (field.value.type.enum_def != nullptr && field.value.type.enum_def->is_union) continue; - code += - "\tt." + field_name_camel + " = rcv." + field_name_camel + "()\n"; + code += "\tt." + field_field + " = rcv." + field_field + "()\n"; } else if (IsString(field.value.type)) { - code += "\tt." + field_name_camel + " = string(rcv." + - field_name_camel + "())\n"; + code += "\tt." + field_field + " = string(rcv." + field_field + "())\n"; } else if (IsVector(field.value.type) && field.value.type.element == BASE_TYPE_UCHAR && field.value.type.enum_def == nullptr) { - code += "\tt." + field_name_camel + " = rcv." + field_name_camel + - "Bytes()\n"; + code += "\tt." + field_field + " = rcv." + field_field + "Bytes()\n"; } else if (IsVector(field.value.type)) { - code += "\t" + length + " := rcv." + field_name_camel + "Length()\n"; - code += "\tt." + field_name_camel + " = make(" + + code += "\t" + length + " := rcv." + field_field + "Length()\n"; + code += "\tt." + field_field + " = make(" + NativeType(field.value.type) + ", " + length + ")\n"; code += "\tfor j := 0; j < " + length + "; j++ {\n"; if (field.value.type.element == BASE_TYPE_STRUCT) { code += "\t\tx := " + - WrapInNameSpaceAndTrack(*field.value.type.struct_def) + + WrapInNameSpaceAndTrack(field.value.type.struct_def, + field.value.type.struct_def->name) + "{}\n"; - code += "\t\trcv." + field_name_camel + "(&x, j)\n"; + code += "\t\trcv." + field_field + "(&x, j)\n"; } - code += "\t\tt." + field_name_camel + "[j] = "; + code += "\t\tt." + field_field + "[j] = "; if (IsScalar(field.value.type.element)) { - code += "rcv." + field_name_camel + "(j)"; + code += "rcv." + field_field + "(j)"; } else if (field.value.type.element == BASE_TYPE_STRING) { - code += "string(rcv." + field_name_camel + "(j))"; + code += "string(rcv." + field_field + "(j))"; } else if (field.value.type.element == BASE_TYPE_STRUCT) { code += "x.UnPack()"; } else { @@ -1030,16 +1241,16 @@ class GoGenerator : public BaseGenerator { code += "\n"; code += "\t}\n"; } else if (field.value.type.base_type == BASE_TYPE_STRUCT) { - code += "\tt." + field_name_camel + " = rcv." + field_name_camel + - "(nil).UnPack()\n"; + code += + "\tt." + field_field + " = rcv." + field_field + "(nil).UnPack()\n"; } else if (field.value.type.base_type == BASE_TYPE_UNION) { - std::string field_table = MakeCamel(field.name, false) + "Table"; + const std::string field_table = field_var + "Table"; code += "\t" + field_table + " := flatbuffers.Table{}\n"; code += - "\tif rcv." + MakeCamel(field.name) + "(&" + field_table + ") {\n"; - code += "\t\tt." + field_name_camel + " = rcv." + - MakeCamel(field.name + UnionTypeFieldSuffix()) + "().UnPack(" + - field_table + ")\n"; + "\tif rcv." + namer_.Method(field) + "(&" + field_table + ") {\n"; + code += "\t\tt." + field_field + " = rcv." + + namer_.Method(field.name + UnionTypeFieldSuffix()) + + "().UnPack(" + field_table + ")\n"; code += "\t}\n"; } else { FLATBUFFERS_ASSERT(0); @@ -1047,9 +1258,9 @@ class GoGenerator : public BaseGenerator { } code += "}\n\n"; - code += "func (rcv *" + struct_def.name + ") UnPack() *" + + code += "func (rcv *" + struct_type + ") UnPack() *" + NativeName(struct_def) + " {\n"; - code += "\tif rcv == nil { return nil }\n"; + code += "\tif rcv == nil {\n\t\treturn nil\n\t}\n"; code += "\tt := &" + NativeName(struct_def) + "{}\n"; code += "\trcv.UnPackTo(t)\n"; code += "\treturn t\n"; @@ -1061,8 +1272,8 @@ class GoGenerator : public BaseGenerator { code += "func (t *" + NativeName(struct_def) + ") Pack(builder *flatbuffers.Builder) flatbuffers.UOffsetT {\n"; - code += "\tif t == nil { return 0 }\n"; - code += "\treturn Create" + struct_def.name + "(builder"; + code += "\tif t == nil {\n\t\treturn 0\n\t}\n"; + code += "\treturn Create" + namer_.Type(struct_def) + "(builder"; StructPackArgs(struct_def, "", code_ptr); code += ")\n"; code += "}\n"; @@ -1076,10 +1287,10 @@ class GoGenerator : public BaseGenerator { const FieldDef &field = **it; if (field.value.type.base_type == BASE_TYPE_STRUCT) { StructPackArgs(*field.value.type.struct_def, - (nameprefix + MakeCamel(field.name) + ".").c_str(), + (nameprefix + namer_.Field(field) + ".").c_str(), code_ptr); } else { - code += std::string(", t.") + nameprefix + MakeCamel(field.name); + code += std::string(", t.") + nameprefix + namer_.Field(field); } } } @@ -1088,24 +1299,24 @@ class GoGenerator : public BaseGenerator { std::string *code_ptr) { std::string &code = *code_ptr; - code += "func (rcv *" + struct_def.name + ") UnPackTo(t *" + + code += "func (rcv *" + namer_.Type(struct_def) + ") UnPackTo(t *" + NativeName(struct_def) + ") {\n"; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { const FieldDef &field = **it; if (field.value.type.base_type == BASE_TYPE_STRUCT) { - code += "\tt." + MakeCamel(field.name) + " = rcv." + - MakeCamel(field.name) + "(nil).UnPack()\n"; + code += "\tt." + namer_.Field(field) + " = rcv." + + namer_.Method(field) + "(nil).UnPack()\n"; } else { - code += "\tt." + MakeCamel(field.name) + " = rcv." + - MakeCamel(field.name) + "()\n"; + code += "\tt." + namer_.Field(field) + " = rcv." + + namer_.Method(field) + "()\n"; } } code += "}\n\n"; - code += "func (rcv *" + struct_def.name + ") UnPack() *" + + code += "func (rcv *" + namer_.Type(struct_def) + ") UnPack() *" + NativeName(struct_def) + " {\n"; - code += "\tif rcv == nil { return nil }\n"; + code += "\tif rcv == nil {\n\t\treturn nil\n\t}\n"; code += "\tt := &" + NativeName(struct_def) + "{}\n"; code += "\trcv.UnPackTo(t)\n"; code += "\treturn t\n"; @@ -1152,14 +1363,14 @@ class GoGenerator : public BaseGenerator { case BASE_TYPE_STRING: return "rcv._tab.ByteVector"; case BASE_TYPE_UNION: return "rcv._tab.Union"; case BASE_TYPE_VECTOR: return GenGetter(type.VectorType()); - default: return "rcv._tab.Get" + MakeCamel(GenTypeBasic(type)); + default: return "rcv._tab.Get" + namer_.Function(GenTypeBasic(type)); } } // Returns the method name for use with add/put calls. std::string GenMethod(const FieldDef &field) { return IsScalar(field.value.type.base_type) - ? MakeCamel(GenTypeBasic(field.value.type)) + ? namer_.Method(GenTypeBasic(field.value.type)) : (IsStruct(field.value.type) ? "Struct" : "UOffsetT"); } @@ -1179,7 +1390,8 @@ class GoGenerator : public BaseGenerator { switch (type.base_type) { case BASE_TYPE_STRING: return "[]byte"; case BASE_TYPE_VECTOR: return GenTypeGet(type.VectorType()); - case BASE_TYPE_STRUCT: return WrapInNameSpaceAndTrack(*type.struct_def); + case BASE_TYPE_STRUCT: + return WrapInNameSpaceAndTrack(type.struct_def, type.struct_def->name); case BASE_TYPE_UNION: // fall through default: return "*flatbuffers.Table"; @@ -1192,7 +1404,9 @@ class GoGenerator : public BaseGenerator { } std::string TypeName(const FieldDef &field) { - return GenTypeGet(field.value.type); + std::string prefix; + if (field.IsScalarOptional()) { prefix = "*"; } + return prefix + GenTypeGet(field.value.type); } // If type is an enum, returns value with a cast to the enum type, otherwise @@ -1216,21 +1430,37 @@ class GoGenerator : public BaseGenerator { } std::string GenConstant(const FieldDef &field) { + if (field.IsScalarOptional()) { return "nil"; } switch (field.value.type.base_type) { case BASE_TYPE_BOOL: return field.value.constant == "0" ? "false" : "true"; + case BASE_TYPE_FLOAT: + case BASE_TYPE_DOUBLE: { + const std::string float_type = + field.value.type.base_type == BASE_TYPE_FLOAT ? "float32" + : "float64"; + if (StringIsFlatbufferNan(field.value.constant)) { + needs_math_import_ = true; + return float_type + "(math.NaN())"; + } else if (StringIsFlatbufferPositiveInfinity(field.value.constant)) { + needs_math_import_ = true; + return float_type + "(math.Inf(1))"; + } else if (StringIsFlatbufferNegativeInfinity(field.value.constant)) { + needs_math_import_ = true; + return float_type + "(math.Inf(-1))"; + } + return field.value.constant; + } default: return field.value.constant; } } - std::string NativeName(const StructDef &struct_def) { - return parser_.opts.object_prefix + struct_def.name + - parser_.opts.object_suffix; + std::string NativeName(const StructDef &struct_def) const { + return namer_.ObjectType(struct_def); } - std::string NativeName(const EnumDef &enum_def) { - return parser_.opts.object_prefix + enum_def.name + - parser_.opts.object_suffix; + std::string NativeName(const EnumDef &enum_def) const { + return namer_.ObjectType(enum_def); } std::string NativeType(const Type &type) { @@ -1245,11 +1475,11 @@ class GoGenerator : public BaseGenerator { } else if (IsVector(type)) { return "[]" + NativeType(type.VectorType()); } else if (type.base_type == BASE_TYPE_STRUCT) { - return "*" + WrapInNameSpaceAndTrack(type.struct_def->defined_namespace, + return "*" + WrapInNameSpaceAndTrack(type.struct_def, NativeName(*type.struct_def)); } else if (type.base_type == BASE_TYPE_UNION) { - return "*" + WrapInNameSpaceAndTrack(type.enum_def->defined_namespace, - NativeName(*type.enum_def)); + return "*" + + WrapInNameSpaceAndTrack(type.enum_def, NativeName(*type.enum_def)); } FLATBUFFERS_ASSERT(0); return std::string(); @@ -1264,6 +1494,7 @@ class GoGenerator : public BaseGenerator { StructBuilderBody(struct_def, "", code_ptr); EndBuilderBody(code_ptr); } + // Begin by declaring namespace and imports. void BeginFile(const std::string &name_space_name, const bool needs_imports, const bool is_enum, std::string *code_ptr) { @@ -1273,26 +1504,47 @@ class GoGenerator : public BaseGenerator { code += "package " + name_space_name + "\n\n"; if (needs_imports) { code += "import (\n"; - if (is_enum) { code += "\t\"strconv\"\n\n"; } + // standard imports, in alphabetical order for go fmt + if (needs_bytes_import_) code += "\t\"bytes\"\n"; if (!parser_.opts.go_import.empty()) { code += "\tflatbuffers \"" + parser_.opts.go_import + "\"\n"; } else { code += "\tflatbuffers \"github.com/google/flatbuffers/go\"\n"; } + // math is needed to support non-finite scalar default values. + if (needs_math_import_) { code += "\t\"math\"\n"; } + if (is_enum) { code += "\t\"strconv\"\n"; } + if (tracked_imported_namespaces_.size() > 0) { code += "\n"; for (auto it = tracked_imported_namespaces_.begin(); it != tracked_imported_namespaces_.end(); ++it) { - code += "\t" + NamespaceImportName(*it) + " \"" + - NamespaceImportPath(*it) + "\"\n"; + if ((*it)->defined_namespace->components.empty()) { + code += "\t" + (*it)->name + " \"" + (*it)->name + "\"\n"; + } else { + code += "\t" + NamespaceImportName((*it)->defined_namespace) + + " \"" + NamespaceImportPath((*it)->defined_namespace) + + "\"\n"; + } } } code += ")\n\n"; } else { if (is_enum) { code += "import \"strconv\"\n\n"; } + if (needs_math_import_) { + // math is needed to support non-finite scalar default values. + code += "import \"math\"\n\n"; + } } } + // Resets the needed imports before generating a new file. + void ResetImports() { + tracked_imported_namespaces_.clear(); + needs_bytes_import_ = false; + needs_math_import_ = false; + } + // Save out the generated code for a Go Table type. bool SaveType(const Definition &def, const std::string &classcode, const bool needs_imports, const bool is_enum) { @@ -1301,56 +1553,45 @@ class GoGenerator : public BaseGenerator { Namespace &ns = go_namespace_.components.empty() ? *def.defined_namespace : go_namespace_; std::string code = ""; - BeginFile(LastNamespacePart(ns), needs_imports, is_enum, &code); + BeginFile(ns.components.empty() ? def.name : LastNamespacePart(ns), + needs_imports, is_enum, &code); code += classcode; // Strip extra newlines at end of file to make it gofmt-clean. while (code.length() > 2 && code.substr(code.length() - 2) == "\n\n") { code.pop_back(); } - std::string filename = NamespaceDir(ns) + def.name + ".go"; + std::string directory = namer_.Directories(ns); + std::string file = namer_.File(def, SkipFile::Suffix); + EnsureDirExists(directory); + std::string filename = directory + file; return SaveFile(filename.c_str(), code, false); } // Create the full name of the imported namespace (format: A__B__C). - std::string NamespaceImportName(const Namespace *ns) { - std::string s = ""; - for (auto it = ns->components.begin(); it != ns->components.end(); ++it) { - if (s.size() == 0) { - s += *it; - } else { - s += "__" + *it; - } - } - return s; + std::string NamespaceImportName(const Namespace *ns) const { + return namer_.Namespace(*ns); } // Create the full path for the imported namespace (format: A/B/C). - std::string NamespaceImportPath(const Namespace *ns) { - std::string s = ""; - for (auto it = ns->components.begin(); it != ns->components.end(); ++it) { - if (s.size() == 0) { - s += *it; - } else { - s += "/" + *it; - } + std::string NamespaceImportPath(const Namespace *ns) const { + std::string path = + namer_.Directories(*ns, SkipDir::OutputPathAndTrailingPathSeparator); + if (!parser_.opts.go_module_name.empty()) { + path = parser_.opts.go_module_name + "/" + path; } - return s; + return path; } // Ensure that a type is prefixed with its go package import name if it is // used outside of its namespace. - std::string WrapInNameSpaceAndTrack(const Namespace *ns, + std::string WrapInNameSpaceAndTrack(const Definition *def, const std::string &name) { - if (CurrentNameSpace() == ns) return name; - - tracked_imported_namespaces_.insert(ns); - - std::string import_name = NamespaceImportName(ns); - return import_name + "." + name; - } - - std::string WrapInNameSpaceAndTrack(const Definition &def) { - return WrapInNameSpaceAndTrack(def.defined_namespace, def.name); + if (CurrentNameSpace() == def->defined_namespace) return name; + tracked_imported_namespaces_.insert(def); + if (def->defined_namespace->components.empty()) + return def->name + "." + name; + else + return NamespaceImportName(def->defined_namespace) + "." + name; } const Namespace *CurrentNameSpace() const { return cur_name_space_; } @@ -1371,4 +1612,59 @@ bool GenerateGo(const Parser &parser, const std::string &path, return generator.generate(); } +namespace { + +class GoCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateGo(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + (void)parser; + (void)path; + (void)filename; + (void)output; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateGoGRPC(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kGo; } + + std::string LanguageName() const override { return "Go"; } +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewGoCodeGenerator() { + return std::unique_ptr<GoCodeGenerator>(new GoCodeGenerator()); +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_go.h b/contrib/libs/flatbuffers/src/idl_gen_go.h new file mode 100644 index 00000000000..d81c1ff650d --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_go.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_GO_H_ +#define FLATBUFFERS_IDL_GEN_GO_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Go code generator. +std::unique_ptr<CodeGenerator> NewGoCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_GO_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_grpc.cpp b/contrib/libs/flatbuffers/src/idl_gen_grpc.cpp index 9aea745d4e8..2be24e71a27 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_grpc.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_grpc.cpp @@ -191,7 +191,7 @@ class FlatBufPrinter : public grpc_generator::Printer { } } - void SetIndentationSize(const int size) { + void SetIndentationSize(const size_t size) { FLATBUFFERS_ASSERT(str_->empty()); indentation_size_ = size; } @@ -199,15 +199,15 @@ class FlatBufPrinter : public grpc_generator::Printer { void Indent() { indent_++; } void Outdent() { + FLATBUFFERS_ASSERT(indent_ > 0); indent_--; - FLATBUFFERS_ASSERT(indent_ >= 0); } private: std::string *str_; char escape_char_; - int indent_; - int indentation_size_; + size_t indent_; + size_t indentation_size_; char indentation_type_; }; @@ -242,10 +242,6 @@ class FlatBufFile : public grpc_generator::File { return StripExtension(file_name_); } - std::string message_header_ext() const { return "_generated.h"; } - - std::string service_header_ext() const { return ".grpc.fb.h"; } - std::string package() const { return parser_.current_namespace_->GetFullyQualifiedName(""); } @@ -345,6 +341,7 @@ bool GenerateGoGRPC(const Parser &parser, const std::string &path, bool GenerateCppGRPC(const Parser &parser, const std::string &path, const std::string &file_name) { + const auto &opts = parser.opts; int nservices = 0; for (auto it = parser.services_.vec.begin(); it != parser.services_.vec.end(); ++it) { @@ -352,9 +349,15 @@ bool GenerateCppGRPC(const Parser &parser, const std::string &path, } if (!nservices) return true; + std::string suffix = ""; + suffix += opts.filename_suffix.empty() ? "_generated" : opts.filename_suffix; + suffix += "."; + suffix += opts.filename_extension.empty() ? "h" : opts.filename_extension; + grpc_cpp_generator::Parameters generator_parameters; // TODO(wvo): make the other parameters in this struct configurable. generator_parameters.use_system_headers = true; + generator_parameters.message_header_extension = suffix; FlatBufFile fbfile(parser, file_name, FlatBufFile::kLanguageCpp); diff --git a/contrib/libs/flatbuffers/src/idl_gen_java.cpp b/contrib/libs/flatbuffers/src/idl_gen_java.cpp index cfd3a55cdb8..34895ee9dcd 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_java.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_java.cpp @@ -16,34 +16,94 @@ // independent from idl_parser, since this code is not needed for most clients +#include "idl_gen_java.h" + #include "flatbuffers/code_generators.h" #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" - -#if defined(FLATBUFFERS_CPP98_STL) -# include <cctype> -#endif // defined(FLATBUFFERS_CPP98_STL) +#include "idl_namer.h" namespace flatbuffers { namespace java { -static TypedFloatConstantGenerator JavaFloatGen("Double.", "Float.", "NaN", +namespace { + +static Namer::Config JavaDefaultConfig() { + return { + /*types=*/Case::kKeep, + /*constants=*/Case::kScreamingSnake, + /*methods=*/Case::kLowerCamel, + /*functions=*/Case::kLowerCamel, + /*fields=*/Case::kLowerCamel, + /*variables=*/Case::kLowerCamel, + /*variants=*/Case::kKeep, + /*enum_variant_seperator=*/".", + /*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase, + /*namespaces=*/Case::kKeep, + /*namespace_seperator=*/".", + /*object_prefix=*/"", + /*object_suffix=*/"T", + /*keyword_prefix=*/"", + /*keyword_suffix=*/"_", + /*filenames=*/Case::kKeep, + /*directories=*/Case::kKeep, + /*output_path=*/"", + /*filename_suffix=*/"_generated", + /*filename_extension=*/".java", + }; +} + +static std::set<std::string> JavaKeywords() { + return { + "abstract", "continue", "for", "new", "switch", + "assert", "default", "goto", "package", "synchronized", + "boolean", "do", "if", "private", "this", + "break", "double", "implements", "protected", "throw", + "byte", "else", "import", "public", "throws", + "case", "enum", "instanceof", "return", "transient", + "catch", "extends", "int", "short", "try", + "char", "final", "interface", "static", "void", + "class", "finally", "long", "strictfp", "volatile", + "const", "float", "native", "super", "while", + }; +} + +static const TypedFloatConstantGenerator JavaFloatGen("Double.", "Float.", "NaN", "POSITIVE_INFINITY", "NEGATIVE_INFINITY"); -static CommentConfig comment_config = { +static const CommentConfig comment_config = { "/**", " *", " */", }; +} // namespace + class JavaGenerator : public BaseGenerator { + struct FieldArrayLength { + std::string name; + int length; + }; + public: JavaGenerator(const Parser &parser, const std::string &path, - const std::string &file_name) + const std::string &file_name, + const std::string &package_prefix) : BaseGenerator(parser, path, file_name, "", ".", "java"), - cur_name_space_(nullptr) {} + cur_name_space_(nullptr), + namer_(WithFlagOptions(JavaDefaultConfig(), parser.opts, path), + JavaKeywords()) { + if (!package_prefix.empty()) { + std::istringstream iss(package_prefix); + std::string component; + while(std::getline(iss, component, '.')) { + package_prefix_ns_.components.push_back(component); + } + package_prefix_ = package_prefix_ns_.GetFullyQualifiedName("") + "."; + } + } JavaGenerator &operator=(const JavaGenerator &); bool generate() { @@ -55,7 +115,7 @@ class JavaGenerator : public BaseGenerator { std::string enumcode; auto &enum_def = **it; if (!parser_.opts.one_file) cur_name_space_ = enum_def.defined_namespace; - GenEnum(enum_def, &enumcode); + GenEnum(enum_def, enumcode); if (parser_.opts.one_file) { one_file_code += enumcode; } else { @@ -63,6 +123,19 @@ class JavaGenerator : public BaseGenerator { /* needs_includes= */ false)) return false; } + + if (parser_.opts.generate_object_based_api && enum_def.is_union) { + enumcode = ""; + GenEnum_ObjectAPI(enum_def, enumcode); + auto class_name = namer_.Type(enum_def) + "Union"; + if (parser_.opts.one_file) { + one_file_code += enumcode; + } else { + if (!SaveType(class_name, *enum_def.defined_namespace, enumcode, + /* needs_includes= */ false)) + return false; + } + } } for (auto it = parser_.structs_.vec.begin(); @@ -71,7 +144,7 @@ class JavaGenerator : public BaseGenerator { auto &struct_def = **it; if (!parser_.opts.one_file) cur_name_space_ = struct_def.defined_namespace; - GenStruct(struct_def, &declcode); + GenStruct(struct_def, declcode, parser_.opts); if (parser_.opts.one_file) { one_file_code += declcode; } else { @@ -79,6 +152,19 @@ class JavaGenerator : public BaseGenerator { /* needs_includes= */ true)) return false; } + + if (parser_.opts.generate_object_based_api) { + declcode = ""; + GenStruct_ObjectAPI(struct_def, declcode); + auto class_name = namer_.ObjectType(struct_def); + if (parser_.opts.one_file) { + one_file_code += declcode; + } else { + if (!SaveType(class_name, *struct_def.defined_namespace, declcode, + /* needs_includes= */ true)) + return false; + } + } } if (parser_.opts.one_file) { @@ -97,15 +183,35 @@ class JavaGenerator : public BaseGenerator { std::string code; code = "// " + std::string(FlatBuffersGeneratedWarning()) + "\n\n"; - std::string namespace_name = FullNamespace(".", ns); + Namespace combined_ns = package_prefix_ns_; + std::copy( + ns.components.begin(), + ns.components.end(), + std::back_inserter(combined_ns.components)); + + const std::string namespace_name = FullNamespace(".", combined_ns); if (!namespace_name.empty()) { code += "package " + namespace_name + ";"; code += "\n\n"; } if (needs_includes) { code += - "import java.nio.*;\nimport java.lang.*;\nimport " - "java.util.*;\nimport com.google.flatbuffers.*;\n"; + "import com.google.flatbuffers.BaseVector;\n" + "import com.google.flatbuffers.BooleanVector;\n" + "import com.google.flatbuffers.ByteVector;\n" + "import com.google.flatbuffers.Constants;\n" + "import com.google.flatbuffers.DoubleVector;\n" + "import com.google.flatbuffers.FlatBufferBuilder;\n" + "import com.google.flatbuffers.FloatVector;\n" + "import com.google.flatbuffers.IntVector;\n" + "import com.google.flatbuffers.LongVector;\n" + "import com.google.flatbuffers.ShortVector;\n" + "import com.google.flatbuffers.StringVector;\n" + "import com.google.flatbuffers.Struct;\n" + "import com.google.flatbuffers.Table;\n" + "import com.google.flatbuffers.UnionVector;\n" + "import java.nio.ByteBuffer;\n" + "import java.nio.ByteOrder;\n"; if (parser_.opts.gen_nullable) { code += "\nimport javax.annotation.Nullable;\n"; } @@ -117,7 +223,10 @@ class JavaGenerator : public BaseGenerator { code += classcode; if (!namespace_name.empty()) code += ""; - auto filename = NamespaceDir(ns) + defname + ".java"; + const std::string dirs = namer_.Directories(combined_ns); + EnsureDirExists(dirs); + const std::string filename = + dirs + namer_.File(defname, /*skips=*/SkipFile::Suffix); return SaveFile(filename.c_str(), code, false); } @@ -154,7 +263,8 @@ class JavaGenerator : public BaseGenerator { switch (type.base_type) { case BASE_TYPE_STRING: return "String"; case BASE_TYPE_VECTOR: return GenTypeGet(type.VectorType()); - case BASE_TYPE_STRUCT: return WrapInNameSpace(*type.struct_def); + case BASE_TYPE_STRUCT: + return Prefixed(namer_.NamespacedType(*type.struct_def)); case BASE_TYPE_UNION: FLATBUFFERS_FALLTHROUGH(); // else fall thru default: return "Table"; } @@ -231,11 +341,11 @@ class JavaGenerator : public BaseGenerator { } else { if (castFromDest) { if (type.base_type == BASE_TYPE_UINT) - return "(int)"; + return "(int) "; else if (type.base_type == BASE_TYPE_USHORT) - return "(short)"; + return "(short) "; else if (type.base_type == BASE_TYPE_UCHAR) - return "(byte)"; + return "(byte) "; } } return ""; @@ -258,8 +368,9 @@ class JavaGenerator : public BaseGenerator { FLATBUFFERS_ASSERT(value.type.enum_def); auto &enum_def = *value.type.enum_def; auto enum_val = enum_def.FindByValue(value.constant); - return enum_val ? (WrapInNameSpace(enum_def) + "." + enum_val->name) - : value.constant; + return + enum_val ? Prefixed(namer_.NamespacedEnumVariant(enum_def, *enum_val)) + : value.constant; } std::string GenDefaultValue(const FieldDef &field) const { @@ -293,8 +404,7 @@ class JavaGenerator : public BaseGenerator { return GenDefaultValue(field); } - void GenEnum(EnumDef &enum_def, std::string *code_ptr) const { - std::string &code = *code_ptr; + void GenEnum(EnumDef &enum_def, std::string &code) const { if (enum_def.generated) return; // Generate enum definitions of the form: @@ -302,28 +412,33 @@ class JavaGenerator : public BaseGenerator { // In Java, we use ints rather than the Enum feature, because we want them // to map directly to how they're used in C/C++ and file formats. // That, and Java Enums are expensive, and not universally liked. - GenComment(enum_def.doc_comment, code_ptr, &comment_config); + GenComment(enum_def.doc_comment, &code, &comment_config); + code += "@SuppressWarnings(\"unused\")\n"; if (enum_def.attributes.Lookup("private")) { // For Java, we leave the enum unmarked to indicate package-private } else { code += "public "; } - code += "final class " + enum_def.name; + code += "final class " + namer_.Type(enum_def); code += " {\n"; - code += " private " + enum_def.name + "() { }\n"; + code += " private " + namer_.Type(enum_def) + "() { }\n"; for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { auto &ev = **it; - GenComment(ev.doc_comment, code_ptr, &comment_config, " "); + GenComment(ev.doc_comment, &code, &comment_config, " "); code += " public static final "; code += GenTypeBasic(DestinationType(enum_def.underlying_type, false)); code += " "; - code += ev.name + " = "; + code += namer_.Variant(ev) + " = "; code += enum_def.ToString(ev); + if (enum_def.underlying_type.base_type == BASE_TYPE_LONG || + enum_def.underlying_type.base_type == BASE_TYPE_ULONG) { + code += "L"; + } code += ";\n"; } - // Generate a generate string table for enum values. + // Generate a string table for enum values. // Problem is, if values are very sparse that could generate really big // tables. Ideally in that case we generate a map lookup instead, but for // the moment we simply don't output a table at all. @@ -331,24 +446,27 @@ class JavaGenerator : public BaseGenerator { // Average distance between values above which we consider a table // "too sparse". Change at will. static const uint64_t kMaxSparseness = 5; - if (range / static_cast<uint64_t>(enum_def.size()) < kMaxSparseness) { + if (range / static_cast<uint64_t>(enum_def.size()) < kMaxSparseness && + GenTypeBasic(DestinationType(enum_def.underlying_type, false)) != + "long") { code += "\n public static final String"; code += "[] names = { "; - auto val = enum_def.Vals().front(); + const EnumVal *prev = enum_def.Vals().front(); for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { - auto ev = *it; - for (auto k = enum_def.Distance(val, ev); k > 1; --k) code += "\"\", "; - val = ev; - code += "\"" + (*it)->name + "\", "; + const EnumVal &ev = **it; + for (auto k = enum_def.Distance(prev, &ev); k > 1; --k) + code += "\"\", "; + prev = &ev; + code += "\"" + namer_.Variant(ev) + "\", "; } code += "};\n\n"; code += " public static "; code += "String"; - code += " " + MakeCamel("name", false); + code += " name"; code += "(int e) { return names[e"; if (enum_def.MinValue()->IsNonZero()) - code += " - " + enum_def.MinValue()->name; + code += " - " + namer_.Variant(enum_def.MinValue()->name); code += "]; }\n"; } @@ -369,7 +487,7 @@ class JavaGenerator : public BaseGenerator { if (type.base_type == BASE_TYPE_BOOL) { getter = "0!=" + getter; } else if (GenTypeBasic(type) != "byte") { - getter += MakeCamel(GenTypeBasic(type)); + getter += ConvertCase(GenTypeBasic(type), Case::kUpperCamel); } return getter; } @@ -385,7 +503,7 @@ class JavaGenerator : public BaseGenerator { auto dest_cast = DestinationCast(type); auto getter = data_buffer + ".get"; if (GenTypeBasic(type) != "byte") { - getter += MakeCamel(GenTypeBasic(type)); + getter += ConvertCase(GenTypeBasic(type), Case::kUpperCamel); } getter = dest_cast + getter + "(" + GenOffsetGetter(key_field, num) + ")" + dest_mask; @@ -398,7 +516,7 @@ class JavaGenerator : public BaseGenerator { if (IsScalar(type.base_type)) { std::string setter = "bb.put"; if (GenTypeBasic(type) != "byte" && type.base_type != BASE_TYPE_BOOL) { - setter += MakeCamel(GenTypeBasic(type)); + setter += ConvertCase(GenTypeBasic(type), Case::kUpperCamel); } return setter; } else { @@ -408,15 +526,15 @@ class JavaGenerator : public BaseGenerator { // Returns the method name for use with add/put calls. std::string GenMethod(const Type &type) const { - return IsScalar(type.base_type) ? MakeCamel(GenTypeBasic(type)) - : (IsStruct(type) ? "Struct" : "Offset"); + return IsScalar(type.base_type) + ? ConvertCase(GenTypeBasic(type), Case::kUpperCamel) + : (IsStruct(type) ? "Struct" : "Offset"); } // Recursively generate arguments for a constructor, to deal with nested // structs. - void GenStructArgs(const StructDef &struct_def, std::string *code_ptr, + void GenStructArgs(const StructDef &struct_def, std::string &code, const char *nameprefix, size_t array_count = 0) const { - std::string &code = *code_ptr; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { auto &field = **it; @@ -429,15 +547,15 @@ class JavaGenerator : public BaseGenerator { // Generate arguments for a struct inside a struct. To ensure names // don't clash, and to make it obvious these arguments are constructing // a nested struct, prefix the name with the field name. - GenStructArgs(*field_type.struct_def, code_ptr, + GenStructArgs(*field_type.struct_def, code, (nameprefix + (field.name + "_")).c_str(), array_cnt); } else { code += ", "; - code += GenTypeBasic(type); + code += GenTypeNameDest(field.value.type); for (size_t i = 0; i < array_cnt; i++) code += "[]"; code += " "; code += nameprefix; - code += MakeCamel(field.name, false); + code += namer_.Field(field); } } } @@ -445,10 +563,9 @@ class JavaGenerator : public BaseGenerator { // Recusively generate struct construction statements of the form: // builder.putType(name); // and insert manual padding. - void GenStructBody(const StructDef &struct_def, std::string *code_ptr, + void GenStructBody(const StructDef &struct_def, std::string &code, const char *nameprefix, size_t index = 0, bool in_array = false) const { - std::string &code = *code_ptr; std::string indent((index + 1) * 2, ' '); code += indent + " builder.prep("; code += NumToString(struct_def.minalign) + ", "; @@ -462,7 +579,7 @@ class JavaGenerator : public BaseGenerator { code += NumToString(field.padding) + ");\n"; } if (IsStruct(field_type)) { - GenStructBody(*field_type.struct_def, code_ptr, + GenStructBody(*field_type.struct_def, code, (nameprefix + (field.name + "_")).c_str(), index, in_array); } else { @@ -476,7 +593,7 @@ class JavaGenerator : public BaseGenerator { in_array = true; } if (IsStruct(type)) { - GenStructBody(*field_type.struct_def, code_ptr, + GenStructBody(*field_type.struct_def, code, (nameprefix + (field.name + "_")).c_str(), index + 1, in_array); } else { @@ -484,7 +601,7 @@ class JavaGenerator : public BaseGenerator { code += indent + " builder.put"; code += GenMethod(type) + "("; code += SourceCast(type); - auto argname = nameprefix + MakeCamel(field.name, false); + auto argname = nameprefix + namer_.Variable(field); code += argname; size_t array_cnt = index + (IsArray(field_type) ? 1 : 0); for (size_t i = 0; in_array && i < array_cnt; i++) { @@ -497,12 +614,6 @@ class JavaGenerator : public BaseGenerator { } } - std::string GenByteBufferLength(const char *bb_name) const { - std::string bb_len = bb_name; - bb_len += ".capacity()"; - return bb_len; - } - std::string GenOffsetGetter(flatbuffers::FieldDef *key_field, const char *num = nullptr) const { std::string key_offset = ""; @@ -511,7 +622,7 @@ class JavaGenerator : public BaseGenerator { key_offset += num; key_offset += ", _bb)"; } else { - key_offset += GenByteBufferLength("bb"); + key_offset += "bb.capacity()"; key_offset += " - tableOffset, bb)"; } return key_offset; @@ -560,9 +671,9 @@ class JavaGenerator : public BaseGenerator { return key_getter; } - void GenStruct(StructDef &struct_def, std::string *code_ptr) const { + void GenStruct(StructDef &struct_def, std::string &code, + const IDLOptions &opts) const { if (struct_def.generated) return; - std::string &code = *code_ptr; // Generate a struct accessor class, with methods of the form: // public type name() { return bb.getType(i + offset); } @@ -570,7 +681,7 @@ class JavaGenerator : public BaseGenerator { // public type name() { // int o = __offset(offset); return o != 0 ? bb.getType(o + i) : default; // } - GenComment(struct_def.doc_comment, code_ptr, &comment_config); + GenComment(struct_def.doc_comment, &code, &comment_config); if (parser_.opts.gen_generated) { code += "@javax.annotation.Generated(value=\"flatc\")\n"; @@ -581,33 +692,35 @@ class JavaGenerator : public BaseGenerator { } else { code += "public "; } - code += "final class " + struct_def.name; + const auto struct_class = namer_.Type(struct_def); + code += "final class " + struct_class; code += " extends "; code += struct_def.fixed ? "Struct" : "Table"; code += " {\n"; if (!struct_def.fixed) { - // Generate verson check method. + // Generate version check method. // Force compile time error if not using the same version runtime. code += " public static void ValidateVersion() {"; code += " Constants."; - code += "FLATBUFFERS_2_0_0(); "; + code += "FLATBUFFERS_23_5_9(); "; code += "}\n"; // Generate a special accessor for the table that when used as the root // of a FlatBuffer - std::string method_name = "getRootAs" + struct_def.name; - std::string method_signature = - " public static " + struct_def.name + " " + method_name; + const std::string method_name = + namer_.LegacyJavaMethod2("getRootAs", struct_def, ""); + const std::string method_signature = + " public static " + struct_class + " " + method_name; // create convenience method that doesn't require an existing object code += method_signature + "(ByteBuffer _bb) "; - code += "{ return " + method_name + "(_bb, new " + struct_def.name + - "()); }\n"; + code += + "{ return " + method_name + "(_bb, new " + struct_class + "()); }\n"; // create method that allows object reuse code += - method_signature + "(ByteBuffer _bb, " + struct_def.name + " obj) { "; + method_signature + "(ByteBuffer _bb, " + struct_class + " obj) { "; code += "_bb.order(ByteOrder.LITTLE_ENDIAN); "; code += "return (obj.__assign(_bb.getInt(_bb."; code += "position()"; @@ -618,8 +731,10 @@ class JavaGenerator : public BaseGenerator { if (parser_.file_identifier_.length()) { // Check if a buffer has the identifier. code += " public static "; - code += "boolean " + struct_def.name; - code += "BufferHasIdentifier(ByteBuffer _bb) { return "; + code += "boolean " + + namer_.LegacyJavaMethod2( + "", struct_def, "BufferHasIdentifier(ByteBuffer _bb)") + + " { return "; code += "__has_identifier(_bb, \""; code += parser_.file_identifier_; code += "\"); }\n"; @@ -632,27 +747,24 @@ class JavaGenerator : public BaseGenerator { code += "{ "; code += "__reset(_i, _bb); "; code += "}\n"; - code += - " public " + struct_def.name + " __assign(int _i, ByteBuffer _bb) "; + code += " public " + struct_class + " __assign(int _i, ByteBuffer _bb) "; code += "{ __init(_i, _bb); return this; }\n\n"; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { auto &field = **it; if (field.deprecated) continue; - GenComment(field.doc_comment, code_ptr, &comment_config, " "); - std::string type_name = GenTypeGet(field.value.type); - std::string type_name_dest = GenTypeNameDest(field.value.type); - std::string conditional_cast = ""; - std::string optional = ""; - std::string dest_mask = DestinationMask(field.value.type, true); - std::string dest_cast = DestinationCast(field.value.type); - std::string src_cast = SourceCast(field.value.type); - std::string method_start = + GenComment(field.doc_comment, &code, &comment_config, " "); + const std::string type_name = GenTypeGet(field.value.type); + const std::string type_name_dest = GenTypeNameDest(field.value.type); + const std::string dest_mask = DestinationMask(field.value.type, true); + const std::string dest_cast = DestinationCast(field.value.type); + const std::string src_cast = SourceCast(field.value.type); + const std::string method_start = " public " + (field.IsRequired() ? "" : GenNullableAnnotation(field.value.type)) + - GenPureAnnotation(field.value.type) + type_name_dest + optional + - " " + MakeCamel(field.name, false); - std::string obj = "obj"; + GenPureAnnotation(field.value.type) + type_name_dest + " " + + namer_.Field(field); + const std::string obj = "obj"; // Most field accessors need to retrieve and test the field offset first, // this is the prefix code for that: @@ -665,22 +777,21 @@ class JavaGenerator : public BaseGenerator { if (field.value.type.base_type == BASE_TYPE_STRUCT) { // Calls the accessor that takes an accessor object with a new object. code += method_start + "() { return "; - code += MakeCamel(field.name, false); + code += namer_.Field(field); code += "(new "; code += type_name + "()); }\n"; - } else if (IsVector(field.value.type) && + } else if (IsSeries(field.value.type) && field.value.type.element == BASE_TYPE_STRUCT) { // Accessors for vectors of structs also take accessor objects, this // generates a variant without that argument. code += method_start + "(int j) { return "; - code += MakeCamel(field.name, false); + code += namer_.Field(field); code += "(new " + type_name + "(), j); }\n"; } if (field.IsScalarOptional()) { code += GenOptionalScalarCheck(field); } std::string getter = dest_cast + GenGetter(field.value.type); code += method_start; - std::string default_cast = ""; std::string member_suffix = "; "; if (IsScalar(field.value.type.base_type)) { code += "()"; @@ -693,7 +804,7 @@ class JavaGenerator : public BaseGenerator { } else { code += offset_prefix + getter; code += "(o + bb_pos)" + dest_mask; - code += " : " + default_cast; + code += " : "; code += GenDefaultValue(field); } } else { @@ -705,7 +816,7 @@ class JavaGenerator : public BaseGenerator { code += "bb_pos + " + NumToString(field.value.offset) + ", "; code += "bb)"; } else { - code += offset_prefix + conditional_cast; + code += offset_prefix; code += obj + ".__assign("; code += field.value.type.struct_def->fixed ? "o + bb_pos" @@ -730,7 +841,7 @@ class JavaGenerator : public BaseGenerator { code += type_name + " obj, "; } code += "int j)"; - const auto body = offset_prefix + conditional_cast + getter + "("; + const auto body = offset_prefix + getter + "("; if (vectortype.base_type == BASE_TYPE_UNION) { code += body + "obj, "; } else { @@ -754,11 +865,9 @@ class JavaGenerator : public BaseGenerator { code += ")" + dest_mask; if (!IsArray(field.value.type)) { code += " : "; - code += - field.value.type.element == BASE_TYPE_BOOL - ? "false" - : (IsScalar(field.value.type.element) ? default_cast + "0" - : "null"); + code += field.value.type.element == BASE_TYPE_BOOL + ? "false" + : (IsScalar(field.value.type.element) ? "0" : "null"); } break; @@ -773,7 +882,7 @@ class JavaGenerator : public BaseGenerator { code += member_suffix; code += "}\n"; if (IsVector(field.value.type)) { - code += " public int " + MakeCamel(field.name, false); + code += " public int " + namer_.Field(field); code += "Length"; code += "()"; code += offset_prefix; @@ -788,9 +897,9 @@ class JavaGenerator : public BaseGenerator { for (auto kit = fields.begin(); kit != fields.end(); ++kit) { auto &key_field = **kit; if (key_field.key) { - auto qualified_name = WrapInNameSpace(sd); + auto qualified_name = Prefixed(namer_.NamespacedType(sd)); code += " public " + qualified_name + " "; - code += MakeCamel(field.name, false) + "ByKey("; + code += namer_.Method(field) + "ByKey("; code += GenTypeNameDest(key_field.value.type) + " key)"; code += offset_prefix; code += qualified_name + ".__lookup_by_key("; @@ -799,7 +908,7 @@ class JavaGenerator : public BaseGenerator { code += "bb) : null; "; code += "}\n"; code += " public " + qualified_name + " "; - code += MakeCamel(field.name, false) + "ByKey("; + code += namer_.Method(field) + "ByKey("; code += qualified_name + " obj, "; code += GenTypeNameDest(key_field.value.type) + " key)"; code += offset_prefix; @@ -817,7 +926,8 @@ class JavaGenerator : public BaseGenerator { std::string vector_type_name; const auto &element_base_type = field.value.type.VectorType().base_type; if (IsScalar(element_base_type)) { - vector_type_name = MakeCamel(type_name, true) + "Vector"; + vector_type_name = + ConvertCase(type_name, Case::kUpperCamel) + "Vector"; } else if (element_base_type == BASE_TYPE_STRING) { vector_type_name = "StringVector"; } else if (element_base_type == BASE_TYPE_UNION) { @@ -826,14 +936,13 @@ class JavaGenerator : public BaseGenerator { vector_type_name = type_name + ".Vector"; } auto vector_method_start = GenNullableAnnotation(field.value.type) + - " public " + vector_type_name + optional + - " " + MakeCamel(field.name, false) + - "Vector"; + " public " + vector_type_name + " " + + namer_.Field(field, "vector"); code += vector_method_start + "() { return "; - code += MakeCamel(field.name, false) + "Vector"; + code += namer_.Field(field, "vector"); code += "(new " + vector_type_name + "()); }\n"; code += vector_method_start + "(" + vector_type_name + " obj)"; - code += offset_prefix + conditional_cast + obj + ".__assign("; + code += offset_prefix + obj + ".__assign("; code += "__vector(o), "; if (!IsScalar(element_base_type)) { auto vectortype = field.value.type.VectorType(); @@ -846,7 +955,7 @@ class JavaGenerator : public BaseGenerator { IsScalar(field.value.type.VectorType().base_type)) || IsString(field.value.type)) { code += " public ByteBuffer "; - code += MakeCamel(field.name, false); + code += namer_.Field(field); code += "AsByteBuffer() { return "; code += "__vector_as_bytebuffer("; code += NumToString(field.value.offset) + ", "; @@ -855,7 +964,7 @@ class JavaGenerator : public BaseGenerator { : InlineSize(field.value.type.VectorType())); code += "); }\n"; code += " public ByteBuffer "; - code += MakeCamel(field.name, false); + code += namer_.Field(field); code += "InByteBuffer(ByteBuffer _bb) { return "; code += "__vector_in_bytebuffer(_bb, "; code += NumToString(field.value.offset) + ", "; @@ -866,9 +975,10 @@ class JavaGenerator : public BaseGenerator { } // generate object accessors if is nested_flatbuffer if (field.nested_flatbuffer) { - auto nested_type_name = WrapInNameSpace(*field.nested_flatbuffer); + auto nested_type_name = + Prefixed(namer_.NamespacedType(*field.nested_flatbuffer)); auto nested_method_name = - MakeCamel(field.name, false) + "As" + field.nested_flatbuffer->name; + namer_.Field(field) + "As" + field.nested_flatbuffer->name; auto get_nested_method_name = nested_method_name; code += " public " + nested_type_name + " "; code += nested_method_name + "() { return "; @@ -879,7 +989,7 @@ class JavaGenerator : public BaseGenerator { code += nested_type_name + " obj"; code += ") { int o = __offset("; code += NumToString(field.value.offset) + "); "; - code += "return o != 0 ? " + conditional_cast + obj + ".__assign("; + code += "return o != 0 ? " + obj + ".__assign("; code += ""; code += "__indirect(__vector(o)), "; code += "bb) : null; }\n"; @@ -894,7 +1004,6 @@ class JavaGenerator : public BaseGenerator { auto setter_parameter = underlying_type.base_type == BASE_TYPE_BOOL ? "(byte)(" + field.name + " ? 1 : 0)" : field.name; - auto mutator_prefix = MakeCamel("mutate", false); // A vector mutator also needs the index of the vector element it should // mutate. auto mutator_params = (is_series ? "(int j, " : "(") + @@ -912,7 +1021,7 @@ class JavaGenerator : public BaseGenerator { if (IsScalar(underlying_type.base_type) && !IsUnion(field.value.type)) { code += " public "; code += struct_def.fixed ? "void " : "boolean "; - code += mutator_prefix + MakeCamel(field.name, true); + code += namer_.Method("mutate", field); code += mutator_params; if (struct_def.fixed) { code += GenSetter(underlying_type) + "(" + setter_index + ", "; @@ -929,7 +1038,7 @@ class JavaGenerator : public BaseGenerator { if (parser_.opts.java_primitive_has_method && IsScalar(field.value.type.base_type) && !struct_def.fixed) { auto vt_offset_constant = " public static final int VT_" + - MakeScreamingCamel(field.name) + " = " + + namer_.Constant(field) + " = " + NumToString(field.value.offset) + ";"; code += vt_offset_constant; @@ -937,15 +1046,18 @@ class JavaGenerator : public BaseGenerator { } } code += "\n"; + auto struct_has_create = false; + std::set<flatbuffers::FieldDef *> field_has_create_set; flatbuffers::FieldDef *key_field = nullptr; if (struct_def.fixed) { + struct_has_create = true; // create a struct constructor function code += " public static " + GenOffsetType() + " "; code += "create"; - code += struct_def.name + "(FlatBufferBuilder builder"; - GenStructArgs(struct_def, code_ptr, ""); + code += struct_class + "(FlatBufferBuilder builder"; + GenStructArgs(struct_def, code, ""); code += ") {\n"; - GenStructBody(struct_def, code_ptr, ""); + GenStructBody(struct_def, code, ""); code += " return "; code += GenOffsetConstruct("builder." + std::string("offset()")); code += ";\n }\n"; @@ -968,19 +1080,21 @@ class JavaGenerator : public BaseGenerator { // JVM specifications restrict default constructor params to be < 255. // Longs and doubles take up 2 units, so we set the limit to be < 127. if (has_no_struct_fields && num_fields && num_fields < 127) { + struct_has_create = true; // Generate a table constructor of the form: // public static int createName(FlatBufferBuilder builder, args...) code += " public static " + GenOffsetType() + " "; - code += "create" + struct_def.name; + code += namer_.LegacyJavaMethod2("create", struct_def, ""); code += "(FlatBufferBuilder builder"; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { auto &field = **it; + auto field_name = namer_.Field(field); if (field.deprecated) continue; code += ",\n "; code += GenTypeBasic(DestinationType(field.value.type, false)); code += " "; - code += field.name; + code += field_name; if (!IsScalar(field.value.type.base_type)) code += "Offset"; } code += ") {\n builder."; @@ -991,19 +1105,19 @@ class JavaGenerator : public BaseGenerator { for (auto it = struct_def.fields.vec.rbegin(); it != struct_def.fields.vec.rend(); ++it) { auto &field = **it; + auto field_name = namer_.Field(field); if (!field.deprecated && (!struct_def.sortbysize || size == SizeOf(field.value.type.base_type))) { - code += " " + struct_def.name + "."; - code += "add"; - code += MakeCamel(field.name) + "(builder, " + field.name; + code += " " + struct_class + "."; + code += namer_.Method("add", field) + "(builder, " + field_name; if (!IsScalar(field.value.type.base_type)) code += "Offset"; code += ");\n"; } } } - code += " return " + struct_def.name + "."; - code += "end" + struct_def.name; + code += " return " + struct_class + "."; + code += namer_.LegacyJavaMethod2("end", struct_def, ""); code += "(builder);\n }\n\n"; } // Generate a set of static methods that allow table construction, @@ -1012,7 +1126,7 @@ class JavaGenerator : public BaseGenerator { // { builder.addShort(id, name, default); } // Unlike the Create function, these always work. code += " public static void start"; - code += struct_def.name; + code += struct_class; code += "(FlatBufferBuilder builder) { builder."; code += "startTable("; code += NumToString(struct_def.fields.vec.size()) + "); }\n"; @@ -1020,48 +1134,59 @@ class JavaGenerator : public BaseGenerator { it != struct_def.fields.vec.end(); ++it) { auto &field = **it; if (field.deprecated) continue; - if (field.key) key_field = &field; - code += " public static void add"; - code += MakeCamel(field.name); + + code += " public static void " + namer_.Method("add", field); code += "(FlatBufferBuilder builder, "; code += GenTypeBasic(DestinationType(field.value.type, false)); - auto argname = MakeCamel(field.name, false); + auto argname = namer_.Field(field); if (!IsScalar(field.value.type.base_type)) argname += "Offset"; code += " " + argname + ") { builder.add"; code += GenMethod(field.value.type) + "("; - code += NumToString(it - struct_def.fields.vec.begin()) + ", "; - code += SourceCastBasic(field.value.type); - code += argname; - code += ", "; - code += SourceCastBasic(field.value.type); - code += GenDefaultValue(field); - code += "); }\n"; + + if (field.key) { + // field has key attribute, so always need to exist + // even if its value is equal to default. + // Generated code will bypass default checking + // resulting in { builder.addShort(name); slot(id); } + key_field = &field; + code += SourceCastBasic(field.value.type); + code += argname; + code += "); builder.slot(" + + NumToString(it - struct_def.fields.vec.begin()) + "); }\n"; + } else { + code += NumToString(it - struct_def.fields.vec.begin()) + ", "; + code += SourceCastBasic(field.value.type); + code += argname; + code += ", "; + code += SourceCastBasic(field.value.type); + code += GenDefaultValue(field); + code += "); }\n"; + } if (IsVector(field.value.type)) { auto vector_type = field.value.type.VectorType(); auto alignment = InlineAlignment(vector_type); auto elem_size = InlineSize(vector_type); if (!IsStruct(vector_type)) { + field_has_create_set.insert(&field); // generate a method to create a vector from a java array. if ((vector_type.base_type == BASE_TYPE_CHAR || vector_type.base_type == BASE_TYPE_UCHAR)) { // Handle byte[] and ByteBuffers separately for Java code += " public static " + GenVectorOffsetType() + " "; - code += "create"; - code += MakeCamel(field.name); + code += namer_.Method("create", field); code += "Vector(FlatBufferBuilder builder, byte[] data) "; code += "{ return builder.createByteVector(data); }\n"; code += " public static " + GenVectorOffsetType() + " "; - code += "create"; - code += MakeCamel(field.name); + code += namer_.Method("create", field); code += "Vector(FlatBufferBuilder builder, ByteBuffer data) "; code += "{ return builder.createByteVector(data); }\n"; } else { code += " public static " + GenVectorOffsetType() + " "; - code += "create"; - code += MakeCamel(field.name); + code += namer_.Method("create", field); code += "Vector(FlatBufferBuilder builder, "; - code += GenTypeBasic(vector_type) + "[] data) "; + code += GenTypeBasic(DestinationType(vector_type, false)) + + "[] data) "; code += "{ builder.startVector("; code += NumToString(elem_size); code += ", data.length, "; @@ -1071,7 +1196,7 @@ class JavaGenerator : public BaseGenerator { code += "add"; code += GenMethod(vector_type); code += "("; - code += SourceCastBasic(vector_type, false); + code += SourceCastBasic(vector_type); code += "data[i]"; code += "); return "; code += "builder.endVector(); }\n"; @@ -1079,8 +1204,7 @@ class JavaGenerator : public BaseGenerator { } // Generate a method to start a vector, data to be added manually // after. - code += " public static void start"; - code += MakeCamel(field.name); + code += " public static void " + namer_.Method("start", field); code += "Vector(FlatBufferBuilder builder, int numElems) "; code += "{ builder.startVector("; code += NumToString(elem_size); @@ -1089,7 +1213,7 @@ class JavaGenerator : public BaseGenerator { } } code += " public static " + GenOffsetType() + " "; - code += "end" + struct_def.name; + code += namer_.LegacyJavaMethod2("end", struct_def, ""); code += "(FlatBufferBuilder builder) {\n int o = builder."; code += "endTable();\n"; for (auto it = struct_def.fields.vec.begin(); @@ -1106,8 +1230,9 @@ class JavaGenerator : public BaseGenerator { std::string size_prefix[] = { "", "SizePrefixed" }; for (int i = 0; i < 2; ++i) { code += " public static void "; - code += "finish" + size_prefix[i] + struct_def.name; - code += "Buffer(FlatBufferBuilder builder, " + GenOffsetType(); + code += namer_.LegacyJavaMethod2("finish" + size_prefix[i], + struct_def, "Buffer"); + code += "(FlatBufferBuilder builder, " + GenOffsetType(); code += " offset) {"; code += " builder.finish" + size_prefix[i] + "(offset"; @@ -1126,9 +1251,9 @@ class JavaGenerator : public BaseGenerator { code += GenKeyGetter(key_field); code += " }\n"; - code += "\n public static " + struct_def.name; + code += "\n public static " + struct_class; code += " __lookup_by_key("; - code += struct_def.name + " obj, "; + code += struct_class + " obj, "; code += "int vectorLocation, "; code += GenTypeNameDest(key_field->value.type); code += " key, ByteBuffer bb) {\n"; @@ -1150,27 +1275,28 @@ class JavaGenerator : public BaseGenerator { code += " span -= middle;\n"; code += " } else {\n"; code += " return "; - code += "(obj == null ? new " + struct_def.name + "() : obj)"; + code += "(obj == null ? new " + struct_class + "() : obj)"; code += ".__assign(tableOffset, bb);\n"; code += " }\n }\n"; code += " return null;\n"; code += " }\n"; } - GenVectorAccessObject(struct_def, code_ptr); - code += "}"; - code += "\n\n"; + GenVectorAccessObject(struct_def, code); + if (opts.generate_object_based_api) { + GenPackUnPack_ObjectAPI(struct_def, code, opts, struct_has_create, + field_has_create_set); + } + code += "}\n\n"; } std::string GenOptionalScalarCheck(FieldDef &field) const { if (!field.IsScalarOptional()) return ""; - return " public boolean has" + MakeCamel(field.name, true) + + return " public boolean " + namer_.Method("has", field) + "() { return 0 != __offset(" + NumToString(field.value.offset) + "); }\n"; } - void GenVectorAccessObject(StructDef &struct_def, - std::string *code_ptr) const { - auto &code = *code_ptr; + void GenVectorAccessObject(StructDef &struct_def, std::string &code) const { // Generate a vector of structs accessor class. code += "\n"; code += " "; @@ -1187,7 +1313,7 @@ class JavaGenerator : public BaseGenerator { code += "__assign(int _vector, int _element_size, ByteBuffer _bb) { "; code += "__reset(_vector, _element_size, _bb); return this; }\n\n"; - auto type_name = struct_def.name; + auto type_name = namer_.Type(struct_def); auto method_start = method_indent + "public " + type_name + " get"; // Generate the accessors that don't do object reuse. code += method_start + "(int j) { return get"; @@ -1229,16 +1355,904 @@ class JavaGenerator : public BaseGenerator { code += " }\n"; } + void GenEnum_ObjectAPI(EnumDef &enum_def, std::string &code) const { + if (enum_def.generated) return; + code += "import com.google.flatbuffers.FlatBufferBuilder;\n\n"; + + if (!enum_def.attributes.Lookup("private")) { code += "public "; } + auto union_name = namer_.Type(enum_def) + "Union"; + auto union_type = + GenTypeBasic(DestinationType(enum_def.underlying_type, false)); + code += "class " + union_name + " {\n"; + // Type + code += " private " + union_type + " type;\n"; + // Value + code += " private Object value;\n"; + code += "\n"; + // getters and setters + code += " public " + union_type + " getType() { return type; }\n\n"; + code += " public void setType(" + union_type + + " type) { this.type = type; }\n\n"; + code += " public Object getValue() { return value; }\n\n"; + code += " public void setValue(Object value) { this.value = value; }\n\n"; + // Constructor + code += " public " + union_name + "() {\n"; + code += + " this.type = " + namer_.EnumVariant(enum_def, *enum_def.Vals()[0]) + + ";\n"; + code += " this.value = null;\n"; + code += " }\n\n"; + // As + for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { + auto &ev = **it; + if (ev.union_type.base_type == BASE_TYPE_NONE) continue; + auto type_name = GenTypeGet_ObjectAPI(ev.union_type, false, true); + if (ev.union_type.base_type == BASE_TYPE_STRUCT && + ev.union_type.struct_def->attributes.Lookup("private")) { + code += " "; + } else { + code += " public "; + } + code += type_name + " as" + ev.name + "() { return (" + type_name + + ") value; }\n"; + } + code += "\n"; + // pack() + code += " public static int pack(FlatBufferBuilder builder, " + + union_name + " _o) {\n"; + code += " switch (_o.type) {\n"; + for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { + auto &ev = **it; + if (ev.union_type.base_type == BASE_TYPE_NONE) { + continue; + } else { + code += " case " + namer_.EnumVariant(enum_def, ev) + ": return "; + if (IsString(ev.union_type)) { + code += "builder.createString(_o.as" + ev.name + "());\n"; + } else { + code += GenTypeGet(ev.union_type) + ".pack(builder, _o.as" + ev.name + + "());\n"; + } + } + } + code += " default: return 0;\n"; + code += " }\n"; + code += " }\n"; + code += "}\n\n"; + } + + void GenUnionUnPack_ObjectAPI(const EnumDef &enum_def, std::string &code, + const std::string &type_name, + const std::string &field_name, + bool is_vector) const { + const std::string variable_type = + is_vector ? type_name.substr(0, type_name.length() - 2) : type_name; + const std::string variable_name = + "_" + namer_.Variable("o", field_name) + (is_vector ? "Element" : ""); + const std::string type_params = is_vector ? "_j" : ""; + const std::string value_params = is_vector ? ", _j" : ""; + const std::string indent = (is_vector ? " " : " "); + + code += indent + variable_type + " " + variable_name + " = new " + + variable_type + "();\n"; + code += indent + + GenTypeBasic(DestinationType(enum_def.underlying_type, false)) + + " " + variable_name + "Type = " + field_name + "Type(" + + type_params + ");\n"; + code += indent + variable_name + ".setType(" + variable_name + "Type);\n"; + code += indent + "Table " + variable_name + "Value;\n"; + code += indent + "switch (" + variable_name + "Type) {\n"; + for (auto eit = enum_def.Vals().begin(); eit != enum_def.Vals().end(); + ++eit) { + auto &ev = **eit; + if (ev.union_type.base_type == BASE_TYPE_NONE) { + continue; + } else { + if (ev.union_type.base_type == BASE_TYPE_STRING || + (ev.union_type.base_type == BASE_TYPE_STRUCT && + ev.union_type.struct_def->fixed)) { + continue; // This branch is due to bad implemantation of Unions in + // Java which doesn't handle non Table types. Should be + // deleted when issue #6561 is fixed. + } + code += indent + " case " + + Prefixed(namer_.NamespacedEnumVariant(enum_def, ev)) + ":\n"; + auto actual_type = GenTypeGet(ev.union_type); + code += indent + " " + variable_name + "Value = " + field_name + + "(new " + actual_type + "()" + value_params + ");\n"; + code += indent + " " + variable_name + ".setValue(" + variable_name + + "Value != null ? ((" + actual_type + ") " + variable_name + + "Value).unpack() : null);\n"; + code += indent + " break;\n"; + } + } + code += indent + " default: break;\n"; + code += indent + "}\n"; + if (is_vector) { + code += indent + "_" + namer_.Variable("o", field_name) + + "[_j] = " + variable_name + ";\n"; + } + } + + void GenPackUnPack_ObjectAPI( + StructDef &struct_def, std::string &code, const IDLOptions &opts, + bool struct_has_create, + const std::set<FieldDef *> &field_has_create) const { + auto struct_name = namer_.ObjectType(struct_def); + // unpack() + code += " public " + struct_name + " unpack() {\n"; + code += " " + struct_name + " _o = new " + struct_name + "();\n"; + code += " unpackTo(_o);\n"; + code += " return _o;\n"; + code += " }\n"; + // unpackTo() + code += " public void unpackTo(" + struct_name + " _o) {\n"; + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + const auto &field = **it; + if (field.deprecated) continue; + if (field.value.type.base_type == BASE_TYPE_UTYPE) continue; + if (field.value.type.element == BASE_TYPE_UTYPE) continue; + const auto accessor = namer_.Method(field); + const auto variable = "_" + namer_.Variable("o", field); + const auto get_field = namer_.Method("get", field); + const auto set_field = namer_.Method("set", field); + + auto type_name = GenTypeGet_ObjectAPI(field.value.type, false, true); + if (field.IsScalarOptional()) + type_name = ConvertPrimitiveTypeToObjectWrapper_ObjectAPI(type_name); + auto start = " " + type_name + " " + variable + " = "; + auto call_setter = true; + switch (field.value.type.base_type) { + case BASE_TYPE_STRUCT: { + auto fixed = struct_def.fixed && field.value.type.struct_def->fixed; + if (fixed) { + code += + " " + accessor + "().unpackTo(_o." + get_field + "());\n"; + } else { + code += " if (" + accessor + "() != null) "; + if (field.value.type.struct_def->fixed) { + code += accessor + "().unpackTo(_o." + get_field + "());\n"; + } else { + code += "_o." + set_field + "(" + accessor + "().unpack());\n"; + } + code += " else _o." + set_field + "(null);\n"; + } + call_setter = false; + break; + } + case BASE_TYPE_ARRAY: { + auto length_str = NumToString(field.value.type.fixed_length); + auto unpack_method = + field.value.type.struct_def == nullptr ? "" : ".unpack()"; + code += start + "_o." + get_field + "();\n"; + code += " for (int _j = 0; _j < " + length_str + "; ++_j) { " + + variable + "[_j] = " + accessor + "(_j)" + unpack_method + + "; }\n"; + call_setter = false; + break; + } + case BASE_TYPE_VECTOR: + if (field.value.type.element == BASE_TYPE_UNION) { + code += start + "new " + + GenConcreteTypeGet_ObjectAPI(field.value.type) + .substr(0, type_name.length() - 1) + + accessor + "Length()];\n"; + code += + " for (int _j = 0; _j < " + accessor + "Length(); ++_j) {\n"; + GenUnionUnPack_ObjectAPI(*field.value.type.enum_def, code, + type_name, accessor, true); + code += " }\n"; + } else if (field.value.type.element != BASE_TYPE_UTYPE) { + auto fixed = field.value.type.struct_def == nullptr; + const auto length_accessor = namer_.Method(field, "length"); + code += start + "new " + + GenConcreteTypeGet_ObjectAPI(field.value.type) + .substr(0, type_name.length() - 1) + + length_accessor + "()];\n"; + code += + " for (int _j = 0; _j < " + length_accessor + "(); ++_j) {"; + code += variable + "[_j] = "; + if (fixed) { + code += accessor + "(_j)"; + } else { + code += "(" + accessor + "(_j) != null ? " + accessor + + "(_j).unpack() : null)"; + } + code += ";}\n"; + } + break; + case BASE_TYPE_UTYPE: break; + case BASE_TYPE_UNION: { + GenUnionUnPack_ObjectAPI(*field.value.type.enum_def, code, type_name, + accessor, false); + break; + } + default: { + if (field.IsScalarOptional()) { + code += start + namer_.Method("has", field) + "() ? " + accessor + + "() : null;\n"; + } else { + code += start + accessor + "();\n"; + } + break; + } + } + if (call_setter) { + code += " _o." + set_field + "(" + variable + ");\n"; + } + } + code += " }\n"; + // pack() + code += " public static " + GenOffsetType() + + " pack(FlatBufferBuilder builder, " + struct_name + " _o) {\n"; + code += " if (_o == null) return 0;\n"; + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + auto &field = **it; + if (field.deprecated) continue; + const auto field_name = namer_.Field(field); + const auto variable = "_" + namer_.Variable("o", field); + const auto get_field = namer_.Method("get", field); + // pre + switch (field.value.type.base_type) { + case BASE_TYPE_STRUCT: { + if (!field.value.type.struct_def->fixed) { + code += " " + GenOffsetType() + " _" + namer_.Variable(field) + + " = _o." + get_field + + "() == null ? 0 : " + GenTypeGet(field.value.type) + + ".pack(builder, _o." + get_field + "());\n"; + } else if (struct_def.fixed && struct_has_create) { + std::vector<FieldArrayLength> array_lengths; + FieldArrayLength tmp_array_length = { + field.name, + field.value.type.fixed_length, + }; + array_lengths.push_back(tmp_array_length); + GenStructPackDecl_ObjectAPI(*field.value.type.struct_def, + array_lengths, code); + } + break; + } + case BASE_TYPE_STRING: { + code += " int _" + field_name + " = _o." + get_field + + "() == null ? 0 : " + "builder.createString(_o." + + get_field + "());\n"; + break; + } + case BASE_TYPE_VECTOR: { + if (field_has_create.find(&field) != field_has_create.end()) { + auto property_name = field_name; + auto gen_for_loop = true; + std::string array_name = "__" + field_name; + std::string array_type = ""; + std::string element_type = ""; + std::string to_array = ""; + switch (field.value.type.element) { + case BASE_TYPE_STRING: { + array_type = "int"; + element_type = "String"; + to_array = "builder.createString(_e)"; + break; + } + case BASE_TYPE_STRUCT: + array_type = "int"; + element_type = + GenTypeGet_ObjectAPI(field.value.type, true, true); + ; + to_array = GenTypeGet(field.value.type) + ".pack(builder, _e)"; + break; + case BASE_TYPE_UTYPE: + property_name = field_name.substr(0, field_name.size() - 4); + array_type = GenTypeBasic(DestinationType( + field.value.type.enum_def->underlying_type, false)); + element_type = field.value.type.enum_def->name + "Union"; + to_array = "_o." + namer_.Method("get", property_name) + + "()[_j].getType()"; + break; + case BASE_TYPE_UNION: + array_type = "int"; + element_type = + Prefixed(namer_.NamespacedType(*field.value.type.enum_def)) + + "Union"; + to_array = element_type + ".pack(builder, _o." + + namer_.Method("get", property_name) + "()[_j])"; + break; + case BASE_TYPE_UCHAR: // TODO this branch of the switch is due to + // inconsistent behavior in unsigned byte. + // Read further at Issue #6574. + array_type = "byte"; + element_type = "int"; + to_array = "(byte) _e"; + break; + default: + gen_for_loop = false; + array_name = "_o." + namer_.Method("get", property_name) + "()"; + array_type = GenTypeNameDest(field.value.type); + element_type = array_type; + to_array = "_e"; + break; + } + code += " int _" + field_name + " = 0;\n"; + code += " if (_o." + namer_.Method("get", property_name) + + "() != null) {\n"; + if (gen_for_loop) { + code += " " + array_type + "[] " + array_name + " = new " + + array_type + "[_o." + + namer_.Method("get", property_name) + "().length];\n"; + code += " int _j = 0;\n"; + code += " for (" + element_type + " _e : _o." + + namer_.Method("get", property_name) + "()) { "; + code += array_name + "[_j] = " + to_array + "; _j++;}\n"; + } + code += " _" + field_name + " = " + + namer_.Method("create", field) + "Vector(builder, " + + array_name + ");\n"; + code += " }\n"; + } else { + auto type_name = GenTypeGet(field.value.type); + auto element_type_name = + GenTypeGet_ObjectAPI(field.value.type, true, true); + auto pack_method = + field.value.type.struct_def == nullptr + ? "builder.add" + GenMethod(field.value.type.VectorType()) + + "(" + variable + "[_j]);" + : "_unused_offset = " + type_name + ".pack(builder, " + + variable + "[_j]);"; + code += " int _" + field_name + " = 0;\n"; + code += " " + element_type_name + "[] " + variable + " = _o." + + get_field + "();\n"; + code += " if (" + variable + " != null) {\n"; + if (field.value.type.struct_def != nullptr) { + code += " int _unused_offset = 0;\n"; + } + code += " " + namer_.Method("start", field) + + "Vector(builder, " + variable + ".length);\n"; + code += " for (int _j = " + variable + + ".length - 1; _j >=0; _j--) { "; + code += pack_method + "}\n"; + code += " _" + field_name + " = builder.endVector();\n"; + code += " }\n"; + } + break; + } + case BASE_TYPE_ARRAY: { + if (field.value.type.struct_def != nullptr) { + std::vector<FieldArrayLength> array_lengths; + FieldArrayLength tmp_array_length = { + field.name, + field.value.type.fixed_length, + }; + array_lengths.push_back(tmp_array_length); + GenStructPackDecl_ObjectAPI(*field.value.type.struct_def, + array_lengths, code); + } else { + code += " " + + GenTypeGet_ObjectAPI(field.value.type, false, true) + " _" + + field_name + " = _o." + get_field + "();\n"; + } + break; + } + case BASE_TYPE_UNION: { + code += " " + + GenTypeBasic(DestinationType( + field.value.type.enum_def->underlying_type, false)) + + " _" + field_name + "Type = _o." + get_field + + "() == null ? " + + Prefixed(namer_.NamespacedType(*field.value.type.enum_def)) + + ".NONE : " + "_o." + get_field + "().getType();\n"; + code += " " + GenOffsetType() + " _" + field_name + " = _o." + + get_field + "() == null ? 0 : " + + Prefixed(namer_.NamespacedType(*field.value.type.enum_def)) + + "Union.pack(builder, _o." + get_field + "());\n"; + break; + } + default: break; + } + } + if (struct_has_create) { + // Create + code += " return " + + namer_.LegacyJavaMethod2("create", struct_def, "") + "(\n"; + code += " builder"; + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + auto &field = **it; + if (field.deprecated) continue; + const auto field_name = namer_.Field(field); + const auto get_field = namer_.Method("get", field); + switch (field.value.type.base_type) { + case BASE_TYPE_STRUCT: { + if (struct_def.fixed) { + GenStructPackCall_ObjectAPI(*field.value.type.struct_def, code, + " _" + field_name + "_"); + } else { + code += ",\n"; + if (field.value.type.struct_def->fixed) { + if (opts.generate_object_based_api) + code += " _o." + field_name; + else + // Seems like unreachable code + code += " " + GenTypeGet(field.value.type) + + ".Pack(builder, _o." + field_name + ")"; + } else { + code += " _" + field_name; + } + } + break; + } + case BASE_TYPE_ARRAY: { + if (field.value.type.struct_def != nullptr) { + GenStructPackCall_ObjectAPI(*field.value.type.struct_def, code, + " _" + field_name + "_"); + } else { + code += ",\n"; + code += " _" + field_name; + } + break; + } + case BASE_TYPE_UNION: FLATBUFFERS_FALLTHROUGH(); // fall thru + case BASE_TYPE_UTYPE: FLATBUFFERS_FALLTHROUGH(); // fall thru + case BASE_TYPE_STRING: FLATBUFFERS_FALLTHROUGH(); // fall thru + case BASE_TYPE_VECTOR: { + code += ",\n"; + code += " _" + field_name; + break; + } + default: // scalar + code += ",\n"; + code += " _o." + get_field + "()"; + break; + } + } + code += ");\n"; + } else { + // Start, End + code += " " + namer_.LegacyJavaMethod2("start", struct_def, "") + + "(builder);\n"; + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + auto &field = **it; + if (field.deprecated) continue; + const auto arg = "_" + namer_.Variable(field); + const auto get_field = namer_.Method("get", field); + const auto add_field = namer_.Method("add", field); + + switch (field.value.type.base_type) { + case BASE_TYPE_STRUCT: { + if (field.value.type.struct_def->fixed) { + code += " " + add_field + "(builder, " + + GenTypeGet(field.value.type) + ".pack(builder, _o." + + get_field + "()));\n"; + } else { + code += " " + add_field + "(builder, " + arg + ");\n"; + } + break; + } + case BASE_TYPE_STRING: FLATBUFFERS_FALLTHROUGH(); // fall thru + case BASE_TYPE_ARRAY: FLATBUFFERS_FALLTHROUGH(); // fall thru + case BASE_TYPE_VECTOR: { + code += " " + add_field + "(builder, " + arg + ");\n"; + break; + } + case BASE_TYPE_UTYPE: break; + case BASE_TYPE_UNION: { + code += " " + add_field + "Type(builder, " + arg + "Type);\n"; + code += " " + add_field + "(builder, " + arg + ");\n"; + break; + } + // scalar + default: { + if (field.IsScalarOptional()) { + code += " if (_o." + get_field + "() != null) { " + add_field + + "(builder, _o." + get_field + "()); }\n"; + } else { + code += + " " + add_field + "(builder, _o." + get_field + "());\n"; + } + break; + } + } + } + code += " return " + namer_.LegacyJavaMethod2("end", struct_def, "") + + "(builder);\n"; + } + code += " }\n"; + } + + void GenStructPackDecl_ObjectAPI(const StructDef &struct_def, + std::vector<FieldArrayLength> &array_lengths, + std::string &code) const { + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + const FieldDef &field = **it; + const bool is_array = IsArray(field.value.type); + const Type &field_type = + is_array ? field.value.type.VectorType() : field.value.type; + FieldArrayLength tmp_array_length = { + field.name, + field_type.fixed_length, + }; + array_lengths.push_back(tmp_array_length); + if (field_type.struct_def != nullptr) { + GenStructPackDecl_ObjectAPI(*field_type.struct_def, array_lengths, + code); + } else { + std::vector<FieldArrayLength> array_only_lengths; + for (size_t i = 0; i < array_lengths.size(); ++i) { + if (array_lengths[i].length > 0) { + array_only_lengths.push_back(array_lengths[i]); + } + } + std::string name; + for (size_t i = 0; i < array_lengths.size(); ++i) { + name += "_" + namer_.Variable(array_lengths[i].name); + } + code += " " + GenTypeBasic(field_type); + if (array_only_lengths.size() > 0) { + for (size_t i = 0; i < array_only_lengths.size(); ++i) { + code += "[]"; + } + code += " " + name + " = "; + code += "new " + GenTypeBasic(field_type) + "["; + for (size_t i = 0; i < array_only_lengths.size(); ++i) { + if (i != 0) { code += "]["; } + code += NumToString(array_only_lengths[i].length); + } + code += "];\n"; + code += " "; + // initialize array + for (size_t i = 0; i < array_only_lengths.size(); ++i) { + auto idx = "idx" + NumToString(i); + code += "for (int " + idx + " = 0; " + idx + " < " + + NumToString(array_only_lengths[i].length) + "; ++" + idx + + ") {"; + } + for (size_t i = 0; i < array_only_lengths.size(); ++i) { + auto idx = "idx" + NumToString(i); + if (i == 0) { + code += name + "[" + idx; + } else { + code += "][" + idx; + } + } + code += "] = _o"; + for (size_t i = 0, j = 0; i < array_lengths.size(); ++i) { + code += "." + namer_.Method("get", array_lengths[i].name) + "()"; + if (array_lengths[i].length <= 0) continue; + code += "[idx" + NumToString(j++) + "]"; + } + code += ";"; + for (size_t i = 0; i < array_only_lengths.size(); ++i) { + code += "}"; + } + } else { + code += " " + name + " = "; + code += SourceCast(field_type); + code += "_o"; + for (size_t i = 0; i < array_lengths.size(); ++i) { + code += "." + namer_.Method("get", array_lengths[i].name) + "()"; + } + code += ";"; + } + code += "\n"; + } + array_lengths.pop_back(); + } + } + + void GenStructPackCall_ObjectAPI(const StructDef &struct_def, + std::string &code, + std::string prefix) const { + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + auto &field = **it; + const auto &field_type = field.value.type; + if (field_type.struct_def != nullptr) { + GenStructPackCall_ObjectAPI(*field_type.struct_def, code, + prefix + namer_.Field(field) + "_"); + } else { + code += ",\n"; + code += prefix + namer_.Field(field); + } + } + } + + std::string ConvertPrimitiveTypeToObjectWrapper_ObjectAPI( + const std::string &type_name) const { + if (type_name == "boolean") + return "Boolean"; + else if (type_name == "byte") + return "Byte"; + else if (type_name == "char") + return "Character"; + else if (type_name == "short") + return "Short"; + else if (type_name == "int") + return "Integer"; + else if (type_name == "long") + return "Long"; + else if (type_name == "float") + return "Float"; + else if (type_name == "double") + return "Double"; + return type_name; + } + + std::string GenTypeGet_ObjectAPI(const flatbuffers::Type &type, + bool vectorelem, + bool wrap_in_namespace) const { + auto type_name = GenTypeNameDest(type); + // Replace to ObjectBaseAPI Type Name + switch (type.base_type) { + case BASE_TYPE_STRUCT: FLATBUFFERS_FALLTHROUGH(); // fall thru + case BASE_TYPE_ARRAY: FLATBUFFERS_FALLTHROUGH(); // fall thru + case BASE_TYPE_VECTOR: { + if (type.struct_def != nullptr) { + auto type_name_length = type.struct_def->name.length(); + auto new_type_name = namer_.ObjectType(*type.struct_def); + type_name.replace(type_name.length() - type_name_length, + type_name_length, new_type_name); + } else if (type.element == BASE_TYPE_UNION) { + if (wrap_in_namespace) { + type_name = + Prefixed(namer_.NamespacedType(*type.enum_def)) + "Union"; + } else { + type_name = namer_.Type(*type.enum_def) + "Union"; + } + } + break; + } + + case BASE_TYPE_UNION: { + if (wrap_in_namespace) { + type_name = + Prefixed(namer_.NamespacedType(*type.enum_def)) + "Union"; + } else { + type_name = namer_.Type(*type.enum_def) + "Union"; + } + break; + } + default: break; + } + if (vectorelem) { return type_name; } + switch (type.base_type) { + case BASE_TYPE_ARRAY: FLATBUFFERS_FALLTHROUGH(); // fall thru + case BASE_TYPE_VECTOR: { + type_name = type_name + "[]"; + break; + } + default: break; + } + return type_name; + } + + std::string GenConcreteTypeGet_ObjectAPI( + const flatbuffers::Type &type) const { + auto type_name = GenTypeNameDest(type); + // Replace to ObjectBaseAPI Type Name + switch (type.base_type) { + case BASE_TYPE_STRUCT: FLATBUFFERS_FALLTHROUGH(); // fall thru + case BASE_TYPE_ARRAY: FLATBUFFERS_FALLTHROUGH(); // fall thru + case BASE_TYPE_VECTOR: { + if (type.struct_def != nullptr) { + auto type_name_length = type.struct_def->name.length(); + auto new_type_name = namer_.ObjectType(*type.struct_def); + type_name.replace(type_name.length() - type_name_length, + type_name_length, new_type_name); + } else if (type.element == BASE_TYPE_UNION) { + type_name = + Prefixed(namer_.NamespacedType(*type.enum_def)) + "Union"; + } + break; + } + + case BASE_TYPE_UNION: { + type_name = + Prefixed(namer_.NamespacedType(*type.enum_def)) + "Union"; + break; + } + default: break; + } + + switch (type.base_type) { + case BASE_TYPE_ARRAY: FLATBUFFERS_FALLTHROUGH(); // fall thru + case BASE_TYPE_VECTOR: { + type_name = type_name + "[]"; + break; + } + default: break; + } + return type_name; + } + + void GenStruct_ObjectAPI(const StructDef &struct_def, + std::string &code) const { + if (struct_def.generated) return; + if (struct_def.attributes.Lookup("private")) { + // For Java, we leave the enum unmarked to indicate package-private + } else { + code += "public "; + } + + const auto class_name = namer_.ObjectType(struct_def); + code += "class " + class_name; + code += " {\n"; + // Generate Properties + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + const auto &field = **it; + if (field.deprecated) continue; + if (field.value.type.base_type == BASE_TYPE_UTYPE) continue; + if (field.value.type.element == BASE_TYPE_UTYPE) continue; + auto type_name = GenTypeGet_ObjectAPI(field.value.type, false, true); + if (field.IsScalarOptional()) + type_name = ConvertPrimitiveTypeToObjectWrapper_ObjectAPI(type_name); + const auto field_name = namer_.Field(field); + code += " private " + type_name + " " + field_name + ";\n"; + } + // Generate Java getters and setters + code += "\n"; + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + const auto &field = **it; + if (field.deprecated) continue; + if (field.value.type.base_type == BASE_TYPE_UTYPE) continue; + if (field.value.type.element == BASE_TYPE_UTYPE) continue; + const auto field_name = namer_.Field(field); + const auto get_field = namer_.Method("get", field); + auto type_name = GenTypeGet_ObjectAPI(field.value.type, false, true); + if (field.IsScalarOptional()) + type_name = ConvertPrimitiveTypeToObjectWrapper_ObjectAPI(type_name); + + code += " public " + type_name + " " + get_field + "() { return " + + field_name + "; }\n\n"; + std::string array_validation = ""; + if (field.value.type.base_type == BASE_TYPE_ARRAY) { + array_validation = + "if (" + field_name + " != null && " + field_name + + ".length == " + NumToString(field.value.type.fixed_length) + ") "; + } + code += " public void " + namer_.Method("set", field) + "(" + type_name + + " " + field_name + ") { " + array_validation + "this." + + field_name + " = " + field_name + "; }\n\n"; + } + // Generate Constructor + code += "\n"; + code += " public " + class_name + "() {\n"; + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + const auto &field = **it; + if (field.deprecated) continue; + if (field.value.type.base_type == BASE_TYPE_UTYPE) continue; + if (field.value.type.element == BASE_TYPE_UTYPE) continue; + const auto get_field = namer_.Method("get", field); + + code += " this." + namer_.Field(field) + " = "; + const auto type_name = + GenTypeGet_ObjectAPI(field.value.type, false, true); + if (IsScalar(field.value.type.base_type)) { + if (field.IsScalarOptional()) { + code += "null;\n"; + } else { + code += GenDefaultValue(field) + ";\n"; + } + } else { + switch (field.value.type.base_type) { + case BASE_TYPE_STRUCT: { + if (IsStruct(field.value.type)) { + code += "new " + type_name + "();\n"; + } else { + code += "null;\n"; + } + break; + } + case BASE_TYPE_ARRAY: { + code += "new " + type_name.substr(0, type_name.length() - 1) + + NumToString(field.value.type.fixed_length) + "];\n"; + break; + } + default: { + code += "null;\n"; + break; + } + } + } + } + code += " }\n"; + if (parser_.root_struct_def_ == &struct_def) { + const std::string struct_type = namer_.Type(struct_def); + code += " public static " + class_name + + " deserializeFromBinary(byte[] fbBuffer) {\n"; + code += " return " + struct_type + "." + + namer_.LegacyJavaMethod2("getRootAs", struct_def, "") + + "(ByteBuffer.wrap(fbBuffer)).unpack();\n"; + code += " }\n"; + code += " public byte[] serializeToBinary() {\n"; + code += " FlatBufferBuilder fbb = new FlatBufferBuilder();\n"; + code += " " + struct_type + "." + + namer_.LegacyJavaMethod2("finish", struct_def, "Buffer") + + "(fbb, " + struct_type + ".pack(fbb, this));\n"; + code += " return fbb.sizedByteArray();\n"; + code += " }\n"; + } + code += "}\n\n"; + } + // This tracks the current namespace used to determine if a type need to be // prefixed by its namespace const Namespace *cur_name_space_; + const IdlNamer namer_; + + private: + std::string Prefixed(const std::string &str) const { + return package_prefix_ + str; + } + + std::string package_prefix_; + Namespace package_prefix_ns_; + }; } // namespace java bool GenerateJava(const Parser &parser, const std::string &path, const std::string &file_name) { - java::JavaGenerator generator(parser, path, file_name); + java::JavaGenerator generator(parser, path, file_name, + parser.opts.java_package_prefix); return generator.generate(); } +namespace { + +class JavaCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateJava(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + output = JavaMakeRule(parser, path, filename); + return Status::OK; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateJavaGRPC(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kJava; } + + std::string LanguageName() const override { return "Java"; } +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewJavaCodeGenerator() { + return std::unique_ptr<JavaCodeGenerator>(new JavaCodeGenerator()); +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_java.h b/contrib/libs/flatbuffers/src/idl_gen_java.h new file mode 100644 index 00000000000..20798a44841 --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_java.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_JAVA_H_ +#define FLATBUFFERS_IDL_GEN_JAVA_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Java code generator. +std::unique_ptr<CodeGenerator> NewJavaCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_JAVA_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_json_schema.cpp b/contrib/libs/flatbuffers/src/idl_gen_json_schema.cpp index d58bb849769..3849da856a2 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_json_schema.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_json_schema.cpp @@ -14,7 +14,11 @@ * limitations under the License. */ +#include "idl_gen_json_schema.h" + +#include <algorithm> #include <iostream> +#include <limits> #include "flatbuffers/code_generators.h" #include "flatbuffers/idl.h" @@ -24,7 +28,10 @@ namespace flatbuffers { namespace jsons { -template<class T> std::string GenFullName(const T *enum_def) { +namespace { + +template<class T> +static std::string GenFullName(const T *enum_def) { std::string full_name; const auto &name_spaces = enum_def->defined_namespace->components; for (auto ns = name_spaces.cbegin(); ns != name_spaces.cend(); ++ns) { @@ -34,15 +41,16 @@ template<class T> std::string GenFullName(const T *enum_def) { return full_name; } -template<class T> std::string GenTypeRef(const T *enum_def) { +template<class T> +static std::string GenTypeRef(const T *enum_def) { return "\"$ref\" : \"#/definitions/" + GenFullName(enum_def) + "\""; } -std::string GenType(const std::string &name) { +static std::string GenType(const std::string &name) { return "\"type\" : \"" + name + "\""; } -std::string GenType(BaseType type) { +static std::string GenType(BaseType type) { switch (type) { case BASE_TYPE_BOOL: return "\"type\" : \"boolean\""; case BASE_TYPE_CHAR: @@ -84,13 +92,13 @@ std::string GenType(BaseType type) { } } -std::string GenBaseType(const Type &type) { +static std::string GenBaseType(const Type &type) { if (type.struct_def != nullptr) { return GenTypeRef(type.struct_def); } if (type.enum_def != nullptr) { return GenTypeRef(type.enum_def); } return GenType(type.base_type); } -std::string GenArrayType(const Type &type) { +static std::string GenArrayType(const Type &type) { std::string element_type; if (type.struct_def != nullptr) { element_type = GenTypeRef(type.struct_def); @@ -103,7 +111,7 @@ std::string GenArrayType(const Type &type) { return "\"type\" : \"array\", \"items\" : {" + element_type + "}"; } -std::string GenType(const Type &type) { +static std::string GenType(const Type &type) { switch (type.base_type) { case BASE_TYPE_ARRAY: FLATBUFFERS_FALLTHROUGH(); // fall thru case BASE_TYPE_VECTOR: { @@ -136,6 +144,8 @@ std::string GenType(const Type &type) { } } +} // namespace + class JsonSchemaGenerator : public BaseGenerator { private: std::string code_; @@ -166,9 +176,45 @@ class JsonSchemaGenerator : public BaseGenerator { return std::string(num_spaces, ' '); } + std::string PrepareDescription( + const std::vector<std::string> &comment_lines) { + std::string comment; + for (auto line_iterator = comment_lines.cbegin(); + line_iterator != comment_lines.cend(); ++line_iterator) { + const auto &comment_line = *line_iterator; + + // remove leading and trailing spaces from comment line + const auto start = std::find_if(comment_line.begin(), comment_line.end(), + [](char c) { return !isspace(c); }); + const auto end = + std::find_if(comment_line.rbegin(), comment_line.rend(), [](char c) { + return !isspace(c); + }).base(); + if (start < end) { + comment.append(start, end); + } else { + comment.append(comment_line); + } + + if (line_iterator + 1 != comment_lines.cend()) comment.append("\n"); + } + if (!comment.empty()) { + std::string description; + if (EscapeString(comment.c_str(), comment.length(), &description, true, + true)) { + return description; + } + return ""; + } + return ""; + } + bool generate() { code_ = ""; - if (parser_.root_struct_def_ == nullptr) { return false; } + if (parser_.root_struct_def_ == nullptr) { + std::cerr << "Error: Binary schema not generated, no root struct found\n"; + return false; + } code_ += "{" + NewLine(); code_ += Indent(1) + "\"$schema\": \"https://json-schema.org/draft/2019-09/schema\"," + @@ -193,21 +239,12 @@ class JsonSchemaGenerator : public BaseGenerator { const auto &structure = *s; code_ += Indent(2) + "\"" + GenFullName(structure) + "\" : {" + NewLine(); code_ += Indent(3) + GenType("object") + "," + NewLine(); - std::string comment; const auto &comment_lines = structure->doc_comment; - for (auto comment_line = comment_lines.cbegin(); - comment_line != comment_lines.cend(); ++comment_line) { - comment.append(*comment_line); - } - if (!comment.empty()) { - std::string description; - if (!EscapeString(comment.c_str(), comment.length(), &description, true, - true)) { - return false; - } - code_ += - Indent(3) + "\"description\" : " + description + "," + NewLine(); + auto comment = PrepareDescription(comment_lines); + if (comment != "") { + code_ += Indent(3) + "\"description\" : " + comment + "," + NewLine(); } + code_ += Indent(3) + "\"properties\" : {" + NewLine(); const auto &properties = structure->fields.vec; @@ -223,13 +260,19 @@ class JsonSchemaGenerator : public BaseGenerator { std::string deprecated_info = ""; if (property->deprecated) { deprecated_info = - "," + NewLine() + Indent(8) + "\"deprecated\" : true,"; + "," + NewLine() + Indent(8) + "\"deprecated\" : true"; } std::string typeLine = Indent(4) + "\"" + property->name + "\""; typeLine += " : {" + NewLine() + Indent(8); typeLine += GenType(property->value.type); typeLine += arrayInfo; typeLine += deprecated_info; + auto description = PrepareDescription(property->doc_comment); + if (description != "") { + typeLine += + "," + NewLine() + Indent(8) + "\"description\" : " + description; + } + typeLine += NewLine() + Indent(7) + "}"; if (property != properties.back()) { typeLine.append(","); } code_ += typeLine + NewLine(); @@ -289,4 +332,63 @@ bool GenerateJsonSchema(const Parser &parser, std::string *json) { *json = generator.getJson(); return true; } + +namespace { + +class JsonSchemaCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateJsonSchema(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + (void)parser; + (void)path; + (void)filename; + (void)output; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { + return IDLOptions::kJsonSchema; + } + + std::string LanguageName() const override { return "JsonSchema"; } +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewJsonSchemaCodeGenerator() { + return std::unique_ptr<JsonSchemaCodeGenerator>( + new JsonSchemaCodeGenerator()); +} } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_json_schema.h b/contrib/libs/flatbuffers/src/idl_gen_json_schema.h new file mode 100644 index 00000000000..37a7a095694 --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_json_schema.h @@ -0,0 +1,32 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_JSON_SCHEMA_H_ +#define FLATBUFFERS_IDL_GEN_JSON_SCHEMA_H_ + +#include <memory> +#include <string> + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new JsonSchema Code generator. +std::unique_ptr<CodeGenerator> NewJsonSchemaCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_JSON_SCHEMA_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_kotlin.cpp b/contrib/libs/flatbuffers/src/idl_gen_kotlin.cpp index fb4ce87a67a..eeaca94854c 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_kotlin.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_kotlin.cpp @@ -16,6 +16,8 @@ // independent from idl_parser, since this code is not needed for most clients +#include "idl_gen_kotlin.h" + #include <functional> #include <unordered_set> @@ -23,14 +25,14 @@ #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" -#if defined(FLATBUFFERS_CPP98_STL) -# include <cctype> -#endif // defined(FLATBUFFERS_CPP98_STL) +#include "idl_namer.h" namespace flatbuffers { namespace kotlin { +namespace { + typedef std::map<std::string, std::pair<std::string, std::string> > FbbParamMap; static TypedFloatConstantGenerator KotlinFloatGen("Double.", "Float.", "NaN", "POSITIVE_INFINITY", @@ -38,40 +40,54 @@ static TypedFloatConstantGenerator KotlinFloatGen("Double.", "Float.", "NaN", static const CommentConfig comment_config = { "/**", " *", " */" }; static const std::string ident_pad = " "; -static const char *keywords[] = { - "package", "as", "typealias", "class", "this", "super", - "val", "var", "fun", "for", "null", "true", - "false", "is", "in", "throw", "return", "break", - "continue", "object", "if", "try", "else", "while", - "do", "when", "interface", "typeof", "Any", "Character" -}; - -// Escape Keywords -static std::string Esc(const std::string &name) { - for (size_t i = 0; i < sizeof(keywords) / sizeof(keywords[0]); i++) { - if (name == keywords[i]) { return MakeCamel(name + "_", false); } - } +static std::set<std::string> KotlinKeywords() { + return { "package", "as", "typealias", "class", "this", "super", + "val", "var", "fun", "for", "null", "true", + "false", "is", "in", "throw", "return", "break", + "continue", "object", "if", "try", "else", "while", + "do", "when", "interface", "typeof", "Any", "Character" }; +} - return MakeCamel(name, false); +static Namer::Config KotlinDefaultConfig() { + return { /*types=*/Case::kKeep, + /*constants=*/Case::kKeep, + /*methods=*/Case::kLowerCamel, + /*functions=*/Case::kKeep, + /*fields=*/Case::kLowerCamel, + /*variables=*/Case::kLowerCamel, + /*variants=*/Case::kKeep, + /*enum_variant_seperator=*/"", // I.e. Concatenate. + /*escape_keywords=*/Namer::Config::Escape::BeforeConvertingCase, + /*namespaces=*/Case::kKeep, + /*namespace_seperator=*/"__", + /*object_prefix=*/"", + /*object_suffix=*/"T", + /*keyword_prefix=*/"", + /*keyword_suffix=*/"_", + /*filenames=*/Case::kKeep, + /*directories=*/Case::kKeep, + /*output_path=*/"", + /*filename_suffix=*/"", + /*filename_extension=*/".kt" }; } +} // namespace class KotlinGenerator : public BaseGenerator { public: KotlinGenerator(const Parser &parser, const std::string &path, const std::string &file_name) : BaseGenerator(parser, path, file_name, "", ".", "kt"), - cur_name_space_(nullptr) {} + namer_(WithFlagOptions(KotlinDefaultConfig(), parser.opts, path), + KotlinKeywords()) {} KotlinGenerator &operator=(const KotlinGenerator &); bool generate() FLATBUFFERS_OVERRIDE { std::string one_file_code; - cur_name_space_ = parser_.current_namespace_; for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end(); ++it) { CodeWriter enumWriter(ident_pad); auto &enum_def = **it; - if (!parser_.opts.one_file) cur_name_space_ = enum_def.defined_namespace; GenEnum(enum_def, enumWriter); if (parser_.opts.one_file) { one_file_code += enumWriter.ToString(); @@ -86,8 +102,6 @@ class KotlinGenerator : public BaseGenerator { it != parser_.structs_.vec.end(); ++it) { CodeWriter structWriter(ident_pad); auto &struct_def = **it; - if (!parser_.opts.one_file) - cur_name_space_ = struct_def.defined_namespace; GenStruct(struct_def, structWriter, parser_.opts); if (parser_.opts.one_file) { one_file_code += structWriter.ToString(); @@ -120,19 +134,31 @@ class KotlinGenerator : public BaseGenerator { code += "\n\n"; } if (needs_includes) { - code += "import java.nio.*\n"; - code += "import kotlin.math.sign\n"; - code += "import com.google.flatbuffers.*\n\n"; + code += + "import com.google.flatbuffers.BaseVector\n" + "import com.google.flatbuffers.BooleanVector\n" + "import com.google.flatbuffers.ByteVector\n" + "import com.google.flatbuffers.Constants\n" + "import com.google.flatbuffers.DoubleVector\n" + "import com.google.flatbuffers.FlatBufferBuilder\n" + "import com.google.flatbuffers.FloatVector\n" + "import com.google.flatbuffers.LongVector\n" + "import com.google.flatbuffers.StringVector\n" + "import com.google.flatbuffers.Struct\n" + "import com.google.flatbuffers.Table\n" + "import com.google.flatbuffers.UnionVector\n" + "import java.nio.ByteBuffer\n" + "import java.nio.ByteOrder\n" + "import kotlin.math.sign\n\n"; } code += classcode; - auto filename = NamespaceDir(ns) + defname + ".kt"; + const std::string dirs = namer_.Directories(ns); + EnsureDirExists(dirs); + const std::string filename = + dirs + namer_.File(defname, /*skips=*/SkipFile::Suffix); return SaveFile(filename.c_str(), code, false); } - const Namespace *CurrentNameSpace() const FLATBUFFERS_OVERRIDE { - return cur_name_space_; - } - static bool IsEnum(const Type &type) { return type.enum_def != nullptr && IsInteger(type.base_type); } @@ -167,10 +193,11 @@ class KotlinGenerator : public BaseGenerator { auto r_type = GenTypeGet(field.value.type); if (field.IsScalarOptional() || // string, structs and unions - (base_type == BASE_TYPE_STRING || base_type == BASE_TYPE_STRUCT || - base_type == BASE_TYPE_UNION) || + (!field.IsRequired() && + (base_type == BASE_TYPE_STRING || base_type == BASE_TYPE_STRUCT || + base_type == BASE_TYPE_UNION)) || // vector of anything not scalar - (base_type == BASE_TYPE_VECTOR && + (base_type == BASE_TYPE_VECTOR && !field.IsRequired() && !IsScalar(field.value.type.VectorType().base_type))) { r_type += "?"; } @@ -262,8 +289,7 @@ class KotlinGenerator : public BaseGenerator { GenerateComment(enum_def.doc_comment, writer, &comment_config); writer += "@Suppress(\"unused\")"; - writer += "@ExperimentalUnsignedTypes"; - writer += "class " + Esc(enum_def.name) + " private constructor() {"; + writer += "class " + namer_.Type(enum_def) + " private constructor() {"; writer.IncrementIdentLevel(); GenerateCompanionObject(writer, [&]() { @@ -274,7 +300,7 @@ class KotlinGenerator : public BaseGenerator { auto field_type = GenTypeBasic(enum_def.underlying_type.base_type); auto val = enum_def.ToString(ev); auto suffix = LiteralSuffix(enum_def.underlying_type.base_type); - writer.SetValue("name", Esc(ev.name)); + writer.SetValue("name", namer_.Variant(ev.name)); writer.SetValue("type", field_type); writer.SetValue("val", val + suffix); GenerateComment(ev.doc_comment, writer, &comment_config); @@ -289,7 +315,10 @@ class KotlinGenerator : public BaseGenerator { // Average distance between values above which we consider a table // "too sparse". Change at will. static const uint64_t kMaxSparseness = 5; - if (range / static_cast<uint64_t>(enum_def.size()) < kMaxSparseness) { + bool generate_names = + range / static_cast<uint64_t>(enum_def.size()) < kMaxSparseness && + parser_.opts.mini_reflect == IDLOptions::kTypesAndNames; + if (generate_names) { GeneratePropertyOneLine(writer, "names", "Array<String>", [&]() { writer += "arrayOf(\\"; auto val = enum_def.Vals().front(); @@ -341,7 +370,8 @@ class KotlinGenerator : public BaseGenerator { case BASE_TYPE_UTYPE: return bb_var_name + ".get"; case BASE_TYPE_BOOL: return "0.toByte() != " + bb_var_name + ".get"; default: - return bb_var_name + ".get" + MakeCamel(GenTypeBasic(type.base_type)); + return bb_var_name + "." + + namer_.Method("get", GenTypeBasic(type.base_type)); } } @@ -361,7 +391,8 @@ class KotlinGenerator : public BaseGenerator { case BASE_TYPE_BOOL: case BASE_TYPE_NONE: case BASE_TYPE_UTYPE: return "bb.put"; - default: return "bb.put" + MakeCamel(GenTypeBasic(type.base_type)); + default: + return "bb." + namer_.Method("put", GenTypeBasic(type.base_type)); } } return ""; @@ -384,8 +415,8 @@ class KotlinGenerator : public BaseGenerator { // Recursively generate arguments for a constructor, to deal with nested // structs. - static void GenStructArgs(const StructDef &struct_def, CodeWriter &writer, - const char *nameprefix) { + void GenStructArgs(const StructDef &struct_def, CodeWriter &writer, + const char *nameprefix) const { for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { auto &field = **it; @@ -398,7 +429,7 @@ class KotlinGenerator : public BaseGenerator { (nameprefix + (field.name + "_")).c_str()); } else { writer += std::string(", ") + nameprefix + "\\"; - writer += MakeCamel(field.name) + ": \\"; + writer += namer_.Field(field) + ": \\"; writer += GenTypeBasic(field.value.type.base_type) + "\\"; } } @@ -407,8 +438,8 @@ class KotlinGenerator : public BaseGenerator { // Recusively generate struct construction statements of the form: // builder.putType(name); // and insert manual padding. - static void GenStructBody(const StructDef &struct_def, CodeWriter &writer, - const char *nameprefix) { + void GenStructBody(const StructDef &struct_def, CodeWriter &writer, + const char *nameprefix) const { writer.SetValue("align", NumToString(struct_def.minalign)); writer.SetValue("size", NumToString(struct_def.bytesize)); writer += "builder.prep({{align}}, {{size}})"; @@ -425,7 +456,7 @@ class KotlinGenerator : public BaseGenerator { (nameprefix + (field.name + "_")).c_str()); } else { writer.SetValue("type", GenMethod(field.value.type)); - writer.SetValue("argname", nameprefix + MakeCamel(field.name, false)); + writer.SetValue("argname", nameprefix + namer_.Variable(field)); writer.SetValue("cast", CastToSigned(field.value.type)); writer += "builder.put{{type}}({{argname}}{{cast}})"; } @@ -459,11 +490,10 @@ class KotlinGenerator : public BaseGenerator { GenerateComment(struct_def.doc_comment, writer, &comment_config); auto fixed = struct_def.fixed; - writer.SetValue("struct_name", Esc(struct_def.name)); + writer.SetValue("struct_name", namer_.Type(struct_def)); writer.SetValue("superclass", fixed ? "Struct" : "Table"); writer += "@Suppress(\"unused\")"; - writer += "@ExperimentalUnsignedTypes"; writer += "class {{struct_name}} : {{superclass}}() {\n"; writer.IncrementIdentLevel(); @@ -476,7 +506,7 @@ class KotlinGenerator : public BaseGenerator { // Generate assign method GenerateFun(writer, "__assign", "_i: Int, _bb: ByteBuffer", - Esc(struct_def.name), [&]() { + namer_.Type(struct_def), [&]() { writer += "__init(_i, _bb)"; writer += "return this"; }); @@ -489,15 +519,15 @@ class KotlinGenerator : public BaseGenerator { if (!struct_def.fixed) { FieldDef *key_field = nullptr; - // Generate verson check method. + // Generate version check method. // Force compile time error if not using the same version // runtime. GenerateFunOneLine( writer, "validateVersion", "", "", - [&]() { writer += "Constants.FLATBUFFERS_2_0_0()"; }, + [&]() { writer += "Constants.FLATBUFFERS_23_5_9()"; }, options.gen_jvmstatic); - GenerateGetRootAsAccessors(Esc(struct_def.name), writer, options); + GenerateGetRootAsAccessors(namer_.Type(struct_def), writer, options); GenerateBufferHasIdentifier(struct_def, writer, options); GenerateTableCreator(struct_def, writer, options); @@ -549,7 +579,7 @@ class KotlinGenerator : public BaseGenerator { void GenerateLookupByKey(FieldDef *key_field, StructDef &struct_def, CodeWriter &writer, const IDLOptions options) const { std::stringstream params; - params << "obj: " << Esc(struct_def.name) << "?" + params << "obj: " << namer_.Type(struct_def) << "?" << ", "; params << "vectorLocation: Int, "; params << "key: " << GenTypeGet(key_field->value.type) << ", "; @@ -557,7 +587,7 @@ class KotlinGenerator : public BaseGenerator { auto statements = [&]() { auto base_type = key_field->value.type.base_type; - writer.SetValue("struct_name", Esc(struct_def.name)); + writer.SetValue("struct_name", namer_.Type(struct_def)); if (base_type == BASE_TYPE_STRING) { writer += "val byteKey = key." @@ -603,7 +633,8 @@ class KotlinGenerator : public BaseGenerator { writer += "return null"; }; GenerateFun(writer, "__lookup_by_key", params.str(), - Esc(struct_def.name) + "?", statements, options.gen_jvmstatic); + namer_.Type(struct_def) + "?", statements, + options.gen_jvmstatic); } void GenerateFinishSizePrefixed(StructDef &struct_def, @@ -612,7 +643,8 @@ class KotlinGenerator : public BaseGenerator { const IDLOptions options) const { auto id = identifier.length() > 0 ? ", \"" + identifier + "\"" : ""; auto params = "builder: FlatBufferBuilder, offset: Int"; - auto method_name = "finishSizePrefixed" + Esc(struct_def.name) + "Buffer"; + auto method_name = + namer_.LegacyJavaMethod2("finishSizePrefixed", struct_def, "Buffer"); GenerateFunOneLine( writer, method_name, params, "", [&]() { writer += "builder.finishSizePrefixed(offset" + id + ")"; }, @@ -624,7 +656,8 @@ class KotlinGenerator : public BaseGenerator { const IDLOptions options) const { auto id = identifier.length() > 0 ? ", \"" + identifier + "\"" : ""; auto params = "builder: FlatBufferBuilder, offset: Int"; - auto method_name = "finish" + Esc(struct_def.name) + "Buffer"; + auto method_name = + namer_.LegacyKotlinMethod("finish", struct_def, "Buffer"); GenerateFunOneLine( writer, method_name, params, "", [&]() { writer += "builder.finish(offset" + id + ")"; }, @@ -634,7 +667,7 @@ class KotlinGenerator : public BaseGenerator { void GenerateEndStructMethod(StructDef &struct_def, CodeWriter &writer, const IDLOptions options) const { // Generate end{{TableName}}(builder: FlatBufferBuilder) method - auto name = "end" + Esc(struct_def.name); + auto name = namer_.LegacyJavaMethod2("end", struct_def, ""); auto params = "builder: FlatBufferBuilder"; auto returns = "Int"; auto field_vec = struct_def.fields.vec; @@ -660,7 +693,7 @@ class KotlinGenerator : public BaseGenerator { void GenerateCreateVectorField(FieldDef &field, CodeWriter &writer, const IDLOptions options) const { auto vector_type = field.value.type.VectorType(); - auto method_name = "create" + MakeCamel(Esc(field.name)) + "Vector"; + auto method_name = namer_.Method("create", field, "vector"); auto params = "builder: FlatBufferBuilder, data: " + GenTypeBasic(vector_type.base_type) + "Array"; writer.SetValue("size", NumToString(InlineSize(vector_type))); @@ -668,6 +701,9 @@ class KotlinGenerator : public BaseGenerator { writer.SetValue("root", GenMethod(vector_type)); writer.SetValue("cast", CastToSigned(vector_type)); + if (IsUnsigned(vector_type.base_type)) { + writer += "@kotlin.ExperimentalUnsignedTypes"; + } GenerateFun( writer, method_name, params, "Int", [&]() { @@ -692,8 +728,7 @@ class KotlinGenerator : public BaseGenerator { writer.SetValue("align", NumToString(InlineAlignment(vector_type))); GenerateFunOneLine( - writer, "start" + MakeCamel(Esc(field.name) + "Vector", true), params, - "", + writer, namer_.Method("start", field, "Vector"), params, "", [&]() { writer += "builder.startVector({{size}}, numElems, {{align}})"; }, @@ -703,23 +738,36 @@ class KotlinGenerator : public BaseGenerator { void GenerateAddField(std::string field_pos, FieldDef &field, CodeWriter &writer, const IDLOptions options) const { auto field_type = GenTypeBasic(field.value.type.base_type); - auto secondArg = MakeCamel(Esc(field.name), false) + ": " + field_type; - - GenerateFunOneLine( - writer, "add" + MakeCamel(Esc(field.name), true), - "builder: FlatBufferBuilder, " + secondArg, "", - [&]() { - auto method = GenMethod(field.value.type); - writer.SetValue("field_name", MakeCamel(Esc(field.name), false)); - writer.SetValue("method_name", method); - writer.SetValue("pos", field_pos); - writer.SetValue("default", GenFBBDefaultValue(field)); - writer.SetValue("cast", GenFBBValueCast(field)); - - writer += "builder.add{{method_name}}({{pos}}, \\"; - writer += "{{field_name}}{{cast}}, {{default}})"; - }, - options.gen_jvmstatic); + auto secondArg = namer_.Variable(field.name) + ": " + field_type; + + auto content = [&]() { + auto method = GenMethod(field.value.type); + writer.SetValue("field_name", namer_.Field(field)); + writer.SetValue("method_name", method); + writer.SetValue("pos", field_pos); + writer.SetValue("default", GenFBBDefaultValue(field)); + writer.SetValue("cast", GenFBBValueCast(field)); + if (field.key) { + // field has key attribute, so always need to exist + // even if its value is equal to default. + // Generated code will bypass default checking + // resulting in { builder.addShort(name); slot(id); } + writer += "builder.add{{method_name}}({{field_name}}{{cast}})"; + writer += "builder.slot({{pos}})"; + } else { + writer += "builder.add{{method_name}}({{pos}}, \\"; + writer += "{{field_name}}{{cast}}, {{default}})"; + } + }; + auto signature = namer_.LegacyKotlinMethod("add", field, ""); + auto params = "builder: FlatBufferBuilder, " + secondArg; + if (field.key) { + GenerateFun(writer, signature, params, "", content, + options.gen_jvmstatic); + } else { + GenerateFunOneLine(writer, signature, params, "", content, + options.gen_jvmstatic); + } } static std::string ToSignedType(const Type &type) { @@ -763,7 +811,8 @@ class KotlinGenerator : public BaseGenerator { void GenerateStartStructMethod(StructDef &struct_def, CodeWriter &code, const IDLOptions options) const { GenerateFunOneLine( - code, "start" + Esc(struct_def.name), "builder: FlatBufferBuilder", "", + code, namer_.LegacyJavaMethod2("start", struct_def, ""), + "builder: FlatBufferBuilder", "", [&]() { code += "builder.startTable(" + NumToString(struct_def.fields.vec.size()) + ")"; @@ -795,13 +844,13 @@ class KotlinGenerator : public BaseGenerator { // Generate a table constructor of the form: // public static int createName(FlatBufferBuilder builder, args...) - auto name = "create" + Esc(struct_def.name); + auto name = namer_.LegacyJavaMethod2("create", struct_def, ""); std::stringstream params; params << "builder: FlatBufferBuilder"; for (auto it = fields_vec.begin(); it != fields_vec.end(); ++it) { auto &field = **it; if (field.deprecated) continue; - params << ", " << MakeCamel(Esc(field.name), false); + params << ", " << namer_.Variable(field); if (!IsScalar(field.value.type.base_type)) { params << "Offset: "; } else { @@ -827,17 +876,15 @@ class KotlinGenerator : public BaseGenerator { auto base_type_size = SizeOf(field.value.type.base_type); if (!field.deprecated && (!sortbysize || size == base_type_size)) { - writer.SetValue("camel_field_name", - MakeCamel(Esc(field.name), true)); - writer.SetValue("field_name", - MakeCamel(Esc(field.name), false)); + writer.SetValue("field_name", namer_.Field(field)); // we wrap on null check for scalar optionals writer += field.IsScalarOptional() ? "{{field_name}}?.run { \\" : "\\"; - writer += "add{{camel_field_name}}(builder, {{field_name}}\\"; + writer += namer_.LegacyKotlinMethod("add", field, "") + + "(builder, {{field_name}}\\"; if (!IsScalar(field.value.type.base_type)) { writer += "Offset\\"; } @@ -857,7 +904,7 @@ class KotlinGenerator : public BaseGenerator { // Check if a buffer has the identifier. if (parser_.root_struct_def_ != &struct_def || !file_identifier.length()) return; - auto name = MakeCamel(Esc(struct_def.name), false); + auto name = namer_.Function(struct_def); GenerateFunOneLine( writer, name + "BufferHasIdentifier", "_bb: ByteBuffer", "Boolean", [&]() { @@ -876,7 +923,7 @@ class KotlinGenerator : public BaseGenerator { GenerateComment(field.doc_comment, writer, &comment_config); - auto field_name = MakeCamel(Esc(field.name), false); + auto field_name = namer_.Field(field); auto field_type = GenTypeGet(field.value.type); auto field_default_value = GenDefaultValue(field); auto return_type = GetterReturnType(field); @@ -960,7 +1007,15 @@ class KotlinGenerator : public BaseGenerator { OffsetWrapper( writer, offset_val, [&]() { writer += "obj.__assign({{seek}}, bb)"; }, - [&]() { writer += "null"; }); + [&]() { + if (field.IsRequired()) { + writer += + "throw AssertionError(\"No value for " + "(required) field {{field_name}}\")"; + } else { + writer += "null"; + } + }); }); } break; @@ -970,12 +1025,30 @@ class KotlinGenerator : public BaseGenerator { // val Name : String? // get() = { // val o = __offset(10) - // return if (o != 0) __string(o + bb_pos) else null + // return if (o != 0) { + // __string(o + bb_pos) + // } else { + // null + // } // } // ? adds nullability annotation GenerateGetter(writer, field_name, return_type, [&]() { writer += "val o = __offset({{offset}})"; - writer += "return if (o != 0) __string(o + bb_pos) else null"; + writer += "return if (o != 0) {"; + writer.IncrementIdentLevel(); + writer += "__string(o + bb_pos)"; + writer.DecrementIdentLevel(); + writer += "} else {"; + writer.IncrementIdentLevel(); + if (field.IsRequired()) { + writer += + "throw AssertionError(\"No value for (required) field " + "{{field_name}}\")"; + } else { + writer += "null"; + } + writer.DecrementIdentLevel(); + writer += "}"; }); break; case BASE_TYPE_VECTOR: { @@ -1000,7 +1073,11 @@ class KotlinGenerator : public BaseGenerator { GenerateFun(writer, field_name, params, return_type, [&]() { auto inline_size = NumToString(InlineSize(vectortype)); auto index = "__vector(o) + j * " + inline_size; - auto not_found = NotFoundReturn(field.value.type.element); + auto not_found = + field.IsRequired() + ? "throw IndexOutOfBoundsException(\"Index out of range: " + "$j, vector {{field_name}} is empty\")" + : NotFoundReturn(field.value.type.element); auto found = ""; writer.SetValue("index", index); switch (vectortype.base_type) { @@ -1047,7 +1124,7 @@ class KotlinGenerator : public BaseGenerator { auto &kfield = **kit; if (kfield.key) { auto qualified_name = WrapInNameSpace(sd); - auto name = MakeCamel(Esc(field.name), false) + "ByKey"; + auto name = namer_.Method(field, "ByKey"); auto params = "key: " + GenTypeGet(kfield.value.type); auto rtype = qualified_name + "?"; GenerateFun(writer, name, params, rtype, [&]() { @@ -1140,9 +1217,9 @@ class KotlinGenerator : public BaseGenerator { auto underlying_type = value_base_type == BASE_TYPE_VECTOR ? value_type.VectorType() : value_type; - auto name = "mutate" + MakeCamel(Esc(field.name), true); + auto name = namer_.LegacyKotlinMethod("mutate", field, ""); auto size = NumToString(InlineSize(underlying_type)); - auto params = Esc(field.name) + ": " + GenTypeGet(underlying_type); + auto params = namer_.Field(field) + ": " + GenTypeGet(underlying_type); // A vector mutator also needs the index of the vector element it should // mutate. if (value_base_type == BASE_TYPE_VECTOR) params.insert(0, "j: Int, "); @@ -1151,8 +1228,8 @@ class KotlinGenerator : public BaseGenerator { // representation. auto setter_parameter = underlying_type.base_type == BASE_TYPE_BOOL - ? "(if(" + Esc(field.name) + ") 1 else 0).toByte()" - : Esc(field.name); + ? "(if(" + namer_.Field(field) + ") 1 else 0).toByte()" + : namer_.Field(field); auto setter_index = value_base_type == BASE_TYPE_VECTOR @@ -1285,9 +1362,9 @@ class KotlinGenerator : public BaseGenerator { } } - static void GenerateGetRootAsAccessors(const std::string &struct_name, - CodeWriter &writer, - IDLOptions options) { + void GenerateGetRootAsAccessors(const std::string &struct_name, + CodeWriter &writer, + IDLOptions options) const { // Generate a special accessor for the table that when used as the root // ex: fun getRootAsMonster(_bb: ByteBuffer): Monster {...} writer.SetValue("gr_name", struct_name); @@ -1313,13 +1390,12 @@ class KotlinGenerator : public BaseGenerator { writer += "}"; } - static void GenerateStaticConstructor(const StructDef &struct_def, - CodeWriter &code, - const IDLOptions options) { + void GenerateStaticConstructor(const StructDef &struct_def, CodeWriter &code, + const IDLOptions options) const { // create a struct constructor function auto params = StructConstructorParams(struct_def); GenerateFun( - code, "create" + Esc(struct_def.name), params, "Int", + code, namer_.LegacyJavaMethod2("create", struct_def, ""), params, "Int", [&]() { GenStructBody(struct_def, code, ""); code += "return builder.offset()"; @@ -1327,8 +1403,8 @@ class KotlinGenerator : public BaseGenerator { options.gen_jvmstatic); } - static std::string StructConstructorParams(const StructDef &struct_def, - const std::string &prefix = "") { + std::string StructConstructorParams(const StructDef &struct_def, + const std::string &prefix = "") const { // builder: FlatBufferBuilder std::stringstream out; auto field_vec = struct_def.fields.vec; @@ -1341,9 +1417,9 @@ class KotlinGenerator : public BaseGenerator { // constructing a nested struct, prefix the name with the field // name. out << StructConstructorParams(*field.value.type.struct_def, - prefix + (Esc(field.name) + "_")); + prefix + (namer_.Variable(field) + "_")); } else { - out << ", " << prefix << MakeCamel(Esc(field.name), false) << ": " + out << ", " << prefix << namer_.Variable(field) << ": " << GenTypeBasic(field.value.type.base_type); } } @@ -1513,9 +1589,7 @@ class KotlinGenerator : public BaseGenerator { if (gen_jvmstatic) { code += "@JvmStatic"; } } - // This tracks the current namespace used to determine if a type need to be - // prefixed by its namespace - const Namespace *cur_name_space_; + const IdlNamer namer_; }; } // namespace kotlin @@ -1524,4 +1598,61 @@ bool GenerateKotlin(const Parser &parser, const std::string &path, kotlin::KotlinGenerator generator(parser, path, file_name); return generator.generate(); } + +namespace { + +class KotlinCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateKotlin(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + (void)parser; + (void)path; + (void)filename; + (void)output; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kKotlin; } + + std::string LanguageName() const override { return "Kotlin"; } +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewKotlinCodeGenerator() { + return std::unique_ptr<KotlinCodeGenerator>(new KotlinCodeGenerator()); +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_kotlin.h b/contrib/libs/flatbuffers/src/idl_gen_kotlin.h new file mode 100644 index 00000000000..22d8ff6ca3e --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_kotlin.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_KOTLIN_H_ +#define FLATBUFFERS_IDL_GEN_KOTLIN_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Kotlin code generator. +std::unique_ptr<CodeGenerator> NewKotlinCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_KOTLIN_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_lobster.cpp b/contrib/libs/flatbuffers/src/idl_gen_lobster.cpp index 6fdd6dc26ac..a8b0a6f7a5e 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_lobster.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_lobster.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "idl_gen_lobster.h" + #include <string> #include <unordered_set> @@ -60,7 +62,10 @@ class LobsterGenerator : public BaseGenerator { std::string GenTypeName(const Type &type) { auto bits = NumToString(SizeOf(type.base_type) * 8); - if (IsInteger(type.base_type)) return "int" + bits; + if (IsInteger(type.base_type)) { + if (IsUnsigned(type.base_type)) return "uint" + bits; + else return "int" + bits; + } if (IsFloat(type.base_type)) return "float" + bits; if (IsString(type)) return "string"; if (type.base_type == BASE_TYPE_STRUCT) return "table"; @@ -69,16 +74,18 @@ class LobsterGenerator : public BaseGenerator { std::string LobsterType(const Type &type) { if (IsFloat(type.base_type)) return "float"; + if (IsBool(type.base_type)) return "bool"; if (IsScalar(type.base_type) && type.enum_def) return NormalizedName(*type.enum_def); if (!IsScalar(type.base_type)) return "flatbuffers_offset"; + if (IsString(type)) return "string"; return "int"; } // Returns the method name for use with add/put calls. std::string GenMethod(const Type &type) { return IsScalar(type.base_type) - ? MakeCamel(GenTypeBasic(type)) + ? ConvertCase(GenTypeBasic(type), Case::kUpperCamel) : (IsStruct(type) ? "Struct" : "UOffsetTRelative"); } @@ -113,21 +120,27 @@ class LobsterGenerator : public BaseGenerator { auto defval = field.IsOptional() ? "0" : field.value.constant; acc = "buf_.flatbuffers_field_" + GenTypeName(field.value.type) + "(pos_, " + offsets + ", " + defval + ")"; + if (IsBool(field.value.type.base_type)) + acc = "bool(" + acc + ")"; } if (field.value.type.enum_def) acc = NormalizedName(*field.value.type.enum_def) + "(" + acc + ")"; - if (field.IsOptional()) + if (field.IsOptional()) { acc += ", buf_.flatbuffers_field_present(pos_, " + offsets + ")"; - code += def + "():\n return " + acc + "\n"; + code += def + "() -> " + LobsterType(field.value.type) + ", bool:\n return " + acc + "\n"; + } else { + code += def + "() -> " + LobsterType(field.value.type) + ":\n return " + acc + "\n"; + } return; } switch (field.value.type.base_type) { case BASE_TYPE_STRUCT: { auto name = NamespacedName(*field.value.type.struct_def); - code += def + "():\n "; if (struct_def.fixed) { + code += def + "() -> " + name + ":\n "; code += "return " + name + "{ buf_, pos_ + " + offsets + " }\n"; } else { + code += def + "() -> " + name + "?:\n "; code += std::string("let o = buf_.flatbuffers_field_") + (field.value.type.struct_def->fixed ? "struct" : "table") + "(pos_, " + offsets + ")\n return if o: " + name + @@ -137,25 +150,28 @@ class LobsterGenerator : public BaseGenerator { } case BASE_TYPE_STRING: code += def + - "():\n return buf_.flatbuffers_field_string(pos_, " + + "() -> string:\n return buf_.flatbuffers_field_string(pos_, " + offsets + ")\n"; break; case BASE_TYPE_VECTOR: { auto vectortype = field.value.type.VectorType(); - code += def + "(i:int):\n return "; if (vectortype.base_type == BASE_TYPE_STRUCT) { auto start = "buf_.flatbuffers_field_vector(pos_, " + offsets + ") + i * " + NumToString(InlineSize(vectortype)); if (!(vectortype.struct_def->fixed)) { start = "buf_.flatbuffers_indirect(" + start + ")"; } + code += def + "(i:int) -> " + NamespacedName(*field.value.type.struct_def) + ":\n return "; code += NamespacedName(*field.value.type.struct_def) + " { buf_, " + start + " }\n"; } else { - if (IsString(vectortype)) + if (IsString(vectortype)) { + code += def + "(i:int) -> string:\n return "; code += "buf_.flatbuffers_string"; - else + } else { + code += def + "(i:int) -> " + LobsterType(vectortype) + ":\n return "; code += "buf_.read_" + GenTypeName(vectortype) + "_le"; + } code += "(buf_.flatbuffers_field_vector(pos_, " + offsets + ") + i * " + NumToString(InlineSize(vectortype)) + ")\n"; } @@ -178,7 +194,7 @@ class LobsterGenerator : public BaseGenerator { } if (IsVector(field.value.type)) { code += def + - "_length():\n return " + "_length() -> int:\n return " "buf_.flatbuffers_field_vector_len(pos_, " + offsets + ")\n"; } @@ -212,7 +228,7 @@ class LobsterGenerator : public BaseGenerator { if (field.deprecated) continue; if (IsVector(field.value.type)) { code += "def " + NormalizedName(struct_def) + "Start" + - MakeCamel(NormalizedName(field)) + + ConvertCase(NormalizedName(field), Case::kUpperCamel) + "Vector(b_:flatbuffers_builder, n_:int):\n b_.StartVector("; auto vector_type = field.value.type.VectorType(); auto alignment = InlineAlignment(vector_type); @@ -222,7 +238,7 @@ class LobsterGenerator : public BaseGenerator { if (vector_type.base_type != BASE_TYPE_STRUCT || !vector_type.struct_def->fixed) { code += "def " + NormalizedName(struct_def) + "Create" + - MakeCamel(NormalizedName(field)) + + ConvertCase(NormalizedName(field), Case::kUpperCamel) + "Vector(b_:flatbuffers_builder, v_:[" + LobsterType(vector_type) + "]):\n b_.StartVector(" + NumToString(elem_size) + ", v_.length, " + @@ -388,4 +404,63 @@ bool GenerateLobster(const Parser &parser, const std::string &path, return generator.generate(); } +namespace { + +class LobsterCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateLobster(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + (void)parser; + (void)path; + (void)filename; + (void)output; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { + return IDLOptions::kLobster; + } + + std::string LanguageName() const override { return "Lobster"; } +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewLobsterCodeGenerator() { + return std::unique_ptr<LobsterCodeGenerator>(new LobsterCodeGenerator()); +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_lobster.h b/contrib/libs/flatbuffers/src/idl_gen_lobster.h new file mode 100644 index 00000000000..284303edcec --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_lobster.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_LOBSTER_H_ +#define FLATBUFFERS_IDL_GEN_LOBSTER_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Lobster code generator. +std::unique_ptr<CodeGenerator> NewLobsterCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_LOBSTER_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_lua.cpp b/contrib/libs/flatbuffers/src/idl_gen_lua.cpp index 9efc435e24f..551a4b26f8a 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_lua.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_lua.cpp @@ -16,6 +16,8 @@ // independent from idl_parser, since this code is not needed for most clients +#include "idl_gen_lua.h" + #include <string> #include <unordered_set> @@ -154,7 +156,8 @@ class LuaGenerator : public BaseGenerator { std::string &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)) + "Length()\n"; + code += + ConvertCase(NormalizedName(field), Case::kUpperCamel) + "Length()\n"; code += OffsetPrefix(field); code += std::string(Indent) + Indent + "return " + SelfData + ":VectorLen(o)\n"; @@ -169,7 +172,7 @@ class LuaGenerator : public BaseGenerator { std::string &code = *code_ptr; std::string getter = GenGetter(field.value.type); GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); + code += ConvertCase(NormalizedName(field), Case::kUpperCamel); code += "()\n"; code += std::string(Indent) + "return " + getter; code += std::string(SelfDataPos) + " + " + NumToString(field.value.offset) + @@ -183,7 +186,7 @@ class LuaGenerator : public BaseGenerator { std::string &code = *code_ptr; std::string getter = GenGetter(field.value.type); GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); + code += ConvertCase(NormalizedName(field), Case::kUpperCamel); code += "()\n"; code += OffsetPrefix(field); getter += std::string("o + ") + SelfDataPos + ")"; @@ -207,7 +210,7 @@ class LuaGenerator : public BaseGenerator { const FieldDef &field, std::string *code_ptr) { std::string &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); + code += ConvertCase(NormalizedName(field), Case::kUpperCamel); code += "(obj)\n"; code += std::string(Indent) + "obj:Init(" + SelfDataBytes + ", " + SelfDataPos + " + "; @@ -222,7 +225,7 @@ class LuaGenerator : public BaseGenerator { std::string *code_ptr) { std::string &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); + code += ConvertCase(NormalizedName(field), Case::kUpperCamel); code += "()\n"; code += OffsetPrefix(field); if (field.value.type.struct_def->fixed) { @@ -246,7 +249,7 @@ class LuaGenerator : public BaseGenerator { std::string *code_ptr) { std::string &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); + code += ConvertCase(NormalizedName(field), Case::kUpperCamel); code += "()\n"; code += OffsetPrefix(field); code += @@ -261,7 +264,7 @@ class LuaGenerator : public BaseGenerator { std::string *code_ptr) { std::string &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)) + "()\n"; + code += ConvertCase(NormalizedName(field), Case::kUpperCamel) + "()\n"; code += OffsetPrefix(field); // TODO(rw): this works and is not the good way to it: @@ -292,7 +295,7 @@ class LuaGenerator : public BaseGenerator { auto vectortype = field.value.type.VectorType(); GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); + code += ConvertCase(NormalizedName(field), Case::kUpperCamel); code += "(j)\n"; code += OffsetPrefix(field); code += @@ -321,7 +324,7 @@ class LuaGenerator : public BaseGenerator { auto vectortype = field.value.type.VectorType(); GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); + code += ConvertCase(NormalizedName(field), Case::kUpperCamel); code += "(j)\n"; code += OffsetPrefix(field); code += @@ -344,7 +347,7 @@ class LuaGenerator : public BaseGenerator { const FieldDef &field, std::string *code_ptr) { std::string &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); + code += ConvertCase(NormalizedName(field), Case::kUpperCamel); code += "AsString(start, stop)\n"; code += std::string(Indent) + "return " + SelfData + ":VectorAsString(" + NumToString(field.value.offset) + ", start, stop)\n"; @@ -377,7 +380,7 @@ class LuaGenerator : public BaseGenerator { } else { std::string &code = *code_ptr; code += std::string(", ") + nameprefix; - code += MakeCamel(NormalizedName(field), false); + code += ConvertCase(NormalizedName(field), Case::kLowerCamel); } } } @@ -409,7 +412,8 @@ class LuaGenerator : public BaseGenerator { } else { code += std::string(Indent) + "builder:Prepend" + GenMethod(field) + "("; - code += nameprefix + MakeCamel(NormalizedName(field), false) + ")\n"; + code += nameprefix + + ConvertCase(NormalizedName(field), Case::kLowerCamel) + ")\n"; } } } @@ -435,9 +439,9 @@ class LuaGenerator : public BaseGenerator { const size_t offset, std::string *code_ptr) { std::string &code = *code_ptr; code += "function " + NormalizedName(struct_def) + ".Add" + - MakeCamel(NormalizedName(field)); + ConvertCase(NormalizedName(field), Case::kUpperCamel); code += "(builder, "; - code += MakeCamel(NormalizedName(field), false); + code += ConvertCase(NormalizedName(field), Case::kLowerCamel); code += ") "; code += "builder:Prepend"; code += GenMethod(field) + "Slot("; @@ -446,9 +450,9 @@ class LuaGenerator : public BaseGenerator { // if (!IsScalar(field.value.type.base_type) && (!struct_def.fixed)) { // code += "flatbuffers.N.UOffsetTFlags.py_type"; // code += "("; - // code += MakeCamel(NormalizedName(field), false) + ")"; + // code += ConvertCase(NormalizedName(field), Case::kLowerCamel) + ")"; // } else { - code += MakeCamel(NormalizedName(field), false); + code += ConvertCase(NormalizedName(field), Case::kLowerCamel); // } code += ", " + field.value.constant; code += ") end\n"; @@ -459,7 +463,7 @@ class LuaGenerator : public BaseGenerator { std::string *code_ptr) { std::string &code = *code_ptr; code += "function " + NormalizedName(struct_def) + ".Start"; - code += MakeCamel(NormalizedName(field)); + code += ConvertCase(NormalizedName(field), Case::kUpperCamel); code += "Vector(builder, numElems) return builder:StartVector("; auto vector_type = field.value.type.VectorType(); auto alignment = InlineAlignment(vector_type); @@ -603,14 +607,14 @@ class LuaGenerator : public BaseGenerator { case BASE_TYPE_VECTOR: return GenGetter(type.VectorType()); default: return std::string(SelfData) + ":Get(flatbuffers.N." + - MakeCamel(GenTypeGet(type)) + ", "; + ConvertCase(GenTypeGet(type), Case::kUpperCamel) + ", "; } } // Returns the method name for use with add/put calls. std::string GenMethod(const FieldDef &field) { return IsScalar(field.value.type.base_type) - ? MakeCamel(GenTypeBasic(field.value.type)) + ? ConvertCase(GenTypeBasic(field.value.type), Case::kUpperCamel) : (IsStruct(field.value.type) ? "Struct" : "UOffsetTRelative"); } @@ -742,4 +746,61 @@ bool GenerateLua(const Parser &parser, const std::string &path, return generator.generate(); } +namespace { + +class LuaCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateLua(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + (void)parser; + (void)path; + (void)filename; + (void)output; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return true; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kLua; } + + std::string LanguageName() const override { return "Lua"; } +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewLuaCodeGenerator() { + return std::unique_ptr<LuaCodeGenerator>(new LuaCodeGenerator()); +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_lua.h b/contrib/libs/flatbuffers/src/idl_gen_lua.h new file mode 100644 index 00000000000..43974a8c33a --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_lua.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_LUA_H_ +#define FLATBUFFERS_IDL_GEN_LUA_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Lua code generator. +std::unique_ptr<CodeGenerator> NewLuaCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_LUA_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_php.cpp b/contrib/libs/flatbuffers/src/idl_gen_php.cpp index dd3ed681890..a245b1a7c3a 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_php.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_php.cpp @@ -16,6 +16,8 @@ // independent from idl_parser, since this code is not needed for most clients +#include "idl_gen_php.h" + #include <string> #include "flatbuffers/code_generators.h" @@ -181,7 +183,7 @@ class PhpGenerator : public BaseGenerator { code += Indent + " * @return int\n"; code += Indent + " */\n"; code += Indent + "public function get"; - code += MakeCamel(field.name) + "Length()\n"; + code += ConvertCase(field.name, Case::kUpperCamel) + "Length()\n"; code += Indent + "{\n"; code += Indent + Indent + "$o = $this->__offset("; code += NumToString(field.value.offset) + ");\n"; @@ -198,7 +200,7 @@ class PhpGenerator : public BaseGenerator { code += Indent + " * @return string\n"; code += Indent + " */\n"; code += Indent + "public function get"; - code += MakeCamel(field.name) + "Bytes()\n"; + code += ConvertCase(field.name, Case::kUpperCamel) + "Bytes()\n"; code += Indent + "{\n"; code += Indent + Indent + "return $this->__vector_as_bytes("; code += NumToString(field.value.offset) + ");\n"; @@ -216,12 +218,12 @@ class PhpGenerator : public BaseGenerator { code += GenTypeGet(field.value.type) + "\n"; code += Indent + " */\n"; code += Indent + "public function " + getter; - code += MakeCamel(field.name) + "()\n"; + code += ConvertCase(field.name, Case::kUpperCamel) + "()\n"; code += Indent + "{\n"; code += Indent + Indent + "return "; code += "$this->bb->get"; - code += MakeCamel(GenTypeGet(field.value.type)); + code += ConvertCase(GenTypeGet(field.value.type), Case::kUpperCamel); code += "($this->bb_pos + "; code += NumToString(field.value.offset) + ")"; code += ";\n"; @@ -237,14 +239,15 @@ class PhpGenerator : public BaseGenerator { code += Indent + " * @return " + GenTypeGet(field.value.type) + "\n"; code += Indent + " */\n"; code += Indent + "public function get"; - code += MakeCamel(field.name); + code += ConvertCase(field.name, Case::kUpperCamel); code += "()\n"; code += Indent + "{\n"; code += Indent + Indent + "$o = $this->__offset(" + NumToString(field.value.offset) + ");\n" + Indent + Indent + "return $o != 0 ? "; code += "$this->bb->get"; - code += MakeCamel(GenTypeGet(field.value.type)) + "($o + $this->bb_pos)"; + code += ConvertCase(GenTypeGet(field.value.type), Case::kUpperCamel) + + "($o + $this->bb_pos)"; code += " : " + GenDefaultValue(field.value) + ";\n"; code += Indent + "}\n\n"; } @@ -258,7 +261,7 @@ class PhpGenerator : public BaseGenerator { code += Indent + " * @return " + GenTypeGet(field.value.type) + "\n"; code += Indent + " */\n"; code += Indent + "public function get"; - code += MakeCamel(field.name) + "()\n"; + code += ConvertCase(field.name, Case::kUpperCamel) + "()\n"; code += Indent + "{\n"; code += Indent + Indent + "$obj = new "; code += GenTypeGet(field.value.type) + "();\n"; @@ -274,11 +277,12 @@ class PhpGenerator : public BaseGenerator { std::string &code = *code_ptr; code += Indent + "public function get"; - code += MakeCamel(field.name); + code += ConvertCase(field.name, Case::kUpperCamel); code += "()\n"; code += Indent + "{\n"; code += Indent + Indent + "$obj = new "; - code += MakeCamel(GenTypeGet(field.value.type)) + "();\n"; + code += + ConvertCase(GenTypeGet(field.value.type), Case::kUpperCamel) + "();\n"; code += Indent + Indent + "$o = $this->__offset(" + NumToString(field.value.offset) + ");\n"; code += Indent + Indent; @@ -296,7 +300,7 @@ class PhpGenerator : public BaseGenerator { void GetStringField(const FieldDef &field, std::string *code_ptr) { std::string &code = *code_ptr; code += Indent + "public function get"; - code += MakeCamel(field.name); + code += ConvertCase(field.name, Case::kUpperCamel); code += "()\n"; code += Indent + "{\n"; code += Indent + Indent + "$o = $this->__offset(" + @@ -315,7 +319,7 @@ class PhpGenerator : public BaseGenerator { code += Indent + " * @return" + GenTypeBasic(field.value.type) + "\n"; code += Indent + " */\n"; code += Indent + "public function get"; - code += MakeCamel(field.name) + "($obj)\n"; + code += ConvertCase(field.name, Case::kUpperCamel) + "($obj)\n"; code += Indent + "{\n"; code += Indent + Indent + "$o = $this->__offset(" + NumToString(field.value.offset) + ");\n"; @@ -334,13 +338,14 @@ class PhpGenerator : public BaseGenerator { code += Indent + " * @return" + GenTypeBasic(field.value.type) + "\n"; code += Indent + " */\n"; code += Indent + "public function get"; - code += MakeCamel(field.name); + code += ConvertCase(field.name, Case::kUpperCamel); code += "($j)\n"; code += Indent + "{\n"; code += Indent + Indent + "$o = $this->__offset(" + NumToString(field.value.offset) + ");\n"; code += Indent + Indent + "$obj = new "; - code += MakeCamel(GenTypeGet(field.value.type)) + "();\n"; + code += + ConvertCase(GenTypeGet(field.value.type), Case::kUpperCamel) + "();\n"; switch (field.value.type.base_type) { case BASE_TYPE_STRUCT: @@ -395,7 +400,7 @@ class PhpGenerator : public BaseGenerator { code += Indent + " * @return " + GenTypeGet(field.value.type) + "\n"; code += Indent + " */\n"; code += Indent + "public function get"; - code += MakeCamel(field.name); + code += ConvertCase(field.name, Case::kUpperCamel); code += "($j)\n"; code += Indent + "{\n"; code += Indent + Indent + "$o = $this->__offset(" + @@ -408,7 +413,7 @@ class PhpGenerator : public BaseGenerator { code += GenDefaultValue(field.value) + ";\n"; } else { code += Indent + Indent + "return $o != 0 ? $this->bb->get"; - code += MakeCamel(GenTypeGet(field.value.type)); + code += ConvertCase(GenTypeGet(field.value.type), Case::kUpperCamel); code += "($this->__vector($o) + $j * "; code += NumToString(InlineSize(vectortype)) + ") : "; code += GenDefaultValue(field.value) + ";\n"; @@ -427,7 +432,7 @@ class PhpGenerator : public BaseGenerator { code += Indent + " * @return " + GenTypeGet(field.value.type) + "\n"; code += Indent + " */\n"; code += Indent + "public function get"; - code += MakeCamel(field.name); + code += ConvertCase(field.name, Case::kUpperCamel); code += "($j, $obj)\n"; code += Indent + "{\n"; code += Indent + Indent + "$o = $this->__offset(" + @@ -455,7 +460,7 @@ class PhpGenerator : public BaseGenerator { } else { std::string &code = *code_ptr; code += std::string(", $") + nameprefix; - code += MakeCamel(field.name, false); + code += ConvertCase(field.name, Case::kLowerCamel); } } } @@ -480,7 +485,8 @@ class PhpGenerator : public BaseGenerator { (nameprefix + (field.name + "_")).c_str(), code_ptr); } else { code += Indent + Indent + "$builder->put" + GenMethod(field) + "($"; - code += nameprefix + MakeCamel(field.name, false) + ");\n"; + code += + nameprefix + ConvertCase(field.name, Case::kLowerCamel) + ");\n"; } } } @@ -528,7 +534,8 @@ class PhpGenerator : public BaseGenerator { if (field.deprecated) continue; code += Indent + Indent + "self::add"; - code += MakeCamel(field.name) + "($builder, $" + field.name + ");\n"; + code += ConvertCase(field.name, Case::kUpperCamel) + "($builder, $" + + field.name + ");\n"; } code += Indent + Indent + "$o = $builder->endObject();\n"; @@ -557,16 +564,16 @@ class PhpGenerator : public BaseGenerator { code += Indent + " * @return void\n"; code += Indent + " */\n"; code += Indent + "public static function "; - code += "add" + MakeCamel(field.name); + code += "add" + ConvertCase(field.name, Case::kUpperCamel); code += "(FlatBufferBuilder $builder, "; - code += "$" + MakeCamel(field.name, false); + code += "$" + ConvertCase(field.name, Case::kLowerCamel); code += ")\n"; code += Indent + "{\n"; code += Indent + Indent + "$builder->add"; code += GenMethod(field) + "X("; code += NumToString(offset) + ", "; - code += "$" + MakeCamel(field.name, false); + code += "$" + ConvertCase(field.name, Case::kLowerCamel); code += ", "; if (field.value.type.base_type == BASE_TYPE_BOOL) { @@ -591,7 +598,7 @@ class PhpGenerator : public BaseGenerator { code += Indent + " * @return int vector offset\n"; code += Indent + " */\n"; code += Indent + "public static function create"; - code += MakeCamel(field.name); + code += ConvertCase(field.name, Case::kUpperCamel); code += "Vector(FlatBufferBuilder $builder, array $data)\n"; code += Indent + "{\n"; code += Indent + Indent + "$builder->startVector("; @@ -603,7 +610,8 @@ class PhpGenerator : public BaseGenerator { if (IsScalar(field.value.type.VectorType().base_type)) { code += Indent + Indent + Indent; code += "$builder->put"; - code += MakeCamel(GenTypeBasic(field.value.type.VectorType())); + code += ConvertCase(GenTypeBasic(field.value.type.VectorType()), + Case::kUpperCamel); code += "($data[$i]);\n"; } else { code += Indent + Indent + Indent; @@ -619,7 +627,7 @@ class PhpGenerator : public BaseGenerator { code += Indent + " * @return void\n"; code += Indent + " */\n"; code += Indent + "public static function start"; - code += MakeCamel(field.name); + code += ConvertCase(field.name, Case::kUpperCamel); code += "Vector(FlatBufferBuilder $builder, $numElems)\n"; code += Indent + "{\n"; code += Indent + Indent + "$builder->startVector("; @@ -726,7 +734,7 @@ class PhpGenerator : public BaseGenerator { if (field.value.type.base_type == BASE_TYPE_UNION) { std::string &code = *code_ptr; code += Indent + "public static function add"; - code += MakeCamel(field.name); + code += ConvertCase(field.name, Case::kUpperCamel); code += "(FlatBufferBuilder $builder, $offset)\n"; code += Indent + "{\n"; code += Indent + Indent + "$builder->addOffsetX("; @@ -853,7 +861,7 @@ class PhpGenerator : public BaseGenerator { // Returns the method name for use with add/put calls. static std::string GenMethod(const FieldDef &field) { return IsScalar(field.value.type.base_type) - ? MakeCamel(GenTypeBasic(field.value.type)) + ? ConvertCase(GenTypeBasic(field.value.type), Case::kUpperCamel) : (IsStruct(field.value.type) ? "Struct" : "Offset"); } @@ -936,4 +944,62 @@ bool GeneratePhp(const Parser &parser, const std::string &path, php::PhpGenerator generator(parser, path, file_name); return generator.generate(); } + +namespace { + +class PhpCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GeneratePhp(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + (void)parser; + (void)path; + (void)filename; + (void)output; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kPhp; } + + std::string LanguageName() const override { return "Php"; } +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewPhpCodeGenerator() { + return std::unique_ptr<PhpCodeGenerator>(new PhpCodeGenerator()); +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_php.h b/contrib/libs/flatbuffers/src/idl_gen_php.h new file mode 100644 index 00000000000..8695ec99361 --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_php.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_PHP_H_ +#define FLATBUFFERS_IDL_GEN_PHP_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Php code generator. +std::unique_ptr<CodeGenerator> NewPhpCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_PHP_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_python.cpp b/contrib/libs/flatbuffers/src/idl_gen_python.cpp index b3f394ebf9e..dc901bb7c48 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_python.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_python.cpp @@ -16,6 +16,8 @@ // independent from idl_parser, since this code is not needed for most clients +#include "idl_gen_python.h" + #include <cctype> #include <set> #include <string> @@ -26,13 +28,53 @@ #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" +#include "idl_namer.h" namespace flatbuffers { namespace python { +namespace { + +typedef std::pair<std::string, std::string> ImportMapEntry; +typedef std::set<ImportMapEntry> ImportMap; + +static std::set<std::string> PythonKeywords() { + return { "False", "None", "True", "and", "as", "assert", + "break", "class", "continue", "def", "del", "elif", + "else", "except", "finally", "for", "from", "global", + "if", "import", "in", "is", "lambda", "nonlocal", + "not", "or", "pass", "raise", "return", "try", + "while", "with", "yield" }; +} + +static Namer::Config PythonDefaultConfig() { + return { /*types=*/Case::kKeep, + /*constants=*/Case::kScreamingSnake, + /*methods=*/Case::kUpperCamel, + /*functions=*/Case::kUpperCamel, + /*fields=*/Case::kLowerCamel, + /*variable=*/Case::kLowerCamel, + /*variants=*/Case::kKeep, + /*enum_variant_seperator=*/".", + /*escape_keywords=*/Namer::Config::Escape::BeforeConvertingCase, + /*namespaces=*/Case::kKeep, // Packages in python. + /*namespace_seperator=*/".", + /*object_prefix=*/"", + /*object_suffix=*/"T", + /*keyword_prefix=*/"", + /*keyword_suffix=*/"_", + /*filenames=*/Case::kKeep, + /*directories=*/Case::kKeep, + /*output_path=*/"", + /*filename_suffix=*/"", + /*filename_extension=*/".py" }; +} + // Hardcode spaces per indentation. -const CommentConfig def_comment = { nullptr, "#", nullptr }; -const std::string Indent = " "; +static const CommentConfig def_comment = { nullptr, "#", nullptr }; +static const std::string Indent = " "; + +} // namespace class PythonGenerator : public BaseGenerator { public: @@ -40,155 +82,147 @@ class PythonGenerator : public BaseGenerator { const std::string &file_name) : BaseGenerator(parser, path, file_name, "" /* not used */, "" /* not used */, "py"), - float_const_gen_("float('nan')", "float('inf')", "float('-inf')") { - static const char *const keywords[] = { - "False", "None", "True", "and", "as", "assert", "break", - "class", "continue", "def", "del", "elif", "else", "except", - "finally", "for", "from", "global", "if", "import", "in", - "is", "lambda", "nonlocal", "not", "or", "pass", "raise", - "return", "try", "while", "with", "yield" - }; - keywords_.insert(std::begin(keywords), std::end(keywords)); - } + float_const_gen_("float('nan')", "float('inf')", "float('-inf')"), + namer_(WithFlagOptions(PythonDefaultConfig(), parser.opts, path), + PythonKeywords()) {} // Most field accessors need to retrieve and test the field offset first, // this is the prefix code for that. - std::string OffsetPrefix(const FieldDef &field) { + std::string OffsetPrefix(const FieldDef &field, bool new_line = true) const { return "\n" + Indent + Indent + "o = flatbuffers.number_types.UOffsetTFlags.py_type" + "(self._tab.Offset(" + NumToString(field.value.offset) + "))\n" + - Indent + Indent + "if o != 0:\n"; + Indent + Indent + "if o != 0:" + (new_line ? "\n" : ""); } // Begin a class declaration. - void BeginClass(const StructDef &struct_def, std::string *code_ptr) { + void BeginClass(const StructDef &struct_def, std::string *code_ptr) const { auto &code = *code_ptr; - code += "class " + NormalizedName(struct_def) + "(object):\n"; + code += "class " + namer_.Type(struct_def) + "(object):\n"; code += Indent + "__slots__ = ['_tab']"; code += "\n\n"; } // Begin enum code with a class declaration. - void BeginEnum(const std::string &class_name, std::string *code_ptr) { + void BeginEnum(const EnumDef &enum_def, std::string *code_ptr) const { auto &code = *code_ptr; - code += "class " + class_name + "(object):\n"; - } - - std::string EscapeKeyword(const std::string &name) const { - return keywords_.find(name) == keywords_.end() ? name : name + "_"; - } - - std::string NormalizedName(const Definition &definition) const { - return EscapeKeyword(definition.name); - } - - std::string NormalizedName(const EnumVal &ev) const { - return EscapeKeyword(ev.name); - } - - // Converts the name of a definition into upper Camel format. - std::string MakeUpperCamel(const Definition &definition) const { - return MakeCamel(NormalizedName(definition), true); - } - - // Converts the name of a definition into lower Camel format. - std::string MakeLowerCamel(const Definition &definition) const { - auto name = MakeCamel(NormalizedName(definition), false); - name[0] = CharToLower(name[0]); - return name; + code += "class " + namer_.Type(enum_def) + "(object):\n"; } // Starts a new line and then indents. - std::string GenIndents(int num) { + std::string GenIndents(int num) const { return "\n" + std::string(num * Indent.length(), ' '); } // A single enum member. void EnumMember(const EnumDef &enum_def, const EnumVal &ev, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; code += Indent; - code += NormalizedName(ev); + code += namer_.Variant(ev); code += " = "; code += enum_def.ToString(ev) + "\n"; } - // End enum code. - void EndEnum(std::string *code_ptr) { - auto &code = *code_ptr; - code += "\n"; - } - // Initialize a new struct or table from existing data. void NewRootTypeFromBuffer(const StructDef &struct_def, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; + const std::string struct_type = namer_.Type(struct_def); code += Indent + "@classmethod\n"; code += Indent + "def GetRootAs"; - code += "(cls, buf, offset=0):"; + if (parser_.opts.python_typing) { + code += "(cls, buf, offset: int = 0):"; + } else { + code += "(cls, buf, offset=0):"; + } code += "\n"; code += Indent + Indent; code += "n = flatbuffers.encode.Get"; code += "(flatbuffers.packer.uoffset, buf, offset)\n"; - code += Indent + Indent + "x = " + NormalizedName(struct_def) + "()\n"; + code += Indent + Indent + "x = " + struct_type + "()\n"; code += Indent + Indent + "x.Init(buf, n + offset)\n"; code += Indent + Indent + "return x\n"; code += "\n"; - // Add an alias with the old name - code += Indent + "@classmethod\n"; - code += Indent + "def GetRootAs"; - code += NormalizedName(struct_def); - code += "(cls, buf, offset=0):\n"; - code += Indent + Indent + "\"\"\"This method is deprecated. Please switch to GetRootAs.\"\"\"\n"; - code += Indent + Indent + "return cls.GetRootAs(buf, offset)\n"; + if (!parser_.opts.python_no_type_prefix_suffix) { + // Add an alias with the old name + code += Indent + "@classmethod\n"; + code += Indent + "def GetRootAs" + struct_type + "(cls, buf, offset=0):\n"; + code += + Indent + Indent + + "\"\"\"This method is deprecated. Please switch to GetRootAs.\"\"\"\n"; + code += Indent + Indent + "return cls.GetRootAs(buf, offset)\n"; + } } // Initialize an existing object with other data, to avoid an allocation. - void InitializeExisting(const StructDef &struct_def, std::string *code_ptr) { + void InitializeExisting(const StructDef &struct_def, + std::string *code_ptr) const { auto &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += "Init(self, buf, pos):\n"; + if (parser_.opts.python_typing) { + code += "Init(self, buf: bytes, pos: int):\n"; + } else { + code += "Init(self, buf, pos):\n"; + } code += Indent + Indent + "self._tab = flatbuffers.table.Table(buf, pos)\n"; code += "\n"; } // Get the length of a vector. void GetVectorLen(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)) + "Length(self"; - code += "):" + OffsetPrefix(field); - code += Indent + Indent + Indent + "return self._tab.VectorLen(o)\n"; - code += Indent + Indent + "return 0\n\n"; + code += namer_.Method(field) + "Length(self)"; + if (parser_.opts.python_typing) { + code += " -> int"; + } + code += ":"; + if(!IsArray(field.value.type)){ + code += OffsetPrefix(field,false); + code += GenIndents(3) + "return self._tab.VectorLen(o)"; + code += GenIndents(2) + "return 0\n\n"; + }else{ + code += GenIndents(2) + "return "+NumToString(field.value.type.fixed_length)+"\n\n"; + } } // Determines whether a vector is none or not. void GetVectorIsNone(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)) + "IsNone(self"; - code += "):"; - code += GenIndents(2) + - "o = flatbuffers.number_types.UOffsetTFlags.py_type" + - "(self._tab.Offset(" + NumToString(field.value.offset) + "))"; - code += GenIndents(2) + "return o == 0"; + code += namer_.Method(field) + "IsNone(self)"; + if (parser_.opts.python_typing) { + code += " -> bool"; + } + code += ":"; + if(!IsArray(field.value.type)){ + code += GenIndents(2) + + "o = flatbuffers.number_types.UOffsetTFlags.py_type" + + "(self._tab.Offset(" + NumToString(field.value.offset) + "))"; + code += GenIndents(2) + "return o == 0"; + } else { + //assume that we always have an array as memory is preassigned + code += GenIndents(2) + "return False"; + } code += "\n\n"; } // Get the value of a struct's scalar. void GetScalarFieldOfStruct(const StructDef &struct_def, - const FieldDef &field, std::string *code_ptr) { + const FieldDef &field, + std::string *code_ptr) const { auto &code = *code_ptr; std::string getter = GenGetter(field.value.type); GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); + code += namer_.Method(field); code += "(self): return " + getter; code += "self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type("; code += NumToString(field.value.offset) + "))\n"; @@ -196,11 +230,11 @@ class PythonGenerator : public BaseGenerator { // Get the value of a table's scalar. void GetScalarFieldOfTable(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; std::string getter = GenGetter(field.value.type); GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); + code += namer_.Method(field); code += "(self):"; code += OffsetPrefix(field); getter += "o + self._tab.Pos)"; @@ -208,7 +242,9 @@ class PythonGenerator : public BaseGenerator { if (is_bool) { getter = "bool(" + getter + ")"; } code += Indent + Indent + Indent + "return " + getter + "\n"; std::string default_value; - if (is_bool) { + if (field.IsScalarOptional()) { + default_value = "None"; + } else if (is_bool) { default_value = field.value.constant == "0" ? "False" : "True"; } else { default_value = IsFloat(field.value.type.base_type) @@ -221,10 +257,11 @@ class PythonGenerator : public BaseGenerator { // Get a struct by initializing an existing struct. // Specific to Struct. void GetStructFieldOfStruct(const StructDef &struct_def, - const FieldDef &field, std::string *code_ptr) { + const FieldDef &field, + std::string *code_ptr) const { auto &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); + code += namer_.Method(field); code += "(self, obj):\n"; code += Indent + Indent + "obj.Init(self._tab.Bytes, self._tab.Pos + "; code += NumToString(field.value.offset) + ")"; @@ -233,36 +270,83 @@ class PythonGenerator : public BaseGenerator { // Get the value of a fixed size array. void GetArrayOfStruct(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { + std::string *code_ptr, ImportMap &imports) const { auto &code = *code_ptr; const auto vec_type = field.value.type.VectorType(); GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); - if (IsStruct(vec_type)) { - code += "(self, obj, i):\n"; - code += Indent + Indent + "obj.Init(self._tab.Bytes, self._tab.Pos + "; - code += NumToString(field.value.offset) + " + i * "; - code += NumToString(InlineSize(vec_type)); - code += ")\n" + Indent + Indent + "return obj\n\n"; + code += namer_.Method(field); + + const ImportMapEntry import_entry = { + GenPackageReference(field.value.type), TypeName(field) + }; + + if (parser_.opts.python_typing) { + const std::string return_type = ReturnType(struct_def, field); + code += "(self, i: int)"; + code += " -> " + return_type + ":"; + + imports.insert(import_entry); } else { - auto getter = GenGetter(vec_type); - code += "(self): return [" + getter; - code += "self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type("; - code += NumToString(field.value.offset) + " + i * "; - code += NumToString(InlineSize(vec_type)); - code += ")) for i in range("; - code += NumToString(field.value.type.fixed_length) + ")]\n"; + code += "(self, i):"; } + + if (parser_.opts.include_dependence_headers && !parser_.opts.python_typing) { + code += GenIndents(2); + code += "from " + import_entry.first + " import " + import_entry.second + + "\n"; + } + + code += GenIndents(2) + "obj = " + TypeName(field) + "()"; + code += GenIndents(2) + "obj.Init(self._tab.Bytes, self._tab.Pos + "; + code += NumToString(field.value.offset) + " + i * "; + code += NumToString(InlineSize(vec_type)); + code += ")" + GenIndents(2) + "return obj\n\n"; + } + + // Get the value of a vector's non-struct member. Uses a named return + // argument to conveniently set the zero value for the result. + void GetArrayOfNonStruct(const StructDef &struct_def, const FieldDef &field, + std::string *code_ptr) const { + auto &code = *code_ptr; + GenReceiver(struct_def, code_ptr); + code += namer_.Method(field); + code += "(self, j = None):"; + code += GenIndents(2) + "if j is None:"; + code += GenIndents(3) + "return [" + GenGetter(field.value.type); + code += "self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type("; + code += NumToString(field.value.offset) + " + i * "; + code += NumToString(InlineSize(field.value.type.VectorType())); + code += ")) for i in range("; + code += "self."+namer_.Method(field)+"Length()" + ")]"; + code += GenIndents(2) +"elif j >= 0 and j < self."+namer_.Method(field)+"Length():"; + code += GenIndents(3) + "return " + GenGetter(field.value.type); + code += "self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type("; + code += NumToString(field.value.offset) + " + j * "; + code += NumToString(InlineSize(field.value.type.VectorType())); + code += "))"; + code += GenIndents(2) + "else:"; + code += GenIndents(3) + "return None\n\n"; } // Get a struct by initializing an existing struct. // Specific to Table. void GetStructFieldOfTable(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { + std::string *code_ptr, ImportMap &imports) const { auto &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); - code += "(self):"; + code += namer_.Method(field) + "(self)"; + + const ImportMapEntry import_entry = { + GenPackageReference(field.value.type), TypeName(field) + }; + + if (parser_.opts.python_typing) { + const std::string return_type = ReturnType(struct_def, field); + code += " -> Optional[" + return_type + "]"; + imports.insert(ImportMapEntry{ "typing", "Optional" }); + imports.insert(import_entry); + } + code += ":"; code += OffsetPrefix(field); if (field.value.type.struct_def->fixed) { code += Indent + Indent + Indent + "x = o + self._tab.Pos\n"; @@ -270,10 +354,11 @@ class PythonGenerator : public BaseGenerator { code += Indent + Indent + Indent; code += "x = self._tab.Indirect(o + self._tab.Pos)\n"; } - if (parser_.opts.include_dependence_headers) { + + if (parser_.opts.include_dependence_headers && !parser_.opts.python_typing) { code += Indent + Indent + Indent; - code += "from " + GenPackageReference(field.value.type) + " import " + - TypeName(field) + "\n"; + code += "from " + import_entry.first + " import " + import_entry.second + + "\n"; } code += Indent + Indent + Indent + "obj = " + TypeName(field) + "()\n"; code += Indent + Indent + Indent + "obj.Init(self._tab.Bytes, x)\n"; @@ -283,11 +368,18 @@ class PythonGenerator : public BaseGenerator { // Get the value of a string. void GetStringField(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { + std::string *code_ptr, ImportMap &imports) const { auto &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); - code += "(self):"; + code += namer_.Method(field); + + if (parser_.opts.python_typing) { + code += "(self) -> Optional[str]:"; + imports.insert(ImportMapEntry{ "typing", "Optional" }); + } else { + code += "(self):"; + } + code += OffsetPrefix(field); code += Indent + Indent + Indent + "return " + GenGetter(field.value.type); code += "o + self._tab.Pos)\n"; @@ -296,21 +388,34 @@ class PythonGenerator : public BaseGenerator { // Get the value of a union from an object. void GetUnionField(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { + std::string *code_ptr, ImportMap &imports) const { auto &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)) + "(self):"; - code += OffsetPrefix(field); + std::string return_ty = "flatbuffers.table.Table"; - // TODO(rw): this works and is not the good way to it: bool is_native_table = TypeName(field) == "*flatbuffers.Table"; + ImportMapEntry import_entry; if (is_native_table) { - code += - Indent + Indent + Indent + "from flatbuffers.table import Table\n"; - } else if (parser_.opts.include_dependence_headers) { + import_entry = ImportMapEntry{ "flatbuffers.table", "Table" }; + } else { + return_ty = TypeName(field); + import_entry = ImportMapEntry{ GenPackageReference(field.value.type), + TypeName(field) }; + } + + code += namer_.Method(field) + "(self)"; + if (parser_.opts.python_typing) { + code += " -> Optional[" + return_ty + "]"; + imports.insert(ImportMapEntry{ "typing", "Optional" }); + imports.insert(import_entry); + } + code += ":"; + code += OffsetPrefix(field); + + if (!parser_.opts.python_typing) { code += Indent + Indent + Indent; - code += "from " + GenPackageReference(field.value.type) + " import " + - TypeName(field) + "\n"; + code += "from " + import_entry.first + " import " + import_entry.second + + "\n"; } code += Indent + Indent + Indent + "obj = Table(bytearray(), 0)\n"; code += Indent + Indent + Indent + GenGetter(field.value.type); @@ -320,28 +425,38 @@ class PythonGenerator : public BaseGenerator { // Generate the package reference when importing a struct or enum from its // module. - std::string GenPackageReference(const Type &type) { - Namespace *namespaces; + std::string GenPackageReference(const Type &type) const { if (type.struct_def) { - namespaces = type.struct_def->defined_namespace; + return namer_.NamespacedType(*type.struct_def); } else if (type.enum_def) { - namespaces = type.enum_def->defined_namespace; + return namer_.NamespacedType(*type.enum_def); } else { return "." + GenTypeGet(type); } - - return namespaces->GetFullyQualifiedName(GenTypeGet(type)); } // Get the value of a vector's struct member. void GetMemberOfVectorOfStruct(const StructDef &struct_def, - const FieldDef &field, std::string *code_ptr) { + const FieldDef &field, std::string *code_ptr, + ImportMap &imports) const { auto &code = *code_ptr; auto vectortype = field.value.type.VectorType(); GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); - code += "(self, j):" + OffsetPrefix(field); + code += namer_.Method(field); + const ImportMapEntry import_entry = { + GenPackageReference(field.value.type), TypeName(field) + }; + + if (parser_.opts.python_typing) { + const std::string return_type = ReturnType(struct_def, field); + code += "(self, j: int) -> Optional[" + return_type + "]"; + imports.insert(ImportMapEntry{ "typing", "Optional" }); + imports.insert(import_entry); + } else { + code += "(self, j)"; + } + code += ":" + OffsetPrefix(field); code += Indent + Indent + Indent + "x = self._tab.Vector(o)\n"; code += Indent + Indent + Indent; code += "x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * "; @@ -349,10 +464,10 @@ class PythonGenerator : public BaseGenerator { if (!(vectortype.struct_def->fixed)) { code += Indent + Indent + Indent + "x = self._tab.Indirect(x)\n"; } - if (parser_.opts.include_dependence_headers) { + if (parser_.opts.include_dependence_headers && !parser_.opts.python_typing) { code += Indent + Indent + Indent; - code += "from " + GenPackageReference(field.value.type) + " import " + - TypeName(field) + "\n"; + code += "from " + import_entry.first + " import " + import_entry.second + + "\n"; } code += Indent + Indent + Indent + "obj = " + TypeName(field) + "()\n"; code += Indent + Indent + Indent + "obj.Init(self._tab.Bytes, x)\n"; @@ -364,13 +479,18 @@ class PythonGenerator : public BaseGenerator { // argument to conveniently set the zero value for the result. void GetMemberOfVectorOfNonStruct(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; auto vectortype = field.value.type.VectorType(); GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)); - code += "(self, j):"; + code += namer_.Method(field); + if (parser_.opts.python_typing) { + code += "(self, j: int)"; + } else { + code += "(self, j)"; + } + code += ":"; code += OffsetPrefix(field); code += Indent + Indent + Indent + "a = self._tab.Vector(o)\n"; code += Indent + Indent + Indent; @@ -389,7 +509,7 @@ class PythonGenerator : public BaseGenerator { // than iterating over the vector element by element. void GetVectorOfNonStructAsNumpy(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; auto vectortype = field.value.type.VectorType(); @@ -398,49 +518,76 @@ class PythonGenerator : public BaseGenerator { if (!(IsScalar(vectortype.base_type))) { return; } GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)) + "AsNumpy(self):"; - code += OffsetPrefix(field); - - code += Indent + Indent + Indent; - code += "return "; - code += "self._tab.GetVectorAsNumpy(flatbuffers.number_types."; - code += MakeCamel(GenTypeGet(field.value.type)); - code += "Flags, o)\n"; - - if (IsString(vectortype)) { - code += Indent + Indent + "return \"\"\n"; - } else { - code += Indent + Indent + "return 0\n"; + code += namer_.Method(field) + "AsNumpy(self):"; + if(!IsArray(field.value.type)){ + code += OffsetPrefix(field, false); + + code += GenIndents(3); + code += "return "; + code += "self._tab.GetVectorAsNumpy(flatbuffers.number_types."; + code += namer_.Method(GenTypeGet(field.value.type)); + code += "Flags, o)"; + + if (IsString(vectortype)) { + code += GenIndents(2) + "return \"\"\n"; + } else { + code += GenIndents(2) + "return 0\n"; + } + }else{ + code += GenIndents(2) + "return "; + code += "self._tab.GetArrayAsNumpy(flatbuffers.number_types."; + code += namer_.Method(GenTypeGet(field.value.type.VectorType())); + code += "Flags, self._tab.Pos + "+NumToString(field.value.offset)+", "+NumToString("self."+namer_.Method(field)+"Length()")+")\n"; } code += "\n"; } + std::string NestedFlatbufferType(std::string unqualified_name) const { + StructDef *nested_root = parser_.LookupStruct(unqualified_name); + std::string qualified_name; + if (nested_root == nullptr) { + qualified_name = namer_.NamespacedType( + parser_.current_namespace_->components, unqualified_name); + // Double check qualified name just to be sure it exists. + nested_root = parser_.LookupStruct(qualified_name); + } + FLATBUFFERS_ASSERT(nested_root); // Guaranteed to exist by parser. + return qualified_name; + } + // Returns a nested flatbuffer as itself. void GetVectorAsNestedFlatbuffer(const StructDef &struct_def, - const FieldDef &field, - std::string *code_ptr) { + const FieldDef &field, std::string *code_ptr, + ImportMap &imports) const { auto nested = field.attributes.Lookup("nested_flatbuffer"); if (!nested) { return; } // There is no nested flatbuffer. - std::string unqualified_name = nested->constant; - std::string qualified_name = nested->constant; - auto nested_root = parser_.LookupStruct(nested->constant); - if (nested_root == nullptr) { - qualified_name = - parser_.current_namespace_->GetFullyQualifiedName(nested->constant); - nested_root = parser_.LookupStruct(qualified_name); + const std::string unqualified_name = nested->constant; + std::string qualified_name = NestedFlatbufferType(unqualified_name); + if (qualified_name.empty()) { + qualified_name = nested->constant; } - FLATBUFFERS_ASSERT(nested_root); // Guaranteed to exist by parser. - (void)nested_root; + + const ImportMapEntry import_entry = { qualified_name, + unqualified_name }; auto &code = *code_ptr; GenReceiver(struct_def, code_ptr); - code += MakeCamel(NormalizedName(field)) + "NestedRoot(self):"; + code += namer_.Method(field) + "NestedRoot(self)"; + if (parser_.opts.python_typing) { + code += " -> Union[" + unqualified_name + ", int]"; + imports.insert(ImportMapEntry{ "typing", "Union" }); + imports.insert(import_entry); + } + code += ":"; code += OffsetPrefix(field); - code += Indent + Indent + Indent; - code += "from " + qualified_name + " import " + unqualified_name + "\n"; + if (!parser_.opts.python_typing) { + code += Indent + Indent + Indent; + code += "from " + import_entry.first + " import " + import_entry.second + + "\n"; + } code += Indent + Indent + Indent + "return " + unqualified_name; code += ".GetRootAs"; code += "(self._tab.Bytes, self._tab.Vector(o))\n"; @@ -449,11 +596,12 @@ class PythonGenerator : public BaseGenerator { } // Begin the creator function signature. - void BeginBuilderArgs(const StructDef &struct_def, std::string *code_ptr) { + void BeginBuilderArgs(const StructDef &struct_def, + std::string *code_ptr) const { auto &code = *code_ptr; code += "\n"; - code += "def Create" + NormalizedName(struct_def); + code += "def Create" + namer_.Type(struct_def); code += "(builder"; } @@ -463,7 +611,7 @@ class PythonGenerator : public BaseGenerator { const std::string nameprefix, const std::string namesuffix, bool has_field_name, const std::string fieldname_suffix, - std::string *code_ptr) { + std::string *code_ptr) const { for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { auto &field = **it; @@ -476,21 +624,21 @@ class PythonGenerator : public BaseGenerator { // a nested struct, prefix the name with the field name. auto subprefix = nameprefix; if (has_field_name) { - subprefix += NormalizedName(field) + fieldname_suffix; + subprefix += namer_.Field(field) + fieldname_suffix; } StructBuilderArgs(*field.value.type.struct_def, subprefix, namesuffix, has_field_name, fieldname_suffix, code_ptr); } else { auto &code = *code_ptr; code += std::string(", ") + nameprefix; - if (has_field_name) { code += MakeCamel(NormalizedName(field), false); } + if (has_field_name) { code += namer_.Field(field); } code += namesuffix; } } } // End the creator function signature. - void EndBuilderArgs(std::string *code_ptr) { + void EndBuilderArgs(std::string *code_ptr) const { auto &code = *code_ptr; code += "):\n"; } @@ -499,7 +647,7 @@ class PythonGenerator : public BaseGenerator { // padding. void StructBuilderBody(const StructDef &struct_def, const char *nameprefix, std::string *code_ptr, size_t index = 0, - bool in_array = false) { + bool in_array = false) const { auto &code = *code_ptr; std::string indent(index * 4, ' '); code += @@ -516,7 +664,7 @@ class PythonGenerator : public BaseGenerator { indent + " builder.Pad(" + NumToString(field.padding) + ")\n"; if (IsStruct(field_type)) { StructBuilderBody(*field_type.struct_def, - (nameprefix + (NormalizedName(field) + "_")).c_str(), + (nameprefix + (namer_.Field(field) + "_")).c_str(), code_ptr, index, in_array); } else { const auto index_var = "_idx" + NumToString(index); @@ -527,14 +675,13 @@ class PythonGenerator : public BaseGenerator { in_array = true; } if (IsStruct(type)) { - StructBuilderBody( - *field_type.struct_def, - (nameprefix + (NormalizedName(field) + "_")).c_str(), code_ptr, - index + 1, in_array); + StructBuilderBody(*field_type.struct_def, + (nameprefix + (namer_.Field(field) + "_")).c_str(), + code_ptr, index + 1, in_array); } else { code += IsArray(field_type) ? " " : ""; code += indent + " builder.Prepend" + GenMethod(field) + "("; - code += nameprefix + MakeCamel(NormalizedName(field), false); + code += nameprefix + namer_.Variable(field); size_t array_cnt = index + (IsArray(field_type) ? 1 : 0); for (size_t i = 0; in_array && i < array_cnt; i++) { code += "[_idx" + NumToString(i) + "-1]"; @@ -545,108 +692,138 @@ class PythonGenerator : public BaseGenerator { } } - void EndBuilderBody(std::string *code_ptr) { + void EndBuilderBody(std::string *code_ptr) const { auto &code = *code_ptr; code += " return builder.Offset()\n"; } // Get the value of a table's starting offset. - void GetStartOfTable(const StructDef &struct_def, std::string *code_ptr) { + void GetStartOfTable(const StructDef &struct_def, + std::string *code_ptr) const { auto &code = *code_ptr; - code += "def Start(builder): "; - code += "builder.StartObject("; + const auto struct_type = namer_.Type(struct_def); + // Generate method with struct name. + + const auto name = parser_.opts.python_no_type_prefix_suffix ? "Start" : struct_type + "Start"; + + code += "def " + name; + if (parser_.opts.python_typing) { + code += "(builder: flatbuffers.Builder):\n"; + } else { + code += "(builder):\n"; + } + + code += Indent + "builder.StartObject("; code += NumToString(struct_def.fields.vec.size()); - code += ")\n"; + code += ")\n\n"; - // Add alias with the old name. - code += "def " + NormalizedName(struct_def) + "Start(builder):\n"; - code += Indent + "\"\"\"This method is deprecated. Please switch to Start.\"\"\"\n"; - code += Indent + "return Start(builder)\n"; + if (!parser_.opts.one_file && !parser_.opts.python_no_type_prefix_suffix) { + // Generate method without struct name. + if (parser_.opts.python_typing) { + code += "def Start(builder: flatbuffers.Builder):\n"; + } else { + code += "def Start(builder):\n"; + } + code += Indent + struct_type + "Start(builder)\n\n"; + } } // Set the value of a table's field. void BuildFieldOfTable(const StructDef &struct_def, const FieldDef &field, - const size_t offset, std::string *code_ptr) { + const size_t offset, std::string *code_ptr) const { auto &code = *code_ptr; - code += "def Add" + MakeCamel(NormalizedName(field)); - code += "(builder, "; - code += MakeCamel(NormalizedName(field), false); - code += "): "; - code += "builder.Prepend"; + const std::string field_var = namer_.Variable(field); + const std::string field_method = namer_.Method(field); + const std::string field_ty = GenFieldTy(field); + + const auto name = parser_.opts.python_no_type_prefix_suffix ? "Add" + field_method : namer_.Type(struct_def) + "Add" + field_method; + + // Generate method with struct name. + code += "def " + name; + if (parser_.opts.python_typing) { + code += "(builder: flatbuffers.Builder, " + field_var + ": " + field_ty; + } else { + code += "(builder, " + field_var; + } + code += "):\n"; + code += Indent + "builder.Prepend"; code += GenMethod(field) + "Slot("; code += NumToString(offset) + ", "; if (!IsScalar(field.value.type.base_type) && (!struct_def.fixed)) { code += "flatbuffers.number_types.UOffsetTFlags.py_type"; - code += "("; - code += MakeCamel(NormalizedName(field), false) + ")"; + code += "(" + field_var + ")"; } else { - code += MakeCamel(NormalizedName(field), false); + code += field_var; } code += ", "; - code += IsFloat(field.value.type.base_type) - ? float_const_gen_.GenFloatConstant(field) - : field.value.constant; - code += ")\n"; - - // Add alias with the old name. - code += "def " + NormalizedName(struct_def) + "Add" + MakeCamel(NormalizedName(field)); - code += "(builder, "; - code += MakeCamel(NormalizedName(field), false); - code += "):\n"; - code += Indent + "\"\"\"This method is deprecated. Please switch to Add"; - code += MakeCamel(NormalizedName(field)) + ".\"\"\"\n"; - code += Indent + "return Add" + MakeCamel(NormalizedName(field)); - code += "(builder, "; - code += MakeCamel(NormalizedName(field), false); - code += ")\n"; + if (field.IsScalarOptional()) { + code += "None"; + } else if (IsFloat(field.value.type.base_type)) { + code += float_const_gen_.GenFloatConstant(field); + } else { + code += field.value.constant; + } + code += ")\n\n"; - // Add alias with the old name. + if (!parser_.opts.one_file && !parser_.opts.python_no_type_prefix_suffix) { + // Generate method without struct name. + code += "def Add" + field_method + "(builder: flatbuffers.Builder, " + field_var + ": " + field_ty + "):\n"; + code += + Indent + namer_.Type(struct_def) + "Add" + field_method; + code += "(builder, "; + code += field_var; + code += ")\n\n"; + } } // Set the value of one of the members of a table's vector. void BuildVectorOfTable(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; - code += "def Start"; - code += MakeCamel(NormalizedName(field)); - code += "Vector(builder, numElems): return builder.StartVector("; + const std::string struct_type = namer_.Type(struct_def); + const std::string field_method = namer_.Method(field); + + // Generate method with struct name. + const auto name = parser_.opts.python_no_type_prefix_suffix ? "Start" + field_method : struct_type + "Start" + field_method; + code += "def " + name; + if (parser_.opts.python_typing) { + code += "Vector(builder, numElems: int) -> int:\n"; + } else { + code += "Vector(builder, numElems):\n"; + } + + code += Indent + "return builder.StartVector("; auto vector_type = field.value.type.VectorType(); auto alignment = InlineAlignment(vector_type); auto elem_size = InlineSize(vector_type); code += NumToString(elem_size); code += ", numElems, " + NumToString(alignment); - code += ")\n"; + code += ")\n\n"; - // Add alias with the old name. - code += "def " + NormalizedName(struct_def) + "Start"; - code += MakeCamel(NormalizedName(field)); - code += "Vector(builder, numElems):\n"; - code += Indent + "\"\"\"This method is deprecated. Please switch to Start.\"\"\"\n"; - code += Indent + "return Start"; - code += MakeCamel(NormalizedName(field)); - code += "Vector(builder, numElems)\n"; + if (!parser_.opts.one_file && !parser_.opts.python_no_type_prefix_suffix) { + // Generate method without struct name. + code += "def Start" + field_method + "Vector(builder, numElems: int) -> int:\n"; + code += Indent + "return " + struct_type + "Start"; + code += field_method + "Vector(builder, numElems)\n\n"; + } } // Set the value of one of the members of a table's vector and fills in the // elements from a bytearray. This is for simplifying the use of nested // flatbuffers. - void BuildVectorOfTableFromBytes(const FieldDef &field, std::string *code_ptr) { + void BuildVectorOfTableFromBytes(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) const { auto nested = field.attributes.Lookup("nested_flatbuffer"); if (!nested) { return; } // There is no nested flatbuffer. - std::string unqualified_name = nested->constant; - std::string qualified_name = nested->constant; - auto nested_root = parser_.LookupStruct(nested->constant); - if (nested_root == nullptr) { - qualified_name = - parser_.current_namespace_->GetFullyQualifiedName(nested->constant); - nested_root = parser_.LookupStruct(qualified_name); - } - FLATBUFFERS_ASSERT(nested_root); // Guaranteed to exist by parser. - (void)nested_root; - auto &code = *code_ptr; - code += "def MakeVectorFromBytes(builder, bytes):\n"; + const std::string field_method = namer_.Method(field); + const std::string struct_type = namer_.Type(struct_def); + + // Generate method with struct and field name. + code += "def " + struct_type + "Make" + field_method; + code += "VectorFromBytes(builder, bytes):\n"; code += Indent + "builder.StartVector("; auto vector_type = field.value.type.VectorType(); auto alignment = InlineAlignment(vector_type); @@ -659,40 +836,50 @@ class PythonGenerator : public BaseGenerator { code += " = bytes\n"; code += Indent + "return builder.EndVector()\n"; - // Add alias with the old name. - code += "def Make" + MakeCamel(NormalizedName(field)); - code += "VectorFromBytes(builder, bytes):\n"; - code += Indent + "builder.StartVector("; - code += NumToString(elem_size); - code += ", len(bytes), " + NumToString(alignment); - code += ")\n"; - code += Indent + "builder.head = builder.head - len(bytes)\n"; - code += Indent + "builder.Bytes[builder.head : builder.head + len(bytes)]"; - code += " = bytes\n"; - code += Indent + "return builder.EndVector()\n"; + if (!parser_.opts.one_file) { + // Generate method without struct and field name. + code += "def Make" + field_method + "VectorFromBytes(builder, bytes):\n"; + code += Indent + "return " + struct_type + "Make" + field_method + + "VectorFromBytes(builder, bytes)\n"; + } } // Get the offset of the end of a table. - void GetEndOffsetOnTable(const StructDef &struct_def, std::string *code_ptr) { + void GetEndOffsetOnTable(const StructDef &struct_def, + std::string *code_ptr) const { auto &code = *code_ptr; - code += "def End(builder): return builder.EndObject()\n"; - // Add alias with the old name. - code += "def " + NormalizedName(struct_def) + "End(builder):\n"; - code += Indent + "\"\"\"This method is deprecated. Please switch to End.\"\"\"\n"; - code += Indent + "return End(builder)"; + const auto name = parser_.opts.python_no_type_prefix_suffix ? "End" : namer_.Type(struct_def) + "End"; + // Generate method with struct name. + if (parser_.opts.python_typing) { + code += "def " + name + "(builder: flatbuffers.Builder) -> int:\n"; + } else { + code += "def " + name + "(builder):\n"; + } + code += Indent + "return builder.EndObject()\n\n"; + + if (!parser_.opts.one_file && !parser_.opts.python_no_type_prefix_suffix) { + // Generate method without struct name. + if (parser_.opts.python_typing) { + code += "def End(builder: flatbuffers.Builder) -> int:\n"; + } else { + code += "def End(builder):\n"; + } + code += Indent + "return " + namer_.Type(struct_def) + "End(builder)"; + code += "\n"; + } } // Generate the receiver for function signatures. - void GenReceiver(const StructDef &struct_def, std::string *code_ptr) { + void GenReceiver(const StructDef &struct_def, std::string *code_ptr) const { auto &code = *code_ptr; - code += Indent + "# " + NormalizedName(struct_def) + "\n"; + code += Indent + "# " + namer_.Type(struct_def) + "\n"; code += Indent + "def "; } // Generate a struct field, conditioned on its child type(s). void GenStructAccessor(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { + std::string *code_ptr, ImportMap &imports) const { GenComment(field.doc_comment, code_ptr, &def_comment, Indent.c_str()); if (IsScalar(field.value.type.base_type)) { if (struct_def.fixed) { @@ -700,32 +887,41 @@ class PythonGenerator : public BaseGenerator { } else { GetScalarFieldOfTable(struct_def, field, code_ptr); } - } else if (IsArray(field.value.type)) { - GetArrayOfStruct(struct_def, field, code_ptr); } else { switch (field.value.type.base_type) { case BASE_TYPE_STRUCT: if (struct_def.fixed) { GetStructFieldOfStruct(struct_def, field, code_ptr); } else { - GetStructFieldOfTable(struct_def, field, code_ptr); + GetStructFieldOfTable(struct_def, field, code_ptr, imports); } break; case BASE_TYPE_STRING: - GetStringField(struct_def, field, code_ptr); + GetStringField(struct_def, field, code_ptr, imports); break; case BASE_TYPE_VECTOR: { auto vectortype = field.value.type.VectorType(); if (vectortype.base_type == BASE_TYPE_STRUCT) { - GetMemberOfVectorOfStruct(struct_def, field, code_ptr); + GetMemberOfVectorOfStruct(struct_def, field, code_ptr, imports); } else { GetMemberOfVectorOfNonStruct(struct_def, field, code_ptr); GetVectorOfNonStructAsNumpy(struct_def, field, code_ptr); - GetVectorAsNestedFlatbuffer(struct_def, field, code_ptr); + GetVectorAsNestedFlatbuffer(struct_def, field, code_ptr, imports); + } + break; + } + case BASE_TYPE_ARRAY: { + auto vectortype = field.value.type.VectorType(); + if (vectortype.base_type == BASE_TYPE_STRUCT) { + GetArrayOfStruct(struct_def, field, code_ptr, imports); + } else { + GetArrayOfNonStruct(struct_def, field, code_ptr); + GetVectorOfNonStructAsNumpy(struct_def, field, code_ptr); + GetVectorAsNestedFlatbuffer(struct_def, field, code_ptr, imports); } break; } - case BASE_TYPE_UNION: GetUnionField(struct_def, field, code_ptr); break; + case BASE_TYPE_UNION: GetUnionField(struct_def, field, code_ptr, imports); break; default: FLATBUFFERS_ASSERT(0); } } @@ -736,17 +932,23 @@ class PythonGenerator : public BaseGenerator { } // Generate struct sizeof. - void GenStructSizeOf(const StructDef &struct_def, std::string *code_ptr) { + void GenStructSizeOf(const StructDef &struct_def, + std::string *code_ptr) const { auto &code = *code_ptr; code += Indent + "@classmethod\n"; - code += Indent + "def SizeOf(cls):\n"; + if (parser_.opts.python_typing) { + code += Indent + "def SizeOf(cls) -> int:\n"; + } else { + code += Indent + "def SizeOf(cls):\n"; + } code += Indent + Indent + "return " + NumToString(struct_def.bytesize) + "\n"; code += "\n"; } // Generate table constructors, conditioned on its members' types. - void GenTableBuilders(const StructDef &struct_def, std::string *code_ptr) { + void GenTableBuilders(const StructDef &struct_def, + std::string *code_ptr) const { GetStartOfTable(struct_def, code_ptr); for (auto it = struct_def.fields.vec.begin(); @@ -758,7 +960,7 @@ class PythonGenerator : public BaseGenerator { BuildFieldOfTable(struct_def, field, offset, code_ptr); if (IsVector(field.value.type)) { BuildVectorOfTable(struct_def, field, code_ptr); - BuildVectorOfTableFromBytes(field, code_ptr); + BuildVectorOfTableFromBytes(struct_def, field, code_ptr); } } @@ -767,7 +969,7 @@ class PythonGenerator : public BaseGenerator { // Generate function to check for proper file identifier void GenHasFileIdentifier(const StructDef &struct_def, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; std::string escapedID; // In the event any of file_identifier characters are special(NULL, \, etc), @@ -779,7 +981,7 @@ class PythonGenerator : public BaseGenerator { } code += Indent + "@classmethod\n"; - code += Indent + "def " + NormalizedName(struct_def); + code += Indent + "def " + namer_.Type(struct_def); code += "BufferHasIdentifier(cls, buf, offset, size_prefixed=False):"; code += "\n"; code += Indent + Indent; @@ -790,7 +992,8 @@ class PythonGenerator : public BaseGenerator { } // Generates struct or table methods. - void GenStruct(const StructDef &struct_def, std::string *code_ptr) { + void GenStruct(const StructDef &struct_def, std::string *code_ptr, + ImportMap &imports) const { if (struct_def.generated) return; GenComment(struct_def.doc_comment, code_ptr, &def_comment); @@ -815,7 +1018,7 @@ class PythonGenerator : public BaseGenerator { auto &field = **it; if (field.deprecated) continue; - GenStructAccessor(struct_def, field, code_ptr); + GenStructAccessor(struct_def, field, code_ptr, imports); } if (struct_def.fixed) { @@ -828,23 +1031,24 @@ class PythonGenerator : public BaseGenerator { } void GenReceiverForObjectAPI(const StructDef &struct_def, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; - code += GenIndents(1) + "# " + NormalizedName(struct_def) + "T"; + code += GenIndents(1) + "# " + namer_.ObjectType(struct_def); code += GenIndents(1) + "def "; } void BeginClassForObjectAPI(const StructDef &struct_def, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; code += "\n"; - code += "class " + NormalizedName(struct_def) + "T(object):"; + code += "class " + namer_.ObjectType(struct_def) + "(object):"; code += "\n"; } // Gets the accoresponding python builtin type of a BaseType for scalars and // string. - std::string GetBasePythonTypeForScalarAndString(const BaseType &base_type) { + std::string GetBasePythonTypeForScalarAndString( + const BaseType &base_type) const { if (IsBool(base_type)) { return "bool"; } else if (IsFloat(base_type)) { @@ -859,9 +1063,11 @@ class PythonGenerator : public BaseGenerator { } } - std::string GetDefaultValue(const FieldDef &field) { + std::string GetDefaultValue(const FieldDef &field) const { BaseType base_type = field.value.type.base_type; - if (IsBool(base_type)) { + if (field.IsScalarOptional()) { + return "None"; + } else if (IsBool(base_type)) { return field.value.constant == "0" ? "False" : "True"; } else if (IsFloat(base_type)) { return float_const_gen_.GenFloatConstant(field); @@ -875,7 +1081,7 @@ class PythonGenerator : public BaseGenerator { void GenUnionInit(const FieldDef &field, std::string *field_types_ptr, std::set<std::string> *import_list, - std::set<std::string> *import_typing_list) { + std::set<std::string> *import_typing_list) const { // Gets all possible types in the union. import_typing_list->insert("Union"); auto &field_types = *field_types_ptr; @@ -890,7 +1096,7 @@ class PythonGenerator : public BaseGenerator { std::string field_type; switch (ev.union_type.base_type) { case BASE_TYPE_STRUCT: - field_type = GenTypeGet(ev.union_type) + "T"; + field_type = namer_.ObjectType(*ev.union_type.struct_def); if (parser_.opts.include_dependence_headers) { auto package_reference = GenPackageReference(ev.union_type); field_type = package_reference + "." + field_type; @@ -910,45 +1116,42 @@ class PythonGenerator : public BaseGenerator { // Gets the import lists for the union. if (parser_.opts.include_dependence_headers) { - // The package reference is generated based on enum_def, instead - // of struct_def in field.type. That's why GenPackageReference() is - // not used. - Namespace *namespaces = field.value.type.enum_def->defined_namespace; - auto package_reference = namespaces->GetFullyQualifiedName( - MakeUpperCamel(*(field.value.type.enum_def))); - auto union_name = MakeUpperCamel(*(field.value.type.enum_def)); + const auto package_reference = GenPackageReference(field.value.type); import_list->insert("import " + package_reference); } } - void GenStructInit(const FieldDef &field, std::string *field_type_ptr, + void GenStructInit(const FieldDef &field, std::string *out_ptr, std::set<std::string> *import_list, - std::set<std::string> *import_typing_list) { + std::set<std::string> *import_typing_list) const { import_typing_list->insert("Optional"); - auto &field_type = *field_type_ptr; + auto &output = *out_ptr; + const Type &type = field.value.type; + const std::string object_type = namer_.ObjectType(*type.struct_def); if (parser_.opts.include_dependence_headers) { - auto package_reference = GenPackageReference(field.value.type); - field_type = package_reference + "." + TypeName(field) + "T]"; + auto package_reference = GenPackageReference(type); + output = package_reference + "." + object_type + "]"; import_list->insert("import " + package_reference); } else { - field_type = TypeName(field) + "T]"; + output = object_type + "]"; } - field_type = "Optional[" + field_type; + output = "Optional[" + output; } void GenVectorInit(const FieldDef &field, std::string *field_type_ptr, std::set<std::string> *import_list, - std::set<std::string> *import_typing_list) { + std::set<std::string> *import_typing_list) const { import_typing_list->insert("List"); auto &field_type = *field_type_ptr; - auto base_type = field.value.type.VectorType().base_type; + const Type &vector_type = field.value.type.VectorType(); + const BaseType base_type = vector_type.base_type; if (base_type == BASE_TYPE_STRUCT) { - field_type = GenTypeGet(field.value.type.VectorType()) + "T]"; + const std::string object_type = + namer_.ObjectType(*vector_type.struct_def); + field_type = object_type + "]"; if (parser_.opts.include_dependence_headers) { - auto package_reference = - GenPackageReference(field.value.type.VectorType()); - field_type = package_reference + "." + - GenTypeGet(field.value.type.VectorType()) + "T]"; + auto package_reference = GenPackageReference(vector_type); + field_type = package_reference + "." + object_type + "]"; import_list->insert("import " + package_reference); } field_type = "List[" + field_type; @@ -959,7 +1162,7 @@ class PythonGenerator : public BaseGenerator { } void GenInitialize(const StructDef &struct_def, std::string *code_ptr, - std::set<std::string> *import_list) { + std::set<std::string> *import_list) const { std::string code; std::set<std::string> import_typing_list; for (auto it = struct_def.fields.vec.begin(); @@ -987,14 +1190,17 @@ class PythonGenerator : public BaseGenerator { default: // Scalar or sting fields. field_type = GetBasePythonTypeForScalarAndString(base_type); + if (field.IsScalarOptional()) { + field_type = "Optional[" + field_type + "]"; + } break; } - auto default_value = GetDefaultValue(field); + const auto default_value = GetDefaultValue(field); // Wrties the init statement. - auto field_instance_name = MakeLowerCamel(field); - code += GenIndents(2) + "self." + field_instance_name + " = " + - default_value + " # type: " + field_type; + const auto field_field = namer_.Field(field); + code += GenIndents(2) + "self." + field_field + " = " + default_value + + " # type: " + field_type; } // Writes __init__ method. @@ -1030,46 +1236,74 @@ class PythonGenerator : public BaseGenerator { } // Removes the import of the struct itself, if applied. - auto package_reference = - struct_def.defined_namespace->GetFullyQualifiedName( - MakeUpperCamel(struct_def)); - auto struct_import = "import " + package_reference; + auto struct_import = "import " + namer_.NamespacedType(struct_def); import_list->erase(struct_import); } - void InitializeFromBuf(const StructDef &struct_def, std::string *code_ptr) { + void InitializeFromBuf(const StructDef &struct_def, + std::string *code_ptr) const { auto &code = *code_ptr; - auto instance_name = MakeLowerCamel(struct_def); - auto struct_name = NormalizedName(struct_def); + const auto struct_var = namer_.Variable(struct_def); + const auto struct_type = namer_.Type(struct_def); code += GenIndents(1) + "@classmethod"; code += GenIndents(1) + "def InitFromBuf(cls, buf, pos):"; - code += GenIndents(2) + instance_name + " = " + struct_name + "()"; - code += GenIndents(2) + instance_name + ".Init(buf, pos)"; - code += GenIndents(2) + "return cls.InitFromObj(" + instance_name + ")"; + code += GenIndents(2) + struct_var + " = " + struct_type + "()"; + code += GenIndents(2) + struct_var + ".Init(buf, pos)"; + code += GenIndents(2) + "return cls.InitFromObj(" + struct_var + ")"; + code += "\n"; + } + + void InitializeFromPackedBuf(const StructDef &struct_def, + std::string *code_ptr) const { + auto &code = *code_ptr; + const auto struct_var = namer_.Variable(struct_def); + const auto struct_type = namer_.Type(struct_def); + + code += GenIndents(1) + "@classmethod"; + code += GenIndents(1) + "def InitFromPackedBuf(cls, buf, pos=0):"; + code += GenIndents(2) + "n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, pos)"; + code += GenIndents(2) + "return cls.InitFromBuf(buf, pos+n)"; code += "\n"; } void InitializeFromObjForObject(const StructDef &struct_def, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; - auto instance_name = MakeLowerCamel(struct_def); - auto struct_name = NormalizedName(struct_def); + const auto struct_var = namer_.Variable(struct_def); + const auto struct_object = namer_.ObjectType(struct_def); code += GenIndents(1) + "@classmethod"; - code += GenIndents(1) + "def InitFromObj(cls, " + instance_name + "):"; - code += GenIndents(2) + "x = " + struct_name + "T()"; - code += GenIndents(2) + "x._UnPack(" + instance_name + ")"; + code += GenIndents(1) + "def InitFromObj(cls, " + struct_var + "):"; + code += GenIndents(2) + "x = " + struct_object + "()"; + code += GenIndents(2) + "x._UnPack(" + struct_var + ")"; code += GenIndents(2) + "return x"; code += "\n"; } + void GenCompareOperator(const StructDef &struct_def, + std::string *code_ptr) const { + auto &code = *code_ptr; + code += GenIndents(1) + "def __eq__(self, other):"; + code += GenIndents(2) + "return type(self) == type(other)"; + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + auto &field = **it; + if (field.deprecated) continue; + + // Wrties the comparison statement for this field. + const auto field_field = namer_.Field(field); + code += " and \\" + GenIndents(3) + "self." + field_field + " == " + "other." + field_field; + } + code += "\n"; + } + void GenUnPackForStruct(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; - auto struct_instance_name = MakeLowerCamel(struct_def); - auto field_instance_name = MakeLowerCamel(field); - auto field_accessor_name = MakeUpperCamel(field); + const auto struct_var = namer_.Variable(struct_def); + const auto field_field = namer_.Field(field); + const auto field_method = namer_.Method(field); auto field_type = TypeName(field); if (parser_.opts.include_dependence_headers) { @@ -1077,16 +1311,14 @@ class PythonGenerator : public BaseGenerator { field_type = package_reference + "." + TypeName(field); } - code += GenIndents(2) + "if " + struct_instance_name + "." + - field_accessor_name + "("; + code += GenIndents(2) + "if " + struct_var + "." + field_method + "("; // if field is a struct, we need to create an instance for it first. if (struct_def.fixed && field.value.type.base_type == BASE_TYPE_STRUCT) { code += field_type + "()"; } code += ") is not None:"; - code += GenIndents(3) + "self." + field_instance_name + " = " + field_type + - "T.InitFromObj(" + struct_instance_name + "." + - field_accessor_name + "("; + code += GenIndents(3) + "self." + field_field + " = " + namer_.ObjectType(field_type) + + + ".InitFromObj(" + struct_var + "." + field_method + "("; // A struct's accessor requires a struct buf instance. if (struct_def.fixed && field.value.type.base_type == BASE_TYPE_STRUCT) { code += field_type + "()"; @@ -1095,82 +1327,110 @@ class PythonGenerator : public BaseGenerator { } void GenUnPackForUnion(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; - auto field_instance_name = MakeLowerCamel(field); - auto field_accessor_name = MakeUpperCamel(field); - auto struct_instance_name = MakeLowerCamel(struct_def); - auto union_name = MakeUpperCamel(*(field.value.type.enum_def)); + const auto field_field = namer_.Field(field); + const auto field_method = namer_.Method(field); + const auto struct_var = namer_.Variable(struct_def); + const EnumDef &enum_def = *field.value.type.enum_def; + auto union_type = namer_.Type(enum_def); if (parser_.opts.include_dependence_headers) { - Namespace *namespaces = field.value.type.enum_def->defined_namespace; - auto package_reference = namespaces->GetFullyQualifiedName( - MakeUpperCamel(*(field.value.type.enum_def))); - union_name = package_reference + "." + union_name; + union_type = namer_.NamespacedType(enum_def) + "." + union_type; } - code += GenIndents(2) + "self." + field_instance_name + " = " + union_name + - "Creator(" + "self." + field_instance_name + "Type, " + - struct_instance_name + "." + field_accessor_name + "())"; + code += GenIndents(2) + "self." + field_field + " = " + union_type + + "Creator(" + "self." + field_field + "Type, " + struct_var + "." + + field_method + "())"; } void GenUnPackForStructVector(const StructDef &struct_def, - const FieldDef &field, std::string *code_ptr) { + const FieldDef &field, + std::string *code_ptr) const { auto &code = *code_ptr; - auto field_instance_name = MakeLowerCamel(field); - auto field_accessor_name = MakeUpperCamel(field); - auto struct_instance_name = MakeLowerCamel(struct_def); - - code += GenIndents(2) + "if not " + struct_instance_name + "." + - field_accessor_name + "IsNone():"; - code += GenIndents(3) + "self." + field_instance_name + " = []"; - code += GenIndents(3) + "for i in range(" + struct_instance_name + "." + - field_accessor_name + "Length()):"; - - auto field_type_name = TypeName(field); - auto one_instance = field_type_name + "_"; - one_instance[0] = CharToLower(one_instance[0]); + const auto field_field = namer_.Field(field); + const auto field_method = namer_.Method(field); + const auto struct_var = namer_.Variable(struct_def); + code += GenIndents(2) + "if not " + struct_var + "." + field_method + + "IsNone():"; + code += GenIndents(3) + "self." + field_field + " = []"; + code += GenIndents(3) + "for i in range(" + struct_var + "." + + field_method + "Length()):"; + + auto field_type = TypeName(field); + auto one_instance = field_type + "_"; + one_instance[0] = CharToLower(one_instance[0]); if (parser_.opts.include_dependence_headers) { auto package_reference = GenPackageReference(field.value.type); - field_type_name = package_reference + "." + TypeName(field); + field_type = package_reference + "." + TypeName(field); } + code += GenIndents(4) + "if " + struct_var + "." + field_method + + "(i) is None:"; + code += GenIndents(5) + "self." + field_field + ".append(None)"; + code += GenIndents(4) + "else:"; + code += GenIndents(5) + one_instance + " = " + namer_.ObjectType(field_type) + + ".InitFromObj(" + struct_var + "." + field_method + "(i))"; + code += + GenIndents(5) + "self." + field_field + ".append(" + one_instance + ")"; + } - code += GenIndents(4) + "if " + struct_instance_name + "." + - field_accessor_name + "(i) is None:"; - code += GenIndents(5) + "self." + field_instance_name + ".append(None)"; + void GenUnpackForTableVector(const StructDef &struct_def, + const FieldDef &field, + std::string *code_ptr) const { + auto &code = *code_ptr; + const auto field_field = namer_.Field(field); + const auto field_method = namer_.Method(field); + const auto struct_var = namer_.Variable(struct_def); + + code += GenIndents(2) + "if not " + struct_var + "." + field_method + + "IsNone():"; + code += GenIndents(3) + "self." + field_field + " = []"; + code += GenIndents(3) + "for i in range(" + struct_var + "." + + field_method + "Length()):"; + + auto field_type = TypeName(field); + auto one_instance = field_type + "_"; + one_instance[0] = CharToLower(one_instance[0]); + if (parser_.opts.include_dependence_headers) { + auto package_reference = GenPackageReference(field.value.type); + field_type = package_reference + "." + TypeName(field); + } + code += GenIndents(4) + "if " + struct_var + "." + field_method + + "(i) is None:"; + code += GenIndents(5) + "self." + field_field + ".append(None)"; code += GenIndents(4) + "else:"; - code += GenIndents(5) + one_instance + " = " + field_type_name + - "T.InitFromObj(" + struct_instance_name + "." + - field_accessor_name + "(i))"; - code += GenIndents(5) + "self." + field_instance_name + ".append(" + - one_instance + ")"; + code += GenIndents(5) + one_instance + " = " + namer_.ObjectType(field_type) + + ".InitFromObj(" + struct_var + "." + field_method + "(i))"; + code += + GenIndents(5) + "self." + field_field + ".append(" + one_instance + ")"; } void GenUnpackforScalarVectorHelper(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr, int indents) { + std::string *code_ptr, + int indents) const { auto &code = *code_ptr; - auto field_instance_name = MakeLowerCamel(field); - auto field_accessor_name = MakeUpperCamel(field); - auto struct_instance_name = MakeLowerCamel(struct_def); + const auto field_field = namer_.Field(field); + const auto field_method = namer_.Method(field); + const auto struct_var = namer_.Variable(struct_def); - code += GenIndents(indents) + "self." + field_instance_name + " = []"; - code += GenIndents(indents) + "for i in range(" + struct_instance_name + - "." + field_accessor_name + "Length()):"; - code += GenIndents(indents + 1) + "self." + field_instance_name + - ".append(" + struct_instance_name + "." + field_accessor_name + - "(i))"; + code += GenIndents(indents) + "self." + field_field + " = []"; + code += GenIndents(indents) + "for i in range(" + struct_var + "." + + field_method + "Length()):"; + code += GenIndents(indents + 1) + "self." + field_field + ".append(" + + struct_var + "." + field_method + "(i))"; } void GenUnPackForScalarVector(const StructDef &struct_def, - const FieldDef &field, std::string *code_ptr) { + const FieldDef &field, + std::string *code_ptr) const { auto &code = *code_ptr; - auto field_instance_name = MakeLowerCamel(field); - auto field_accessor_name = MakeUpperCamel(field); - auto struct_instance_name = MakeLowerCamel(struct_def); + const auto field_field = namer_.Field(field); + const auto field_method = namer_.Method(field); + const auto struct_var = namer_.Variable(struct_def); - code += GenIndents(2) + "if not " + struct_instance_name + "." + - field_accessor_name + "IsNone():"; + code += GenIndents(2) + "if not " + struct_var + "." + field_method + + "IsNone():"; // String does not have the AsNumpy method. if (!(IsScalar(field.value.type.VectorType().base_type))) { @@ -1183,23 +1443,23 @@ class PythonGenerator : public BaseGenerator { // If numpy exists, use the AsNumpy method to optimize the unpack speed. code += GenIndents(3) + "else:"; - code += GenIndents(4) + "self." + field_instance_name + " = " + - struct_instance_name + "." + field_accessor_name + "AsNumpy()"; + code += GenIndents(4) + "self." + field_field + " = " + struct_var + "." + + field_method + "AsNumpy()"; } void GenUnPackForScalar(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; - auto field_instance_name = MakeLowerCamel(field); - auto field_accessor_name = MakeUpperCamel(field); - auto struct_instance_name = MakeLowerCamel(struct_def); + const auto field_field = namer_.Field(field); + const auto field_method = namer_.Method(field); + const auto struct_var = namer_.Variable(struct_def); - code += GenIndents(2) + "self." + field_instance_name + " = " + - struct_instance_name + "." + field_accessor_name + "()"; + code += GenIndents(2) + "self." + field_field + " = " + struct_var + "." + + field_method + "()"; } // Generates the UnPack method for the object class. - void GenUnPack(const StructDef &struct_def, std::string *code_ptr) { + void GenUnPack(const StructDef &struct_def, std::string *code_ptr) const { std::string code; // Items that needs to be imported. No duplicate modules will be imported. std::set<std::string> import_list; @@ -1219,6 +1479,7 @@ class PythonGenerator : public BaseGenerator { GenUnPackForUnion(struct_def, field, &code); break; } + case BASE_TYPE_ARRAY: case BASE_TYPE_VECTOR: { auto vectortype = field.value.type.VectorType(); if (vectortype.base_type == BASE_TYPE_STRUCT) { @@ -1228,22 +1489,17 @@ class PythonGenerator : public BaseGenerator { } break; } - case BASE_TYPE_ARRAY: { - GenUnPackForScalarVector(struct_def, field, &code); - break; - } default: GenUnPackForScalar(struct_def, field, &code); } } // Writes import statements and code into the generated file. auto &code_base = *code_ptr; - auto struct_instance_name = MakeLowerCamel(struct_def); - auto struct_name = MakeUpperCamel(struct_def); + const auto struct_var = namer_.Variable(struct_def); GenReceiverForObjectAPI(struct_def, code_ptr); - code_base += "_UnPack(self, " + struct_instance_name + "):"; - code_base += GenIndents(2) + "if " + struct_instance_name + " is None:"; + code_base += "_UnPack(self, " + struct_var + "):"; + code_base += GenIndents(2) + "if " + struct_var + " is None:"; code_base += GenIndents(3) + "return"; // Write the import statements. @@ -1257,13 +1513,14 @@ class PythonGenerator : public BaseGenerator { code_base += "\n"; } - void GenPackForStruct(const StructDef &struct_def, std::string *code_ptr) { + void GenPackForStruct(const StructDef &struct_def, + std::string *code_ptr) const { auto &code = *code_ptr; - auto struct_name = MakeUpperCamel(struct_def); + const auto struct_fn = namer_.Function(struct_def); GenReceiverForObjectAPI(struct_def, code_ptr); code += "Pack(self, builder):"; - code += GenIndents(2) + "return Create" + struct_name + "(builder"; + code += GenIndents(2) + "return Create" + struct_fn + "(builder"; StructBuilderArgs(struct_def, /* nameprefix = */ "self.", @@ -1276,65 +1533,61 @@ class PythonGenerator : public BaseGenerator { void GenPackForStructVectorField(const StructDef &struct_def, const FieldDef &field, std::string *code_prefix_ptr, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code_prefix = *code_prefix_ptr; auto &code = *code_ptr; - auto field_instance_name = MakeLowerCamel(field); - auto struct_name = NormalizedName(struct_def); - auto field_accessor_name = MakeUpperCamel(field); + const auto field_field = namer_.Field(field); + const auto struct_type = namer_.Type(struct_def); + const auto field_method = namer_.Method(field); // Creates the field. - code_prefix += - GenIndents(2) + "if self." + field_instance_name + " is not None:"; + code_prefix += GenIndents(2) + "if self." + field_field + " is not None:"; if (field.value.type.struct_def->fixed) { - code_prefix += GenIndents(3) + "Start" + - field_accessor_name + "Vector(builder, len(self." + - field_instance_name + "))"; + code_prefix += GenIndents(3) + struct_type + "Start" + field_method + + "Vector(builder, len(self." + field_field + "))"; code_prefix += GenIndents(3) + "for i in reversed(range(len(self." + - field_instance_name + "))):"; + field_field + "))):"; code_prefix += - GenIndents(4) + "self." + field_instance_name + "[i].Pack(builder)"; - code_prefix += - GenIndents(3) + field_instance_name + " = builder.EndVector()"; + GenIndents(4) + "self." + field_field + "[i].Pack(builder)"; + code_prefix += GenIndents(3) + field_field + " = builder.EndVector()"; } else { // If the vector is a struct vector, we need to first build accessor for // each struct element. - code_prefix += GenIndents(3) + field_instance_name + "list = []"; + code_prefix += GenIndents(3) + field_field + "list = []"; code_prefix += GenIndents(3); - code_prefix += "for i in range(len(self." + field_instance_name + ")):"; - code_prefix += GenIndents(4) + field_instance_name + "list.append(self." + - field_instance_name + "[i].Pack(builder))"; + code_prefix += "for i in range(len(self." + field_field + ")):"; + code_prefix += GenIndents(4) + field_field + "list.append(self." + + field_field + "[i].Pack(builder))"; - code_prefix += GenIndents(3) + "Start" + - field_accessor_name + "Vector(builder, len(self." + - field_instance_name + "))"; + code_prefix += GenIndents(3) + struct_type + "Start" + field_method + + "Vector(builder, len(self." + field_field + "))"; code_prefix += GenIndents(3) + "for i in reversed(range(len(self." + - field_instance_name + "))):"; + field_field + "))):"; code_prefix += GenIndents(4) + "builder.PrependUOffsetTRelative" + "(" + - field_instance_name + "list[i])"; - code_prefix += - GenIndents(3) + field_instance_name + " = builder.EndVector()"; + field_field + "list[i])"; + code_prefix += GenIndents(3) + field_field + " = builder.EndVector()"; } // Adds the field into the struct. - code += GenIndents(2) + "if self." + field_instance_name + " is not None:"; - code += GenIndents(3) + "Add" + field_accessor_name + - "(builder, " + field_instance_name + ")"; + code += GenIndents(2) + "if self." + field_field + " is not None:"; + code += GenIndents(3) + struct_type + "Add" + field_method + "(builder, " + + field_field + ")"; } void GenPackForScalarVectorFieldHelper(const StructDef &struct_def, const FieldDef &field, - std::string *code_ptr, int indents) { + std::string *code_ptr, + int indents) const { auto &code = *code_ptr; - auto field_instance_name = MakeLowerCamel(field); - auto field_accessor_name = MakeUpperCamel(field); - auto struct_name = NormalizedName(struct_def); - auto vectortype = field.value.type.VectorType(); + const auto field_field = namer_.Field(field); + const auto field_method = namer_.Method(field); + const auto struct_type = namer_.Type(struct_def); + const auto vectortype = field.value.type.VectorType(); - code += GenIndents(indents) + "Start" + field_accessor_name + - "Vector(builder, len(self." + field_instance_name + "))"; + code += GenIndents(indents) + struct_type + "Start" + field_method + + "Vector(builder, len(self." + field_field + "))"; code += GenIndents(indents) + "for i in reversed(range(len(self." + - field_instance_name + "))):"; + field_field + "))):"; code += GenIndents(indents + 1) + "builder.Prepend"; std::string type_name; @@ -1359,118 +1612,109 @@ class PythonGenerator : public BaseGenerator { void GenPackForScalarVectorField(const StructDef &struct_def, const FieldDef &field, std::string *code_prefix_ptr, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; auto &code_prefix = *code_prefix_ptr; - auto field_instance_name = MakeLowerCamel(field); - auto field_accessor_name = MakeUpperCamel(field); - auto struct_name = NormalizedName(struct_def); + const auto field_field = namer_.Field(field); + const auto field_method = namer_.Method(field); + const auto struct_type = namer_.Type(struct_def); // Adds the field into the struct. - code += GenIndents(2) + "if self." + field_instance_name + " is not None:"; - code += GenIndents(3) + "Add" + field_accessor_name + - "(builder, " + field_instance_name + ")"; + code += GenIndents(2) + "if self." + field_field + " is not None:"; + code += GenIndents(3) + struct_type + "Add" + field_method + "(builder, " + + field_field + ")"; // Creates the field. - code_prefix += - GenIndents(2) + "if self." + field_instance_name + " is not None:"; + code_prefix += GenIndents(2) + "if self." + field_field + " is not None:"; // If the vector is a string vector, we need to first build accessor for // each string element. And this generated code, needs to be // placed ahead of code_prefix. auto vectortype = field.value.type.VectorType(); if (IsString(vectortype)) { - code_prefix += GenIndents(3) + MakeLowerCamel(field) + "list = []"; - code_prefix += GenIndents(3) + "for i in range(len(self." + - field_instance_name + ")):"; - code_prefix += GenIndents(4) + MakeLowerCamel(field) + - "list.append(builder.CreateString(self." + - field_instance_name + "[i]))"; - GenPackForScalarVectorFieldHelper(struct_def, field, code_prefix_ptr, 3); - code_prefix += "(" + MakeLowerCamel(field) + "list[i])"; + code_prefix += GenIndents(3) + field_field + "list = []"; code_prefix += - GenIndents(3) + field_instance_name + " = builder.EndVector()"; + GenIndents(3) + "for i in range(len(self." + field_field + ")):"; + code_prefix += GenIndents(4) + field_field + + "list.append(builder.CreateString(self." + field_field + + "[i]))"; + GenPackForScalarVectorFieldHelper(struct_def, field, code_prefix_ptr, 3); + code_prefix += "(" + field_field + "list[i])"; + code_prefix += GenIndents(3) + field_field + " = builder.EndVector()"; return; } code_prefix += GenIndents(3) + "if np is not None and type(self." + - field_instance_name + ") is np.ndarray:"; - code_prefix += GenIndents(4) + field_instance_name + - " = builder.CreateNumpyVector(self." + field_instance_name + - ")"; + field_field + ") is np.ndarray:"; + code_prefix += GenIndents(4) + field_field + + " = builder.CreateNumpyVector(self." + field_field + ")"; code_prefix += GenIndents(3) + "else:"; GenPackForScalarVectorFieldHelper(struct_def, field, code_prefix_ptr, 4); - code_prefix += "(self." + field_instance_name + "[i])"; - code_prefix += - GenIndents(4) + field_instance_name + " = builder.EndVector()"; + code_prefix += "(self." + field_field + "[i])"; + code_prefix += GenIndents(4) + field_field + " = builder.EndVector()"; } void GenPackForStructField(const StructDef &struct_def, const FieldDef &field, std::string *code_prefix_ptr, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code_prefix = *code_prefix_ptr; auto &code = *code_ptr; - auto field_instance_name = MakeLowerCamel(field); - - auto field_accessor_name = MakeUpperCamel(field); - auto struct_name = NormalizedName(struct_def); + const auto field_field = namer_.Field(field); + const auto field_method = namer_.Method(field); + const auto struct_type = namer_.Type(struct_def); if (field.value.type.struct_def->fixed) { // Pure struct fields need to be created along with their parent // structs. - code += - GenIndents(2) + "if self." + field_instance_name + " is not None:"; - code += GenIndents(3) + field_instance_name + " = self." + - field_instance_name + ".Pack(builder)"; + code += GenIndents(2) + "if self." + field_field + " is not None:"; + code += GenIndents(3) + field_field + " = self." + field_field + + ".Pack(builder)"; } else { // Tables need to be created before their parent structs are created. - code_prefix += - GenIndents(2) + "if self." + field_instance_name + " is not None:"; - code_prefix += GenIndents(3) + field_instance_name + " = self." + - field_instance_name + ".Pack(builder)"; - code += - GenIndents(2) + "if self." + field_instance_name + " is not None:"; + code_prefix += GenIndents(2) + "if self." + field_field + " is not None:"; + code_prefix += GenIndents(3) + field_field + " = self." + field_field + + ".Pack(builder)"; + code += GenIndents(2) + "if self." + field_field + " is not None:"; } - code += GenIndents(3) + "Add" + field_accessor_name + - "(builder, " + field_instance_name + ")"; + code += GenIndents(3) + struct_type + "Add" + field_method + "(builder, " + + field_field + ")"; } void GenPackForUnionField(const StructDef &struct_def, const FieldDef &field, std::string *code_prefix_ptr, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code_prefix = *code_prefix_ptr; auto &code = *code_ptr; - auto field_instance_name = MakeLowerCamel(field); - - auto field_accessor_name = MakeUpperCamel(field); - auto struct_name = NormalizedName(struct_def); + const auto field_field = namer_.Field(field); + const auto field_method = namer_.Method(field); + const auto struct_type = namer_.Type(struct_def); // TODO(luwa): TypeT should be moved under the None check as well. - code_prefix += - GenIndents(2) + "if self." + field_instance_name + " is not None:"; - code_prefix += GenIndents(3) + field_instance_name + " = self." + - field_instance_name + ".Pack(builder)"; - code += GenIndents(2) + "if self." + field_instance_name + " is not None:"; - code += GenIndents(3) + "Add" + field_accessor_name + - "(builder, " + field_instance_name + ")"; + code_prefix += GenIndents(2) + "if self." + field_field + " is not None:"; + code_prefix += GenIndents(3) + field_field + " = self." + field_field + + ".Pack(builder)"; + code += GenIndents(2) + "if self." + field_field + " is not None:"; + code += GenIndents(3) + struct_type + "Add" + field_method + "(builder, " + + field_field + ")"; } - void GenPackForTable(const StructDef &struct_def, std::string *code_ptr) { + void GenPackForTable(const StructDef &struct_def, + std::string *code_ptr) const { auto &code_base = *code_ptr; std::string code, code_prefix; - auto struct_instance_name = MakeLowerCamel(struct_def); - auto struct_name = NormalizedName(struct_def); + const auto struct_var = namer_.Variable(struct_def); + const auto struct_type = namer_.Type(struct_def); GenReceiverForObjectAPI(struct_def, code_ptr); code_base += "Pack(self, builder):"; - code += GenIndents(2) + "Start(builder)"; + code += GenIndents(2) + struct_type + "Start(builder)"; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { auto &field = **it; if (field.deprecated) continue; - auto field_accessor_name = MakeUpperCamel(field); - auto field_instance_name = MakeLowerCamel(field); + const auto field_method = namer_.Method(field); + const auto field_field = namer_.Field(field); switch (field.value.type.base_type) { case BASE_TYPE_STRUCT: { @@ -1481,6 +1725,7 @@ class PythonGenerator : public BaseGenerator { GenPackForUnionField(struct_def, field, &code_prefix, &code); break; } + case BASE_TYPE_ARRAY: case BASE_TYPE_VECTOR: { auto vectortype = field.value.type.VectorType(); if (vectortype.base_type == BASE_TYPE_STRUCT) { @@ -1490,41 +1735,35 @@ class PythonGenerator : public BaseGenerator { } break; } - case BASE_TYPE_ARRAY: { - GenPackForScalarVectorField(struct_def, field, &code_prefix, &code); - break; - } case BASE_TYPE_STRING: { - code_prefix += GenIndents(2) + "if self." + field_instance_name + - " is not None:"; - code_prefix += GenIndents(3) + field_instance_name + - " = builder.CreateString(self." + field_instance_name + - ")"; - code += GenIndents(2) + "if self." + field_instance_name + - " is not None:"; - code += GenIndents(3) + "Add" + field_accessor_name + - "(builder, " + field_instance_name + ")"; + code_prefix += + GenIndents(2) + "if self." + field_field + " is not None:"; + code_prefix += GenIndents(3) + field_field + + " = builder.CreateString(self." + field_field + ")"; + code += GenIndents(2) + "if self." + field_field + " is not None:"; + code += GenIndents(3) + struct_type + "Add" + field_method + + "(builder, " + field_field + ")"; break; } default: // Generates code for scalar values. If the value equals to the // default value, builder will automatically ignore it. So we don't // need to check the value ahead. - code += GenIndents(2) + "Add" + field_accessor_name + - "(builder, self." + field_instance_name + ")"; + code += GenIndents(2) + struct_type + "Add" + field_method + + "(builder, self." + field_field + ")"; break; } } - code += GenIndents(2) + struct_instance_name + " = " + "End(builder)"; - code += GenIndents(2) + "return " + struct_instance_name; + code += GenIndents(2) + struct_var + " = " + struct_type + "End(builder)"; + code += GenIndents(2) + "return " + struct_var; code_base += code_prefix + code; code_base += "\n"; } void GenStructForObjectAPI(const StructDef &struct_def, - std::string *code_ptr) { + std::string *code_ptr) const { if (struct_def.generated) return; std::set<std::string> import_list; @@ -1537,8 +1776,14 @@ class PythonGenerator : public BaseGenerator { InitializeFromBuf(struct_def, &code); + InitializeFromPackedBuf(struct_def, &code); + InitializeFromObjForObject(struct_def, &code); + if (parser_.opts.gen_compare) { + GenCompareOperator(struct_def, &code); + } + GenUnPack(struct_def, &code); if (struct_def.fixed) { @@ -1558,14 +1803,14 @@ class PythonGenerator : public BaseGenerator { } void GenUnionCreatorForStruct(const EnumDef &enum_def, const EnumVal &ev, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; - auto union_name = NormalizedName(enum_def); - auto field_name = NormalizedName(ev); - auto field_type = GenTypeGet(ev.union_type) + "T"; + const auto union_type = namer_.Type(enum_def); + const auto variant = namer_.Variant(ev); + auto field_type = namer_.ObjectType(*ev.union_type.struct_def); - code += GenIndents(1) + "if unionType == " + union_name + "()." + - field_name + ":"; + code += + GenIndents(1) + "if unionType == " + union_type + "()." + variant + ":"; if (parser_.opts.include_dependence_headers) { auto package_reference = GenPackageReference(ev.union_type); code += GenIndents(2) + "import " + package_reference; @@ -1576,25 +1821,27 @@ class PythonGenerator : public BaseGenerator { } void GenUnionCreatorForString(const EnumDef &enum_def, const EnumVal &ev, - std::string *code_ptr) { + std::string *code_ptr) const { auto &code = *code_ptr; - auto union_name = NormalizedName(enum_def); - auto field_name = NormalizedName(ev); + const auto union_type = namer_.Type(enum_def); + const auto variant = namer_.Variant(ev); - code += GenIndents(1) + "if unionType == " + union_name + "()." + - field_name + ":"; + code += + GenIndents(1) + "if unionType == " + union_type + "()." + variant + ":"; code += GenIndents(2) + "tab = Table(table.Bytes, table.Pos)"; code += GenIndents(2) + "union = tab.String(table.Pos)"; code += GenIndents(2) + "return union"; } // Creates an union object based on union type. - void GenUnionCreator(const EnumDef &enum_def, std::string *code_ptr) { + void GenUnionCreator(const EnumDef &enum_def, std::string *code_ptr) const { + if (enum_def.generated) return; + auto &code = *code_ptr; - auto union_name = MakeUpperCamel(enum_def); + const auto enum_fn = namer_.Function(enum_def); code += "\n"; - code += "def " + union_name + "Creator(unionType, table):"; + code += "def " + enum_fn + "Creator(unionType, table):"; code += GenIndents(1) + "from flatbuffers.table import Table"; code += GenIndents(1) + "if not isinstance(table, Table):"; code += GenIndents(2) + "return None"; @@ -1617,39 +1864,63 @@ class PythonGenerator : public BaseGenerator { } // Generate enum declarations. - void GenEnum(const EnumDef &enum_def, std::string *code_ptr) { + void GenEnum(const EnumDef &enum_def, std::string *code_ptr) const { if (enum_def.generated) return; GenComment(enum_def.doc_comment, code_ptr, &def_comment); - BeginEnum(NormalizedName(enum_def), code_ptr); + BeginEnum(enum_def, code_ptr); for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { auto &ev = **it; GenComment(ev.doc_comment, code_ptr, &def_comment, Indent.c_str()); EnumMember(enum_def, ev, code_ptr); } - EndEnum(code_ptr); } // Returns the function name that is able to read a value of the given type. - std::string GenGetter(const Type &type) { + std::string GenGetter(const Type &type) const { switch (type.base_type) { case BASE_TYPE_STRING: return "self._tab.String("; case BASE_TYPE_UNION: return "self._tab.Union("; case BASE_TYPE_VECTOR: return GenGetter(type.VectorType()); default: return "self._tab.Get(flatbuffers.number_types." + - MakeCamel(GenTypeGet(type)) + "Flags, "; + namer_.Method(GenTypeGet(type)) + "Flags, "; + } + } + + std::string GenFieldTy(const FieldDef &field) const { + if (IsScalar(field.value.type.base_type) || IsArray(field.value.type)) { + const std::string ty = GenTypeBasic(field.value.type); + if (ty.find("int") != std::string::npos) { + return "int"; + } + + if (ty.find("float") != std::string::npos) { + return "float"; + } + + if (ty == "bool") { + return "bool"; + } + + return "Any"; + } else { + if (IsStruct(field.value.type)) { + return "Any"; + } else { + return "int"; + } } } // Returns the method name for use with add/put calls. - std::string GenMethod(const FieldDef &field) { + std::string GenMethod(const FieldDef &field) const { return (IsScalar(field.value.type.base_type) || IsArray(field.value.type)) - ? MakeCamel(GenTypeBasic(field.value.type)) + ? namer_.Method(GenTypeBasic(field.value.type)) : (IsStruct(field.value.type) ? "Struct" : "UOffsetTRelative"); } - std::string GenTypeBasic(const Type &type) { + std::string GenTypeBasic(const Type &type) const { // clang-format off static const char *ctypename[] = { #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ @@ -1663,10 +1934,12 @@ class PythonGenerator : public BaseGenerator { : type.base_type]; } - std::string GenTypePointer(const Type &type) { + std::string GenTypePointer(const Type &type) const { switch (type.base_type) { case BASE_TYPE_STRING: return "string"; - case BASE_TYPE_VECTOR: return GenTypeGet(type.VectorType()); + case BASE_TYPE_VECTOR: + // fall through + case BASE_TYPE_ARRAY: return GenTypeGet(type.VectorType()); case BASE_TYPE_STRUCT: return type.struct_def->name; case BASE_TYPE_UNION: // fall through @@ -1674,16 +1947,42 @@ class PythonGenerator : public BaseGenerator { } } - std::string GenTypeGet(const Type &type) { + std::string GenTypeGet(const Type &type) const { return IsScalar(type.base_type) ? GenTypeBasic(type) : GenTypePointer(type); } - std::string TypeName(const FieldDef &field) { + std::string TypeName(const FieldDef &field) const { return GenTypeGet(field.value.type); } + std::string ReturnType(const StructDef &struct_def, + const FieldDef &field) const { + // If we have a class member that returns an instance of the same class, + // for example: + // class Field(object): + // def Children(self, j: int) -> Optional[Field]: + // pass + // + // we need to quote the return type: + // class Field(object): + // def Children(self, j: int) -> Optional['Field']: + // pass + // + // because Python is unable to resolve the name during parse and will return + // an error. + // (see PEP 484 under forward references: + // https://peps.python.org/pep-0484/#forward-references) + const std::string self_type = struct_def.name; + std::string field_type = TypeName(field); + + if (self_type == field_type) { field_type = "'" + field_type + "'"; } + + return field_type; + } + // Create a struct with a builder and the struct's arguments. - void GenStructBuilder(const StructDef &struct_def, std::string *code_ptr) { + void GenStructBuilder(const StructDef &struct_def, + std::string *code_ptr) const { BeginBuilderArgs(struct_def, code_ptr); StructBuilderArgs(struct_def, /* nameprefix = */ "", @@ -1697,13 +1996,24 @@ class PythonGenerator : public BaseGenerator { } bool generate() { - if (!generateEnums()) return false; - if (!generateStructs()) return false; + std::string one_file_code; + ImportMap one_file_imports; + if (!generateEnums(&one_file_code)) return false; + if (!generateStructs(&one_file_code, one_file_imports)) return false; + + if (parser_.opts.one_file) { + const std::string mod = file_name_ + "_generated"; + + // Legacy file format uses keep casing. + return SaveType(mod + ".py", *parser_.current_namespace_, one_file_code, + one_file_imports, mod, true); + } + return true; } private: - bool generateEnums() { + bool generateEnums(std::string *one_file_code) const { for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end(); ++it) { auto &enum_def = **it; @@ -1712,63 +2022,115 @@ class PythonGenerator : public BaseGenerator { if (parser_.opts.generate_object_based_api & enum_def.is_union) { GenUnionCreator(enum_def, &enumcode); } - if (!SaveType(enum_def, enumcode, false)) return false; + + if (parser_.opts.one_file && !enumcode.empty()) { + *one_file_code += enumcode + "\n\n"; + } else { + ImportMap imports; + const std::string mod = + namer_.File(enum_def, SkipFile::SuffixAndExtension); + + if (!SaveType(namer_.File(enum_def, SkipFile::Suffix), + *enum_def.defined_namespace, enumcode, imports, mod, + false)) + return false; + } } return true; } - bool generateStructs() { + bool generateStructs(std::string *one_file_code, + ImportMap &one_file_imports) const { for (auto it = parser_.structs_.vec.begin(); it != parser_.structs_.vec.end(); ++it) { auto &struct_def = **it; std::string declcode; - GenStruct(struct_def, &declcode); + ImportMap imports; + GenStruct(struct_def, &declcode, imports); if (parser_.opts.generate_object_based_api) { GenStructForObjectAPI(struct_def, &declcode); } - if (!SaveType(struct_def, declcode, true)) return false; + + if (parser_.opts.one_file) { + if (!declcode.empty()) { + *one_file_code += declcode + "\n\n"; + } + + for (auto import_str: imports) { + one_file_imports.insert(import_str); + } + } else { + const std::string mod = + namer_.File(struct_def, SkipFile::SuffixAndExtension); + if (!SaveType(namer_.File(struct_def, SkipFile::Suffix), + *struct_def.defined_namespace, declcode, imports, mod, + true)) + return false; + } } return true; } // Begin by declaring namespace and imports. void BeginFile(const std::string &name_space_name, const bool needs_imports, - std::string *code_ptr) { + std::string *code_ptr, const std::string &mod, + const ImportMap &imports) const { auto &code = *code_ptr; code = code + "# " + FlatBuffersGeneratedWarning() + "\n\n"; code += "# namespace: " + name_space_name + "\n\n"; + if (needs_imports) { + const std::string local_import = "." + mod; + code += "import flatbuffers\n"; code += "from flatbuffers.compat import import_numpy\n"; + if (parser_.opts.python_typing) { + code += "from typing import Any\n"; + + for (auto import_entry : imports) { + // If we have a file called, say, "MyType.py" and in it we have a + // class "MyType", we can generate imports -- usually when we + // have a type that contains arrays of itself -- of the type + // "from .MyType import MyType", which Python can't resolve. So + // if we are trying to import ourself, we skip. + if (import_entry.first != local_import) { + code += "from " + import_entry.first + " import " + + import_entry.second + "\n"; + } + } + } code += "np = import_numpy()\n\n"; } } // Save out the generated code for a Python Table type. - bool SaveType(const Definition &def, const std::string &classcode, - bool needs_imports) { + bool SaveType(const std::string &defname, const Namespace &ns, + const std::string &classcode, const ImportMap &imports, + const std::string &mod, bool needs_imports) const { if (!classcode.length()) return true; - std::string namespace_dir = path_; - auto &namespaces = def.defined_namespace->components; - for (auto it = namespaces.begin(); it != namespaces.end(); ++it) { - if (it != namespaces.begin()) namespace_dir += kPathSeparator; - namespace_dir += *it; - std::string init_py_filename = namespace_dir + "/__init__.py"; - SaveFile(init_py_filename.c_str(), "", false); - } - std::string code = ""; - BeginFile(LastNamespacePart(*def.defined_namespace), needs_imports, &code); + BeginFile(LastNamespacePart(ns), needs_imports, &code, mod, imports); code += classcode; - std::string filename = - NamespaceDir(*def.defined_namespace) + NormalizedName(def) + ".py"; + + const std::string directories = + parser_.opts.one_file ? path_ : namer_.Directories(ns.components); + EnsureDirExists(directories); + + for (size_t i = path_.size() + 1; i != std::string::npos; + i = directories.find(kPathSeparator, i + 1)) { + const std::string init_py = + directories.substr(0, i) + kPathSeparator + "__init__.py"; + SaveFile(init_py.c_str(), "", false); + } + + const std::string filename = directories + defname; return SaveFile(filename.c_str(), code, false); } private: - std::unordered_set<std::string> keywords_; const SimpleFloatConstantGenerator float_const_gen_; + const IdlNamer namer_; }; } // namespace python @@ -1779,4 +2141,58 @@ bool GeneratePython(const Parser &parser, const std::string &path, return generator.generate(); } +namespace { + +class PythonCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GeneratePython(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + (void)parser; + (void)path; + (void)filename; + (void)output; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GeneratePythonGRPC(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return false; } + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kPython; } + + std::string LanguageName() const override { return "Python"; } +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewPythonCodeGenerator() { + return std::unique_ptr<PythonCodeGenerator>(new PythonCodeGenerator()); +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_python.h b/contrib/libs/flatbuffers/src/idl_gen_python.h new file mode 100644 index 00000000000..cd0cf9f4fd1 --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_python.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_PYTHON_H_ +#define FLATBUFFERS_IDL_GEN_PYTHON_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Python code generator. +std::unique_ptr<CodeGenerator> NewPythonCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_PYTHON_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_rust.cpp b/contrib/libs/flatbuffers/src/idl_gen_rust.cpp index 455780cd948..ac6097faae4 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_rust.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_rust.cpp @@ -16,39 +16,129 @@ // independent from idl_parser, since this code is not needed for most clients +#include "idl_gen_rust.h" + +#include <cmath> + #include "flatbuffers/code_generators.h" #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" +#include "idl_namer.h" namespace flatbuffers { - -// Convert a camelCaseIdentifier or CamelCaseIdentifier to a -// snake_case_identifier. -std::string MakeSnakeCase(const std::string &in) { - std::string s; - for (size_t i = 0; i < in.length(); i++) { - if (i == 0) { - s += CharToLower(in[0]); - } else if (in[i] == '_') { - s += '_'; - } else if (!islower(in[i])) { - // Prevent duplicate underscores for Upper_Snake_Case strings - // and UPPERCASE strings. - if (islower(in[i - 1])) { s += '_'; } - s += CharToLower(in[i]); - } else { - s += in[i]; - } - } - return s; +namespace { + +static Namer::Config RustDefaultConfig() { + // Historical note: We've been using "keep" casing since the original + // implementation, presumably because Flatbuffers schema style and Rust style + // roughly align. We are not going to enforce proper casing since its an + // unnecessary breaking change. + return { /*types=*/Case::kKeep, + /*constants=*/Case::kScreamingSnake, + /*methods=*/Case::kSnake, + /*functions=*/Case::kSnake, + /*fields=*/Case::kKeep, + /*variables=*/Case::kUnknown, // Unused. + /*variants=*/Case::kKeep, + /*enum_variant_seperator=*/"::", + /*escape_keywords=*/Namer::Config::Escape::BeforeConvertingCase, + /*namespaces=*/Case::kSnake, + /*namespace_seperator=*/"::", + /*object_prefix=*/"", + /*object_suffix=*/"T", + /*keyword_prefix=*/"", + /*keyword_suffix=*/"_", + /*filenames=*/Case::kSnake, + /*directories=*/Case::kSnake, + /*output_path=*/"", + /*filename_suffix=*/"_generated", + /*filename_extension=*/".rs" }; } -// Convert a string to all uppercase. -std::string MakeUpper(const std::string &in) { - std::string s; - for (size_t i = 0; i < in.length(); i++) { s += CharToUpper(in[i]); } - return s; +static std::set<std::string> RustKeywords() { + return { + // https://doc.rust-lang.org/book/second-edition/appendix-01-keywords.html + "as", + "break", + "const", + "continue", + "crate", + "else", + "enum", + "extern", + "false", + "fn", + "for", + "if", + "impl", + "in", + "let", + "loop", + "match", + "mod", + "move", + "mut", + "pub", + "ref", + "return", + "Self", + "self", + "static", + "struct", + "super", + "trait", + "true", + "type", + "unsafe", + "use", + "where", + "while", + // future possible keywords + "abstract", + "alignof", + "become", + "box", + "do", + "final", + "macro", + "offsetof", + "override", + "priv", + "proc", + "pure", + "sizeof", + "typeof", + "unsized", + "virtual", + "yield", + // other rust terms we should not use + "std", + "usize", + "isize", + "u8", + "i8", + "u16", + "i16", + "u32", + "i32", + "u64", + "i64", + "u128", + "i128", + "f32", + "f64", + // Terms that we use ourselves + "follow", + "push", + "size", + "alignment", + "to_little_endian", + "from_little_endian", + "ENUM_MAX", + "ENUM_MIN", + "ENUM_VALUES", + }; } // Encapsulate all logical field types in this enum. This allows us to write @@ -88,7 +178,7 @@ enum FullType { }; // Convert a Type to a FullType (exhaustive). -FullType GetFullType(const Type &type) { +static FullType GetFullType(const Type &type) { // N.B. The order of these conditionals matters for some types. if (IsString(type)) { @@ -178,37 +268,81 @@ FullType GetFullType(const Type &type) { return ftBool; } -// If the second parameter is false then wrap the first with Option<...> -std::string WrapInOptionIfNotRequired(std::string s, bool required) { - if (required) { - return s; - } else { - return "Option<" + s + ">"; - } -} - -// If the second parameter is false then add .unwrap() -std::string AddUnwrapIfRequired(std::string s, bool required) { - if (required) { - return s + ".unwrap()"; - } else { - return s; - } -} - -bool IsBitFlagsEnum(const EnumDef &enum_def) { +static bool IsBitFlagsEnum(const EnumDef &enum_def) { return enum_def.attributes.Lookup("bit_flags") != nullptr; } -bool IsBitFlagsEnum(const FieldDef &field) { - EnumDef *ed = field.value.type.enum_def; - return ed && IsBitFlagsEnum(*ed); -} // TableArgs make required non-scalars "Option<_>". // TODO(cneo): Rework how we do defaults and stuff. -bool IsOptionalToBuilder(const FieldDef &field) { +static bool IsOptionalToBuilder(const FieldDef &field) { return field.IsOptional() || !IsScalar(field.value.type.base_type); } +} // namespace + +bool GenerateRustModuleRootFile(const Parser &parser, + const std::string &output_dir) { + if (!parser.opts.rust_module_root_file) { + // Don't generate a root file when generating one file. This isn't an error + // so return true. + return true; + } + Namer namer(WithFlagOptions(RustDefaultConfig(), parser.opts, output_dir), + RustKeywords()); + // We gather the symbols into a tree of namespaces (which are rust mods) and + // generate a file that gathers them all. + struct Module { + std::map<std::string, Module> sub_modules; + std::vector<std::string> generated_files; + // Add a symbol into the tree. + void Insert(const Namer &namer, const Definition *s) { + const Definition &symbol = *s; + Module *current_module = this; + for (auto it = symbol.defined_namespace->components.begin(); + it != symbol.defined_namespace->components.end(); it++) { + std::string ns_component = namer.Namespace(*it); + current_module = ¤t_module->sub_modules[ns_component]; + } + current_module->generated_files.push_back( + namer.File(symbol.name, SkipFile::Extension)); + } + // Recursively create the importer file. + void GenerateImports(CodeWriter &code) { + for (auto it = sub_modules.begin(); it != sub_modules.end(); it++) { + code += "pub mod " + it->first + " {"; + code.IncrementIdentLevel(); + code += "use super::*;"; + it->second.GenerateImports(code); + code.DecrementIdentLevel(); + code += "} // " + it->first; + } + for (auto it = generated_files.begin(); it != generated_files.end(); + it++) { + code += "mod " + *it + ";"; + code += "pub use self::" + *it + "::*;"; + } + } + }; + Module root_module; + for (auto it = parser.enums_.vec.begin(); it != parser.enums_.vec.end(); + it++) { + root_module.Insert(namer, *it); + } + for (auto it = parser.structs_.vec.begin(); it != parser.structs_.vec.end(); + it++) { + root_module.Insert(namer, *it); + } + CodeWriter code(" "); + // TODO(caspern): Move generated warning out of BaseGenerator. + code += + "// Automatically generated by the Flatbuffers compiler. " + "Do not modify."; + code += "// @generated"; + root_module.GenerateImports(code); + const bool success = + SaveFile((output_dir + "mod.rs").c_str(), code.ToString(), false); + code.Clear(); + return success; +} namespace rust { @@ -217,118 +351,87 @@ class RustGenerator : public BaseGenerator { RustGenerator(const Parser &parser, const std::string &path, const std::string &file_name) : BaseGenerator(parser, path, file_name, "", "::", "rs"), - cur_name_space_(nullptr) { - const char *keywords[] = { - // clang-format off - // list taken from: - // https://doc.rust-lang.org/book/second-edition/appendix-01-keywords.html - // - // we write keywords one per line so that we can easily compare them with - // changes to that webpage in the future. - - // currently-used keywords - "as", - "break", - "const", - "continue", - "crate", - "else", - "enum", - "extern", - "false", - "fn", - "for", - "if", - "impl", - "in", - "let", - "loop", - "match", - "mod", - "move", - "mut", - "pub", - "ref", - "return", - "Self", - "self", - "static", - "struct", - "super", - "trait", - "true", - "type", - "unsafe", - "use", - "where", - "while", - - // future possible keywords - "abstract", - "alignof", - "become", - "box", - "do", - "final", - "macro", - "offsetof", - "override", - "priv", - "proc", - "pure", - "sizeof", - "typeof", - "unsized", - "virtual", - "yield", - - // other rust terms we should not use - "std", - "usize", - "isize", - "u8", - "i8", - "u16", - "i16", - "u32", - "i32", - "u64", - "i64", - "u128", - "i128", - "f32", - "f64", - - // These are terms the code generator can implement on types. - // - // In Rust, the trait resolution rules (as described at - // https://github.com/rust-lang/rust/issues/26007) mean that, as long - // as we impl table accessors as inherent methods, we'll never create - // conflicts with these keywords. However, that's a fairly nuanced - // implementation detail, and how we implement methods could change in - // the future. as a result, we proactively block these out as reserved - // words. - "follow", - "push", - "size", - "alignment", - "to_little_endian", - "from_little_endian", - nullptr, - - // used by Enum constants - "ENUM_MAX", - "ENUM_MIN", - "ENUM_VALUES", - }; - for (auto kw = keywords; *kw; kw++) keywords_.insert(*kw); + cur_name_space_(nullptr), + namer_(WithFlagOptions(RustDefaultConfig(), parser.opts, path), + RustKeywords()) { + // TODO: Namer flag overrides should be in flatc or flatc_main. + code_.SetPadding(" "); + } + + bool generate() { + if (!parser_.opts.rust_module_root_file) { + return GenerateOneFile(); + } else { + return GenerateIndividualFiles(); + } + } + + template<typename T> + bool GenerateSymbols(const SymbolTable<T> &symbols, + std::function<void(const T &)> gen_symbol) { + for (auto it = symbols.vec.begin(); it != symbols.vec.end(); it++) { + const T &symbol = **it; + if (symbol.generated) continue; + code_.Clear(); + code_ += "// " + std::string(FlatBuffersGeneratedWarning()); + code_ += "// @generated"; + code_ += "extern crate alloc;"; + code_ += "extern crate flatbuffers;"; + code_ += "use alloc::boxed::Box;"; + code_ += "use alloc::string::{String, ToString};"; + code_ += "use alloc::vec::Vec;"; + code_ += "use core::mem;"; + code_ += "use core::cmp::Ordering;"; + if (parser_.opts.rust_serialize) { + code_ += "extern crate serde;"; + code_ += + "use self::serde::ser::{Serialize, Serializer, SerializeStruct};"; + } + code_ += "use self::flatbuffers::{EndianScalar, Follow};"; + code_ += "use super::*;"; + cur_name_space_ = symbol.defined_namespace; + gen_symbol(symbol); + + const std::string directories = + namer_.Directories(*symbol.defined_namespace); + EnsureDirExists(directories); + const std::string file_path = directories + namer_.File(symbol); + const bool save_success = + SaveFile(file_path.c_str(), code_.ToString(), /*binary=*/false); + if (!save_success) return false; + } + return true; + } + + bool GenerateIndividualFiles() { + code_.Clear(); + // Don't bother with imports. Use absolute paths everywhere. + return GenerateSymbols<EnumDef>( + parser_.enums_, [&](const EnumDef &e) { this->GenEnum(e); }) && + GenerateSymbols<StructDef>( + parser_.structs_, [&](const StructDef &s) { + if (s.fixed) { + this->GenStruct(s); + } else { + this->GenTable(s); + if (this->parser_.opts.generate_object_based_api) { + this->GenTableObject(s); + } + } + if (this->parser_.root_struct_def_ == &s) { + this->GenRootTableFuncs(s); + } + }); } + // Generates code organized by .fbs files. This is broken legacy behavior + // that does not work with multiple fbs files with shared namespaces. // Iterate through all definitions we haven't generated code for (enums, // structs, and tables) and output them to a single file. - bool generate() { + bool GenerateOneFile() { code_.Clear(); code_ += "// " + std::string(FlatBuffersGeneratedWarning()) + "\n\n"; + code_ += "// @generated"; assert(!cur_name_space_); @@ -399,8 +502,6 @@ class RustGenerator : public BaseGenerator { private: CodeWriter code_; - std::set<std::string> keywords_; - // This tracks the current namespace so we can insert namespace declarations. const Namespace *cur_name_space_; @@ -439,25 +540,16 @@ class RustGenerator : public BaseGenerator { return false; } - std::string EscapeKeyword(const std::string &name) const { - return keywords_.find(name) == keywords_.end() ? name : name + "_"; - } - std::string NamespacedNativeName(const Definition &def) { - return WrapInNameSpace(def.defined_namespace, NativeName(def)); - } - - std::string NativeName(const Definition &def) { - return parser_.opts.object_prefix + Name(def) + parser_.opts.object_suffix; + std::string NamespacedNativeName(const EnumDef &def) { + return WrapInNameSpace(def.defined_namespace, namer_.ObjectType(def)); } - - std::string Name(const Definition &def) const { - return EscapeKeyword(def.name); + std::string NamespacedNativeName(const StructDef &def) { + return WrapInNameSpace(def.defined_namespace, namer_.ObjectType(def)); } - std::string Name(const EnumVal &ev) const { return EscapeKeyword(ev.name); } - std::string WrapInNameSpace(const Definition &def) const { - return WrapInNameSpace(def.defined_namespace, Name(def)); + return WrapInNameSpace(def.defined_namespace, + namer_.EscapeKeyword(def.name)); } std::string WrapInNameSpace(const Namespace *ns, const std::string &name) const { @@ -466,19 +558,6 @@ class RustGenerator : public BaseGenerator { return prefix + name; } - // Determine the namespace traversal needed from the Rust crate root. - // This may be useful in the future for referring to included files, but is - // currently unused. - std::string GetAbsoluteNamespaceTraversal(const Namespace *dst) const { - std::stringstream stream; - - stream << "::"; - for (auto d = dst->components.begin(); d != dst->components.end(); ++d) { - stream << MakeSnakeCase(*d) + "::"; - } - return stream.str(); - } - // Determine the relative namespace traversal needed to reference one // namespace from another namespace. This is useful because it does not force // the user to have a particular file layout. (If we output absolute @@ -496,32 +575,25 @@ class RustGenerator : public BaseGenerator { // example: f(A, D::E) -> super::D::E // does not include leaf object (typically a struct type). - size_t i = 0; std::stringstream stream; - - auto s = src->components.begin(); - auto d = dst->components.begin(); - for (;;) { - if (s == src->components.end()) { break; } - if (d == dst->components.end()) { break; } - if (*s != *d) { break; } - ++s; - ++d; - ++i; - } - - for (; s != src->components.end(); ++s) { stream << "super::"; } - for (; d != dst->components.end(); ++d) { - stream << MakeSnakeCase(*d) + "::"; - } + size_t common = 0; + std::vector<std::string> s, d; + if (src) s = src->components; + if (dst) d = dst->components; + while (common < s.size() && common < d.size() && s[common] == d[common]) + common++; + // If src namespace is empty, this must be an absolute path. + for (size_t i = common; i < s.size(); i++) stream << "super::"; + for (size_t i = common; i < d.size(); i++) + stream << namer_.Namespace(d[i]) + "::"; return stream.str(); } // Generate a comment from the schema. void GenComment(const std::vector<std::string> &dc, const char *prefix = "") { - std::string text; - ::flatbuffers::GenComment(dc, &text, nullptr, prefix); - code_ += text + "\\"; + for (auto it = dc.begin(); it != dc.end(); it++) { + code_ += std::string(prefix) + "///" + *it; + } } // Return a Rust type from the table in idl.h. @@ -608,7 +680,7 @@ class RustGenerator : public BaseGenerator { std::string GetEnumValue(const EnumDef &enum_def, const EnumVal &enum_val) const { - return Name(enum_def) + "::" + Name(enum_val); + return namer_.EnumVariant(enum_def, enum_val); } // 1 suffix since old C++ can't figure out the overload. @@ -616,9 +688,11 @@ class RustGenerator : public BaseGenerator { std::function<void(const EnumVal &)> cb) { for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { const auto &ev = **it; - code_.SetValue("VARIANT", Name(ev)); + code_.SetValue("VARIANT", namer_.Variant(ev)); code_.SetValue("VALUE", enum_def.ToString(ev)); + code_.IncrementIdentLevel(); cb(ev); + code_.DecrementIdentLevel(); } } void ForAllEnumValues(const EnumDef &enum_def, std::function<void()> cb) { @@ -633,10 +707,13 @@ class RustGenerator : public BaseGenerator { // an enum match function, // and an enum array of values void GenEnum(const EnumDef &enum_def) { - code_.SetValue("ENUM_NAME", Name(enum_def)); + const bool is_private = parser_.opts.no_leak_private_annotations && + (enum_def.attributes.Lookup("private") != nullptr); + code_.SetValue("ACCESS_TYPE", is_private ? "pub(crate)" : "pub"); + code_.SetValue("ENUM_TY", namer_.Type(enum_def)); code_.SetValue("BASE_TYPE", GetEnumTypeForDecl(enum_def.underlying_type)); - code_.SetValue("ENUM_NAME_SNAKE", MakeSnakeCase(Name(enum_def))); - code_.SetValue("ENUM_NAME_CAPS", MakeUpper(MakeSnakeCase(Name(enum_def)))); + code_.SetValue("ENUM_NAMESPACE", namer_.Namespace(enum_def.name)); + code_.SetValue("ENUM_CONSTANT", namer_.Constant(enum_def.name)); const EnumVal *minv = enum_def.MinValue(); const EnumVal *maxv = enum_def.MaxValue(); FLATBUFFERS_ASSERT(minv && maxv); @@ -649,22 +726,21 @@ class RustGenerator : public BaseGenerator { // matches Flatbuffers c-modeled enums where variants are associated // constants but in camel case. code_ += "#[allow(non_upper_case_globals)]"; - code_ += "mod bitflags_{{ENUM_NAME_SNAKE}} {"; + code_ += "mod bitflags_{{ENUM_NAMESPACE}} {"; code_ += " flatbuffers::bitflags::bitflags! {"; GenComment(enum_def.doc_comment, " "); code_ += " #[derive(Default)]"; - code_ += " pub struct {{ENUM_NAME}}: {{BASE_TYPE}} {"; + code_ += " {{ACCESS_TYPE}} struct {{ENUM_TY}}: {{BASE_TYPE}} {"; ForAllEnumValues1(enum_def, [&](const EnumVal &ev) { - this->GenComment(ev.doc_comment, " "); - code_ += " const {{VARIANT}} = {{VALUE}};"; + this->GenComment(ev.doc_comment, " "); + code_ += " const {{VARIANT}} = {{VALUE}};"; }); code_ += " }"; code_ += " }"; code_ += "}"; - code_ += "pub use self::bitflags_{{ENUM_NAME_SNAKE}}::{{ENUM_NAME}};"; + code_ += "pub use self::bitflags_{{ENUM_NAMESPACE}}::{{ENUM_TY}};"; code_ += ""; - code_.SetValue("FROM_BASE", "unsafe { Self::from_bits_unchecked(b) }"); code_.SetValue("INTO_BASE", "self.bits()"); } else { // Normal, c-modelled enums. @@ -674,19 +750,19 @@ class RustGenerator : public BaseGenerator { " instead. This will no longer be generated in 2021.\")]"; code_ += deprecation_warning; code_ += - "pub const ENUM_MIN_{{ENUM_NAME_CAPS}}: {{BASE_TYPE}}" + "pub const ENUM_MIN_{{ENUM_CONSTANT}}: {{BASE_TYPE}}" " = {{ENUM_MIN_BASE_VALUE}};"; code_ += deprecation_warning; code_ += - "pub const ENUM_MAX_{{ENUM_NAME_CAPS}}: {{BASE_TYPE}}" + "pub const ENUM_MAX_{{ENUM_CONSTANT}}: {{BASE_TYPE}}" " = {{ENUM_MAX_BASE_VALUE}};"; auto num_fields = NumToString(enum_def.size()); code_ += deprecation_warning; code_ += "#[allow(non_camel_case_types)]"; - code_ += "pub const ENUM_VALUES_{{ENUM_NAME_CAPS}}: [{{ENUM_NAME}}; " + + code_ += "pub const ENUM_VALUES_{{ENUM_CONSTANT}}: [{{ENUM_TY}}; " + num_fields + "] = ["; ForAllEnumValues1(enum_def, [&](const EnumVal &ev) { - code_ += " " + GetEnumValue(enum_def, ev) + ","; + code_ += namer_.EnumVariant(enum_def, ev) + ","; }); code_ += "];"; code_ += ""; @@ -699,25 +775,25 @@ class RustGenerator : public BaseGenerator { "#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, " "Default)]"; code_ += "#[repr(transparent)]"; - code_ += "pub struct {{ENUM_NAME}}(pub {{BASE_TYPE}});"; + code_ += "{{ACCESS_TYPE}} struct {{ENUM_TY}}(pub {{BASE_TYPE}});"; code_ += "#[allow(non_upper_case_globals)]"; - code_ += "impl {{ENUM_NAME}} {"; + code_ += "impl {{ENUM_TY}} {"; ForAllEnumValues1(enum_def, [&](const EnumVal &ev) { - this->GenComment(ev.doc_comment, " "); - code_ += " pub const {{VARIANT}}: Self = Self({{VALUE}});"; + this->GenComment(ev.doc_comment); + code_ += "pub const {{VARIANT}}: Self = Self({{VALUE}});"; }); code_ += ""; // Generate Associated constants code_ += " pub const ENUM_MIN: {{BASE_TYPE}} = {{ENUM_MIN_BASE_VALUE}};"; code_ += " pub const ENUM_MAX: {{BASE_TYPE}} = {{ENUM_MAX_BASE_VALUE}};"; code_ += " pub const ENUM_VALUES: &'static [Self] = &["; - ForAllEnumValues(enum_def, [&]() { code_ += " Self::{{VARIANT}},"; }); + ForAllEnumValues(enum_def, [&]() { code_ += " Self::{{VARIANT}},"; }); code_ += " ];"; code_ += " /// Returns the variant's name or \"\" if unknown."; code_ += " pub fn variant_name(self) -> Option<&'static str> {"; code_ += " match self {"; ForAllEnumValues(enum_def, [&]() { - code_ += " Self::{{VARIANT}} => Some(\"{{VARIANT}}\"),"; + code_ += " Self::{{VARIANT}} => Some(\"{{VARIANT}}\"),"; }); code_ += " _ => None,"; code_ += " }"; @@ -725,10 +801,10 @@ class RustGenerator : public BaseGenerator { code_ += "}"; // Generate Debug. Unknown variants are printed like "<UNKNOWN 42>". - code_ += "impl std::fmt::Debug for {{ENUM_NAME}} {"; + code_ += "impl core::fmt::Debug for {{ENUM_TY}} {"; code_ += - " fn fmt(&self, f: &mut std::fmt::Formatter) ->" - " std::fmt::Result {"; + " fn fmt(&self, f: &mut core::fmt::Formatter) ->" + " core::fmt::Result {"; code_ += " if let Some(name) = self.variant_name() {"; code_ += " f.write_str(name)"; code_ += " } else {"; @@ -737,49 +813,89 @@ class RustGenerator : public BaseGenerator { code_ += " }"; code_ += "}"; - code_.SetValue("FROM_BASE", "Self(b)"); code_.SetValue("INTO_BASE", "self.0"); } + // Implement serde::Serialize + if (parser_.opts.rust_serialize) { + code_ += "impl Serialize for {{ENUM_TY}} {"; + code_ += + " fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>"; + code_ += " where"; + code_ += " S: Serializer,"; + code_ += " {"; + if (IsBitFlagsEnum(enum_def)) { + code_ += " serializer.serialize_u32(self.bits() as u32)"; + } else { + code_ += + " serializer.serialize_unit_variant(\"{{ENUM_TY}}\", self.0 " + "as " + "u32, self.variant_name().unwrap())"; + } + code_ += " }"; + code_ += "}"; + code_ += ""; + } + // Generate Follow and Push so we can serialize and stuff. - code_ += "impl<'a> flatbuffers::Follow<'a> for {{ENUM_NAME}} {"; + code_ += "impl<'a> flatbuffers::Follow<'a> for {{ENUM_TY}} {"; code_ += " type Inner = Self;"; code_ += " #[inline]"; - code_ += " fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {"; - code_ += " let b = unsafe {"; - code_ += " flatbuffers::read_scalar_at::<{{BASE_TYPE}}>(buf, loc)"; - code_ += " };"; - code_ += " {{FROM_BASE}}"; + code_ += " unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {"; + code_ += " let b = flatbuffers::read_scalar_at::<{{BASE_TYPE}}>(buf, loc);"; + if (IsBitFlagsEnum(enum_def)) { + // Safety: + // This is safe because we know bitflags is implemented with a repr transparent uint of the correct size. + // from_bits_unchecked will be replaced by an equivalent but safe from_bits_retain in bitflags 2.0 + // https://github.com/bitflags/bitflags/issues/262 + code_ += " // Safety:"; + code_ += " // This is safe because we know bitflags is implemented with a repr transparent uint of the correct size."; + code_ += " // from_bits_unchecked will be replaced by an equivalent but safe from_bits_retain in bitflags 2.0"; + code_ += " // https://github.com/bitflags/bitflags/issues/262"; + code_ += " Self::from_bits_unchecked(b)"; + } else { + code_ += " Self(b)"; + } code_ += " }"; code_ += "}"; code_ += ""; - code_ += "impl flatbuffers::Push for {{ENUM_NAME}} {"; - code_ += " type Output = {{ENUM_NAME}};"; + code_ += "impl flatbuffers::Push for {{ENUM_TY}} {"; + code_ += " type Output = {{ENUM_TY}};"; code_ += " #[inline]"; - code_ += " fn push(&self, dst: &mut [u8], _rest: &[u8]) {"; - code_ += - " unsafe { flatbuffers::emplace_scalar::<{{BASE_TYPE}}>" - "(dst, {{INTO_BASE}}); }"; + code_ += " unsafe fn push(&self, dst: &mut [u8], _written_len: usize) {"; + code_ += " flatbuffers::emplace_scalar::<{{BASE_TYPE}}>(dst, {{INTO_BASE}});"; code_ += " }"; code_ += "}"; code_ += ""; - code_ += "impl flatbuffers::EndianScalar for {{ENUM_NAME}} {"; + code_ += "impl flatbuffers::EndianScalar for {{ENUM_TY}} {"; + code_ += " type Scalar = {{BASE_TYPE}};"; code_ += " #[inline]"; - code_ += " fn to_little_endian(self) -> Self {"; - code_ += " let b = {{BASE_TYPE}}::to_le({{INTO_BASE}});"; - code_ += " {{FROM_BASE}}"; + code_ += " fn to_little_endian(self) -> {{BASE_TYPE}} {"; + code_ += " {{INTO_BASE}}.to_le()"; code_ += " }"; code_ += " #[inline]"; code_ += " #[allow(clippy::wrong_self_convention)]"; - code_ += " fn from_little_endian(self) -> Self {"; - code_ += " let b = {{BASE_TYPE}}::from_le({{INTO_BASE}});"; - code_ += " {{FROM_BASE}}"; + code_ += " fn from_little_endian(v: {{BASE_TYPE}}) -> Self {"; + code_ += " let b = {{BASE_TYPE}}::from_le(v);"; + if (IsBitFlagsEnum(enum_def)) { + // Safety: + // This is safe because we know bitflags is implemented with a repr transparent uint of the correct size. + // from_bits_unchecked will be replaced by an equivalent but safe from_bits_retain in bitflags 2.0 + // https://github.com/bitflags/bitflags/issues/262 + code_ += " // Safety:"; + code_ += " // This is safe because we know bitflags is implemented with a repr transparent uint of the correct size."; + code_ += " // from_bits_unchecked will be replaced by an equivalent but safe from_bits_retain in bitflags 2.0"; + code_ += " // https://github.com/bitflags/bitflags/issues/262"; + code_ += " unsafe { Self::from_bits_unchecked(b) }"; + } else { + code_ += " Self(b)"; + } code_ += " }"; code_ += "}"; code_ += ""; // Generate verifier - deferring to the base type. - code_ += "impl<'a> flatbuffers::Verifiable for {{ENUM_NAME}} {"; + code_ += "impl<'a> flatbuffers::Verifiable for {{ENUM_TY}} {"; code_ += " #[inline]"; code_ += " fn run_verifier("; code_ += " v: &mut flatbuffers::Verifier, pos: usize"; @@ -790,64 +906,70 @@ class RustGenerator : public BaseGenerator { code_ += "}"; code_ += ""; // Enums are basically integers. - code_ += "impl flatbuffers::SimpleToVerifyInSlice for {{ENUM_NAME}} {}"; + code_ += "impl flatbuffers::SimpleToVerifyInSlice for {{ENUM_TY}} {}"; if (enum_def.is_union) { // Generate typesafe offset(s) for unions - code_.SetValue("NAME", Name(enum_def)); - code_.SetValue("UNION_OFFSET_NAME", Name(enum_def) + "UnionTableOffset"); - code_ += "pub struct {{UNION_OFFSET_NAME}} {}"; + code_.SetValue("UNION_TYPE", namer_.Type(enum_def)); + code_ += "{{ACCESS_TYPE}} struct {{UNION_TYPE}}UnionTableOffset {}"; code_ += ""; if (parser_.opts.generate_object_based_api) { GenUnionObject(enum_def); } } } - // CASPER: dedup Object versions from non object versions. + // TODO(cneo): dedup Object versions from non object versions. void ForAllUnionObjectVariantsBesidesNone(const EnumDef &enum_def, std::function<void()> cb) { for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { auto &enum_val = **it; if (enum_val.union_type.base_type == BASE_TYPE_NONE) continue; - code_.SetValue("VARIANT_NAME", Name(enum_val)); - code_.SetValue("NATIVE_VARIANT", MakeCamel(Name(enum_val))); - code_.SetValue("U_ELEMENT_NAME", MakeSnakeCase(Name(enum_val))); + code_.SetValue("VARIANT_NAME", namer_.Variant(enum_val)); + // For legacy reasons, enum variants are Keep case while enum native + // variants are UpperCamel case. + code_.SetValue("NATIVE_VARIANT", + namer_.LegacyRustNativeVariant(enum_val)); + code_.SetValue("U_ELEMENT_NAME", namer_.Method(enum_val)); code_.SetValue("U_ELEMENT_TABLE_TYPE", NamespacedNativeName(*enum_val.union_type.struct_def)); + code_.IncrementIdentLevel(); cb(); + code_.DecrementIdentLevel(); } } void GenUnionObject(const EnumDef &enum_def) { - code_.SetValue("ENUM_NAME", Name(enum_def)); - code_.SetValue("ENUM_NAME_SNAKE", MakeSnakeCase(Name(enum_def))); - code_.SetValue("NATIVE_NAME", NativeName(enum_def)); + code_.SetValue("ENUM_TY", namer_.Type(enum_def)); + code_.SetValue("ENUM_FN", namer_.Function(enum_def)); + code_.SetValue("ENUM_OTY", namer_.ObjectType(enum_def)); // Generate native union. + code_ += "#[allow(clippy::upper_case_acronyms)]"; // NONE's spelling is + // intended. code_ += "#[non_exhaustive]"; code_ += "#[derive(Debug, Clone, PartialEq)]"; - code_ += "pub enum {{NATIVE_NAME}} {"; + code_ += "{{ACCESS_TYPE}} enum {{ENUM_OTY}} {"; code_ += " NONE,"; ForAllUnionObjectVariantsBesidesNone(enum_def, [&] { - code_ += " {{NATIVE_VARIANT}}(Box<{{U_ELEMENT_TABLE_TYPE}}>),"; + code_ += "{{NATIVE_VARIANT}}(Box<{{U_ELEMENT_TABLE_TYPE}}>),"; }); code_ += "}"; // Generate Default (NONE). - code_ += "impl Default for {{NATIVE_NAME}} {"; + code_ += "impl Default for {{ENUM_OTY}} {"; code_ += " fn default() -> Self {"; code_ += " Self::NONE"; code_ += " }"; code_ += "}"; // Generate native union methods. - code_ += "impl {{NATIVE_NAME}} {"; + code_ += "impl {{ENUM_OTY}} {"; // Get flatbuffers union key. - // CASPER: add docstrings? - code_ += " pub fn {{ENUM_NAME_SNAKE}}_type(&self) -> {{ENUM_NAME}} {"; + // TODO(cneo): add docstrings? + code_ += " pub fn {{ENUM_FN}}_type(&self) -> {{ENUM_TY}} {"; code_ += " match self {"; - code_ += " Self::NONE => {{ENUM_NAME}}::NONE,"; + code_ += " Self::NONE => {{ENUM_TY}}::NONE,"; ForAllUnionObjectVariantsBesidesNone(enum_def, [&] { code_ += - " Self::{{NATIVE_VARIANT}}(_) => {{ENUM_NAME}}::" + " Self::{{NATIVE_VARIANT}}(_) => {{ENUM_TY}}::" "{{VARIANT_NAME}},"; }); code_ += " }"; @@ -860,9 +982,8 @@ class RustGenerator : public BaseGenerator { code_ += " match self {"; code_ += " Self::NONE => None,"; ForAllUnionObjectVariantsBesidesNone(enum_def, [&] { - code_ += - " Self::{{NATIVE_VARIANT}}(v) => " - "Some(v.pack(fbb).as_union_value()),"; + code_ += " Self::{{NATIVE_VARIANT}}(v) => \\"; + code_ += "Some(v.pack(fbb).as_union_value()),"; }); code_ += " }"; code_ += " }"; @@ -871,52 +992,48 @@ class RustGenerator : public BaseGenerator { ForAllUnionObjectVariantsBesidesNone(enum_def, [&] { // Move accessor. code_ += - " /// If the union variant matches, return the owned " + "/// If the union variant matches, return the owned " "{{U_ELEMENT_TABLE_TYPE}}, setting the union to NONE."; code_ += - " pub fn take_{{U_ELEMENT_NAME}}(&mut self) -> " + "pub fn take_{{U_ELEMENT_NAME}}(&mut self) -> " "Option<Box<{{U_ELEMENT_TABLE_TYPE}}>> {"; - code_ += " if let Self::{{NATIVE_VARIANT}}(_) = self {"; - code_ += " let v = std::mem::replace(self, Self::NONE);"; - code_ += " if let Self::{{NATIVE_VARIANT}}(w) = v {"; - code_ += " Some(w)"; - code_ += " } else {"; - code_ += " unreachable!()"; - code_ += " }"; + code_ += " if let Self::{{NATIVE_VARIANT}}(_) = self {"; + code_ += " let v = core::mem::replace(self, Self::NONE);"; + code_ += " if let Self::{{NATIVE_VARIANT}}(w) = v {"; + code_ += " Some(w)"; code_ += " } else {"; - code_ += " None"; + code_ += " unreachable!()"; code_ += " }"; + code_ += " } else {"; + code_ += " None"; code_ += " }"; + code_ += "}"; // Immutable reference accessor. code_ += - " /// If the union variant matches, return a reference to the " + "/// If the union variant matches, return a reference to the " "{{U_ELEMENT_TABLE_TYPE}}."; code_ += - " pub fn as_{{U_ELEMENT_NAME}}(&self) -> " + "pub fn as_{{U_ELEMENT_NAME}}(&self) -> " "Option<&{{U_ELEMENT_TABLE_TYPE}}> {"; code_ += - " if let Self::{{NATIVE_VARIANT}}(v) = self " + " if let Self::{{NATIVE_VARIANT}}(v) = self " "{ Some(v.as_ref()) } else { None }"; - code_ += " }"; + code_ += "}"; // Mutable reference accessor. code_ += - " /// If the union variant matches, return a mutable reference" + "/// If the union variant matches, return a mutable reference" " to the {{U_ELEMENT_TABLE_TYPE}}."; code_ += - " pub fn as_{{U_ELEMENT_NAME}}_mut(&mut self) -> " + "pub fn as_{{U_ELEMENT_NAME}}_mut(&mut self) -> " "Option<&mut {{U_ELEMENT_TABLE_TYPE}}> {"; code_ += - " if let Self::{{NATIVE_VARIANT}}(v) = self " + " if let Self::{{NATIVE_VARIANT}}(v) = self " "{ Some(v.as_mut()) } else { None }"; - code_ += " }"; + code_ += "}"; }); code_ += "}"; // End union methods impl. } - std::string GetFieldOffsetName(const FieldDef &field) { - return "VT_" + MakeUpper(Name(field)); - } - enum DefaultContext { kBuilder, kAccessor, kObject }; std::string GetDefaultValue(const FieldDef &field, const DefaultContext context) { @@ -933,8 +1050,19 @@ class RustGenerator : public BaseGenerator { if (field.IsOptional() && !IsUnion(field.value.type)) { return "None"; } } switch (GetFullType(field.value.type)) { - case ftInteger: + case ftInteger: { + return field.value.constant; + } case ftFloat: { + const std::string float_prefix = + (field.value.type.base_type == BASE_TYPE_FLOAT) ? "f32::" : "f64::"; + if (StringIsFlatbufferNan(field.value.constant)) { + return float_prefix + "NAN"; + } else if (StringIsFlatbufferPositiveInfinity(field.value.constant)) { + return float_prefix + "INFINITY"; + } else if (StringIsFlatbufferNegativeInfinity(field.value.constant)) { + return float_prefix + "NEG_INFINITY"; + } return field.value.constant; } case ftBool: { @@ -944,8 +1072,9 @@ class RustGenerator : public BaseGenerator { case ftEnumKey: { auto ev = field.value.type.enum_def->FindByValue(field.value.constant); if (!ev) return "Default::default()"; // Bitflags enum. - return WrapInNameSpace(field.value.type.enum_def->defined_namespace, - GetEnumValue(*field.value.type.enum_def, *ev)); + return WrapInNameSpace( + field.value.type.enum_def->defined_namespace, + namer_.EnumVariant(*field.value.type.enum_def, *ev)); } case ftUnionValue: { return ObjectFieldType(field, true) + "::NONE"; @@ -960,7 +1089,7 @@ class RustGenerator : public BaseGenerator { field.IsRequired() ? "\"\"" : "\"" + field.value.constant + "\""; if (context == kObject) return defval + ".to_string()"; if (context == kAccessor) return "&" + defval; - FLATBUFFERS_ASSERT("Unreachable."); + FLATBUFFERS_ASSERT(false); return "INVALID_CODE_GENERATION"; } @@ -986,7 +1115,7 @@ class RustGenerator : public BaseGenerator { return "Default::default()"; } } - FLATBUFFERS_ASSERT("Unreachable."); + FLATBUFFERS_ASSERT(false); return "INVALID_CODE_GENERATION"; } @@ -1329,11 +1458,7 @@ class RustGenerator : public BaseGenerator { case ftVectorOfBool: case ftVectorOfFloat: { const auto typname = GetTypeBasic(type.VectorType()); - const auto vector_type = - IsOneByte(type.VectorType().base_type) - ? "&" + lifetime + " [" + typname + "]" - : "flatbuffers::Vector<" + lifetime + ", " + typname + ">"; - return WrapOption(vector_type); + return WrapOption("flatbuffers::Vector<" + lifetime + ", " + typname + ">"); } case ftVectorOfEnumKey: { const auto typname = WrapInNameSpace(*type.enum_def); @@ -1342,7 +1467,7 @@ class RustGenerator : public BaseGenerator { } case ftVectorOfStruct: { const auto typname = WrapInNameSpace(*type.struct_def); - return WrapOption("&" + lifetime + " [" + typname + "]"); + return WrapOption("flatbuffers::Vector<" + lifetime + ", " + typname + ">"); } case ftVectorOfTable: { const auto typname = WrapInNameSpace(*type.struct_def); @@ -1451,7 +1576,7 @@ class RustGenerator : public BaseGenerator { std::string GenTableAccessorFuncBody(const FieldDef &field, const std::string &lifetime) { - const std::string vt_offset = GetFieldOffsetName(field); + const std::string vt_offset = namer_.LegacyRustFieldOffsetName(field); const std::string typname = FollowType(field.value.type, lifetime); // Default-y fields (scalars so far) are neither optional nor required. const std::string default_value = @@ -1460,28 +1585,18 @@ class RustGenerator : public BaseGenerator { : "None"; const std::string unwrap = field.IsOptional() ? "" : ".unwrap()"; - const auto t = GetFullType(field.value.type); - - // TODO(caspern): Shouldn't 1byte VectorOfEnumKey be slice too? - const std::string safe_slice = - (t == ftVectorOfStruct || - ((t == ftVectorOfBool || t == ftVectorOfFloat || - t == ftVectorOfInteger) && - IsOneByte(field.value.type.VectorType().base_type))) - ? ".map(|v| v.safe_slice())" - : ""; - - return "self._tab.get::<" + typname + ">({{STRUCT_NAME}}::" + vt_offset + - ", " + default_value + ")" + safe_slice + unwrap; + return "unsafe { self._tab.get::<" + typname + ">({{STRUCT_TY}}::" + vt_offset + + ", " + default_value + ")" + unwrap + "}"; } // Generates a fully-qualified name getter for use with --gen-name-strings void GenFullyQualifiedNameGetter(const StructDef &struct_def, const std::string &name) { - code_ += " pub const fn get_fully_qualified_name() -> &'static str {"; - code_ += " \"" + - struct_def.defined_namespace->GetFullyQualifiedName(name) + "\""; - code_ += " }"; + const std::string fully_qualified_name = + struct_def.defined_namespace->GetFullyQualifiedName(name); + code_ += " pub const fn get_fully_qualified_name() -> &'static str {"; + code_ += " \"" + fully_qualified_name + "\""; + code_ += " }"; code_ += ""; } @@ -1495,12 +1610,12 @@ class RustGenerator : public BaseGenerator { if (ev.union_type.base_type == BASE_TYPE_NONE) { continue; } code_.SetValue( "U_ELEMENT_ENUM_TYPE", - WrapInNameSpace(def.defined_namespace, GetEnumValue(def, ev))); + WrapInNameSpace(def.defined_namespace, namer_.EnumVariant(def, ev))); code_.SetValue( "U_ELEMENT_TABLE_TYPE", WrapInNameSpace(ev.union_type.struct_def->defined_namespace, ev.union_type.struct_def->name)); - code_.SetValue("U_ELEMENT_NAME", MakeSnakeCase(Name(ev))); + code_.SetValue("U_ELEMENT_NAME", namer_.Function(ev.name)); cb(ev); } } @@ -1512,11 +1627,14 @@ class RustGenerator : public BaseGenerator { // diff when refactoring to the `ForAllX` helper functions. auto go = [&](const FieldDef &field) { if (field.deprecated) return; - code_.SetValue("OFFSET_NAME", GetFieldOffsetName(field)); + code_.SetValue("OFFSET_NAME", namer_.LegacyRustFieldOffsetName(field)); code_.SetValue("OFFSET_VALUE", NumToString(field.value.offset)); - code_.SetValue("FIELD_NAME", Name(field)); + code_.SetValue("FIELD", namer_.Field(field)); code_.SetValue("BLDR_DEF_VAL", GetDefaultValue(field, kBuilder)); + code_.SetValue("DISCRIMINANT", namer_.LegacyRustUnionTypeMethod(field)); + code_.IncrementIdentLevel(); cb(field); + code_.DecrementIdentLevel(); }; const auto &fields = struct_def.fields.vec; if (reversed) { @@ -1528,58 +1646,67 @@ class RustGenerator : public BaseGenerator { // Generate an accessor struct, builder struct, and create function for a // table. void GenTable(const StructDef &struct_def) { - code_.SetValue("STRUCT_NAME", Name(struct_def)); - code_.SetValue("OFFSET_TYPELABEL", Name(struct_def) + "Offset"); - code_.SetValue("STRUCT_NAME_SNAKECASE", MakeSnakeCase(Name(struct_def))); + + const bool is_private = parser_.opts.no_leak_private_annotations && + (struct_def.attributes.Lookup("private") != nullptr); + code_.SetValue("ACCESS_TYPE", is_private ? "pub(crate)" : "pub"); + code_.SetValue("STRUCT_TY", namer_.Type(struct_def)); + code_.SetValue("STRUCT_FN", namer_.Function(struct_def)); // Generate an offset type, the base type, the Follow impl, and the // init_from_table impl. - code_ += "pub enum {{OFFSET_TYPELABEL}} {}"; + code_ += "{{ACCESS_TYPE}} enum {{STRUCT_TY}}Offset {}"; code_ += "#[derive(Copy, Clone, PartialEq)]"; code_ += ""; GenComment(struct_def.doc_comment); - code_ += "pub struct {{STRUCT_NAME}}<'a> {"; + code_ += "{{ACCESS_TYPE}} struct {{STRUCT_TY}}<'a> {"; code_ += " pub _tab: flatbuffers::Table<'a>,"; code_ += "}"; code_ += ""; - code_ += "impl<'a> flatbuffers::Follow<'a> for {{STRUCT_NAME}}<'a> {"; - code_ += " type Inner = {{STRUCT_NAME}}<'a>;"; - code_ += " #[inline]"; - code_ += " fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {"; - code_ += " Self { _tab: flatbuffers::Table { buf, loc } }"; - code_ += " }"; + code_ += "impl<'a> flatbuffers::Follow<'a> for {{STRUCT_TY}}<'a> {"; + code_ += " type Inner = {{STRUCT_TY}}<'a>;"; + code_ += " #[inline]"; + code_ += " unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {"; + code_ += " Self { _tab: flatbuffers::Table::new(buf, loc) }"; + code_ += " }"; code_ += "}"; code_ += ""; - code_ += "impl<'a> {{STRUCT_NAME}}<'a> {"; + code_ += "impl<'a> {{STRUCT_TY}}<'a> {"; + + // Generate field id constants. + ForAllTableFields(struct_def, [&](const FieldDef &unused) { + (void)unused; + code_ += + "pub const {{OFFSET_NAME}}: flatbuffers::VOffsetT = " + "{{OFFSET_VALUE}};"; + }); + code_ += ""; if (parser_.opts.generate_name_strings) { GenFullyQualifiedNameGetter(struct_def, struct_def.name); } - code_ += " #[inline]"; + code_ += " #[inline]"; code_ += - " pub fn init_from_table(table: flatbuffers::Table<'a>) -> " + " pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> " "Self {"; - code_ += " {{STRUCT_NAME}} { _tab: table }"; - code_ += " }"; + code_ += " {{STRUCT_TY}} { _tab: table }"; + code_ += " }"; // Generate a convenient create* function that uses the above builder // to create a table in one function call. code_.SetValue("MAYBE_US", struct_def.fields.vec.size() == 0 ? "_" : ""); code_.SetValue("MAYBE_LT", TableBuilderArgsNeedsLifetime(struct_def) ? "<'args>" : ""); - code_ += " #[allow(unused_mut)]"; - code_ += " pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>("; - code_ += - " _fbb: " - "&'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,"; - code_ += - " {{MAYBE_US}}args: &'args {{STRUCT_NAME}}Args{{MAYBE_LT}})" - " -> flatbuffers::WIPOffset<{{STRUCT_NAME}}<'bldr>> {"; + code_ += " #[allow(unused_mut)]"; + code_ += " pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>("; + code_ += " _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>,"; + code_ += " {{MAYBE_US}}args: &'args {{STRUCT_TY}}Args{{MAYBE_LT}}"; + code_ += " ) -> flatbuffers::WIPOffset<{{STRUCT_TY}}<'bldr>> {"; - code_ += " let mut builder = {{STRUCT_NAME}}Builder::new(_fbb);"; + code_ += " let mut builder = {{STRUCT_TY}}Builder::new(_fbb);"; for (size_t size = struct_def.sortbysize ? sizeof(largest_scalar_t) : 1; size; size /= 2) { ForAllTableFields( @@ -1590,23 +1717,23 @@ class RustGenerator : public BaseGenerator { return; if (IsOptionalToBuilder(field)) { code_ += - " if let Some(x) = args.{{FIELD_NAME}} " - "{ builder.add_{{FIELD_NAME}}(x); }"; + " if let Some(x) = args.{{FIELD}} " + "{ builder.add_{{FIELD}}(x); }"; } else { - code_ += " builder.add_{{FIELD_NAME}}(args.{{FIELD_NAME}});"; + code_ += " builder.add_{{FIELD}}(args.{{FIELD}});"; } }, /*reverse=*/true); } - code_ += " builder.finish()"; - code_ += " }"; + code_ += " builder.finish()"; + code_ += " }"; code_ += ""; // Generate Object API Packer function. if (parser_.opts.generate_object_based_api) { // TODO(cneo): Replace more for loops with ForAllX stuff. // TODO(cneo): Manage indentation with IncrementIdentLevel? - code_.SetValue("OBJECT_NAME", NativeName(struct_def)); - code_ += " pub fn unpack(&self) -> {{OBJECT_NAME}} {"; + code_.SetValue("STRUCT_OTY", namer_.ObjectType(struct_def)); + code_ += " pub fn unpack(&self) -> {{STRUCT_OTY}} {"; ForAllObjectTableFields(struct_def, [&](const FieldDef &field) { const Type &type = field.value.type; switch (GetFullType(type)) { @@ -1614,36 +1741,33 @@ class RustGenerator : public BaseGenerator { case ftBool: case ftFloat: case ftEnumKey: { - code_ += " let {{FIELD_NAME}} = self.{{FIELD_NAME}}();"; + code_ += " let {{FIELD}} = self.{{FIELD}}();"; return; } case ftUnionKey: return; case ftUnionValue: { const auto &enum_def = *type.enum_def; - code_.SetValue("ENUM_NAME", WrapInNameSpace(enum_def)); + code_.SetValue("ENUM_TY", WrapInNameSpace(enum_def)); code_.SetValue("NATIVE_ENUM_NAME", NamespacedNativeName(enum_def)); - code_ += - " let {{FIELD_NAME}} = match " - "self.{{FIELD_NAME}}_type() {"; - code_ += - " {{ENUM_NAME}}::NONE =>" - " {{NATIVE_ENUM_NAME}}::NONE,"; + code_.SetValue("UNION_TYPE_METHOD", + namer_.LegacyRustUnionTypeMethod(field)); + + code_ += " let {{FIELD}} = match self.{{UNION_TYPE_METHOD}}() {"; + code_ += " {{ENUM_TY}}::NONE => {{NATIVE_ENUM_NAME}}::NONE,"; ForAllUnionObjectVariantsBesidesNone(enum_def, [&] { code_ += - " {{ENUM_NAME}}::{{VARIANT_NAME}} => " + " {{ENUM_TY}}::{{VARIANT_NAME}} => " "{{NATIVE_ENUM_NAME}}::{{NATIVE_VARIANT}}(Box::new("; + code_ += " self.{{FIELD}}_as_{{U_ELEMENT_NAME}}()"; code_ += - " self.{{FIELD_NAME}}_as_" - "{{U_ELEMENT_NAME}}()"; - code_ += - " .expect(\"Invalid union table, " - "expected `{{ENUM_NAME}}::{{VARIANT_NAME}}`.\")"; - code_ += " .unpack()"; - code_ += " )),"; + " .expect(\"Invalid union table, " + "expected `{{ENUM_TY}}::{{VARIANT_NAME}}`.\")"; + code_ += " .unpack()"; + code_ += " )),"; }); // Maybe we shouldn't throw away unknown discriminants? - code_ += " _ => {{NATIVE_ENUM_NAME}}::NONE,"; - code_ += " };"; + code_ += " _ => {{NATIVE_ENUM_NAME}}::NONE,"; + code_ += " };"; return; } // The rest of the types need special handling based on if the field @@ -1661,16 +1785,7 @@ class RustGenerator : public BaseGenerator { break; } case ftVectorOfInteger: - case ftVectorOfBool: { - if (IsOneByte(type.VectorType().base_type)) { - // 1 byte stuff is viewed w/ slice instead of flatbuffer::Vector - // and thus needs to be cloned out of the slice. - code_.SetValue("EXPR", "x.to_vec()"); - break; - } - code_.SetValue("EXPR", "x.into_iter().collect()"); - break; - } + case ftVectorOfBool: case ftVectorOfFloat: case ftVectorOfEnumKey: { code_.SetValue("EXPR", "x.into_iter().collect()"); @@ -1698,32 +1813,25 @@ class RustGenerator : public BaseGenerator { } } if (field.IsOptional()) { - code_ += " let {{FIELD_NAME}} = self.{{FIELD_NAME}}().map(|x| {"; - code_ += " {{EXPR}}"; - code_ += " });"; + code_ += " let {{FIELD}} = self.{{FIELD}}().map(|x| {"; + code_ += " {{EXPR}}"; + code_ += " });"; } else { - code_ += " let {{FIELD_NAME}} = {"; - code_ += " let x = self.{{FIELD_NAME}}();"; - code_ += " {{EXPR}}"; - code_ += " };"; + code_ += " let {{FIELD}} = {"; + code_ += " let x = self.{{FIELD}}();"; + code_ += " {{EXPR}}"; + code_ += " };"; } }); - code_ += " {{OBJECT_NAME}} {"; + code_ += " {{STRUCT_OTY}} {"; ForAllObjectTableFields(struct_def, [&](const FieldDef &field) { if (field.value.type.base_type == BASE_TYPE_UTYPE) return; - code_ += " {{FIELD_NAME}},"; + code_ += " {{FIELD}},"; }); - code_ += " }"; code_ += " }"; + code_ += " }"; } - // Generate field id constants. - ForAllTableFields(struct_def, [&](const FieldDef &unused) { - (void)unused; - code_ += - " pub const {{OFFSET_NAME}}: flatbuffers::VOffsetT = " - "{{OFFSET_VALUE}};"; - }); if (struct_def.fields.vec.size() > 0) code_ += ""; // Generate the accessors. Each has one of two forms: @@ -1741,11 +1849,14 @@ class RustGenerator : public BaseGenerator { code_.SetValue("RETURN_TYPE", GenTableAccessorFuncReturnType(field, "'a")); - this->GenComment(field.doc_comment, " "); - code_ += " #[inline]"; - code_ += " pub fn {{FIELD_NAME}}(&self) -> {{RETURN_TYPE}} {"; - code_ += " " + GenTableAccessorFuncBody(field, "'a"); - code_ += " }"; + this->GenComment(field.doc_comment); + code_ += "#[inline]"; + code_ += "pub fn {{FIELD}}(&self) -> {{RETURN_TYPE}} {"; + code_ += " // Safety:"; + code_ += " // Created from valid Table for this object"; + code_ += " // which contains a valid value in this slot"; + code_ += " " + GenTableAccessorFuncBody(field, "'a"); + code_ += "}"; // Generate a comparison function for this field if it is a key. if (field.key) { GenKeyFieldMethods(field); } @@ -1763,38 +1874,43 @@ class RustGenerator : public BaseGenerator { FLATBUFFERS_ASSERT(nested_root); // Guaranteed to exist by parser. code_.SetValue("NESTED", WrapInNameSpace(*nested_root)); - code_ += " pub fn {{FIELD_NAME}}_nested_flatbuffer(&'a self) -> \\"; + code_ += "pub fn {{FIELD}}_nested_flatbuffer(&'a self) -> \\"; if (field.IsRequired()) { code_ += "{{NESTED}}<'a> {"; - code_ += " let data = self.{{FIELD_NAME}}();"; - code_ += " use flatbuffers::Follow;"; + code_ += " let data = self.{{FIELD}}();"; + code_ += " use flatbuffers::Follow;"; + code_ += " // Safety:"; + code_ += " // Created from a valid Table for this object"; + code_ += " // Which contains a valid flatbuffer in this slot"; code_ += - " <flatbuffers::ForwardsUOffset<{{NESTED}}<'a>>>" - "::follow(data, 0)"; + " unsafe { <flatbuffers::ForwardsUOffset<{{NESTED}}<'a>>>" + "::follow(data.bytes(), 0) }"; } else { code_ += "Option<{{NESTED}}<'a>> {"; - code_ += " self.{{FIELD_NAME}}().map(|data| {"; - code_ += " use flatbuffers::Follow;"; + code_ += " self.{{FIELD}}().map(|data| {"; + code_ += " use flatbuffers::Follow;"; + code_ += " // Safety:"; + code_ += " // Created from a valid Table for this object"; + code_ += " // Which contains a valid flatbuffer in this slot"; code_ += - " <flatbuffers::ForwardsUOffset<{{NESTED}}<'a>>>" - "::follow(data, 0)"; - code_ += " })"; + " unsafe { <flatbuffers::ForwardsUOffset<{{NESTED}}<'a>>>" + "::follow(data.bytes(), 0) }"; + code_ += " })"; } - code_ += " }"; + code_ += "}"; } }); // Explicit specializations for union accessors ForAllTableFields(struct_def, [&](const FieldDef &field) { if (field.value.type.base_type != BASE_TYPE_UNION) return; - code_.SetValue("FIELD_TYPE_FIELD_NAME", field.name); ForAllUnionVariantsBesidesNone( *field.value.type.enum_def, [&](const EnumVal &unused) { (void)unused; - code_ += " #[inline]"; - code_ += " #[allow(non_snake_case)]"; + code_ += "#[inline]"; + code_ += "#[allow(non_snake_case)]"; code_ += - " pub fn {{FIELD_NAME}}_as_{{U_ELEMENT_NAME}}(&self) -> " + "pub fn {{FIELD}}_as_{{U_ELEMENT_NAME}}(&self) -> " "Option<{{U_ELEMENT_TABLE_TYPE}}<'a>> {"; // If the user defined schemas name a field that clashes with a // language reserved word, flatc will try to escape the field name @@ -1808,24 +1924,28 @@ class RustGenerator : public BaseGenerator { // // To avoid this problem the type field name is used unescaped here: code_ += - " if self.{{FIELD_TYPE_FIELD_NAME}}_type() == " - "{{U_ELEMENT_ENUM_TYPE}} {"; + " if self.{{DISCRIMINANT}}() == {{U_ELEMENT_ENUM_TYPE}} {"; // The following logic is not tested in the integration test, // as of April 10, 2020 if (field.IsRequired()) { - code_ += " let u = self.{{FIELD_NAME}}();"; - code_ += - " Some({{U_ELEMENT_TABLE_TYPE}}::init_from_table(u))"; + code_ += " let u = self.{{FIELD}}();"; + code_ += " // Safety:"; + code_ += " // Created from a valid Table for this object"; + code_ += " // Which contains a valid union in this slot"; + code_ += " Some(unsafe { {{U_ELEMENT_TABLE_TYPE}}::init_from_table(u) })"; } else { - code_ += - " self.{{FIELD_NAME}}().map(" - "{{U_ELEMENT_TABLE_TYPE}}::init_from_table)"; + code_ +=" self.{{FIELD}}().map(|t| {"; + code_ += " // Safety:"; + code_ += " // Created from a valid Table for this object"; + code_ += " // Which contains a valid union in this slot"; + code_ += " unsafe { {{U_ELEMENT_TABLE_TYPE}}::init_from_table(t) }"; + code_ += " })"; } - code_ += " } else {"; - code_ += " None"; - code_ += " }"; + code_ += " } else {"; + code_ += " None"; code_ += " }"; + code_ += "}"; code_ += ""; }); }); @@ -1833,7 +1953,7 @@ class RustGenerator : public BaseGenerator { code_ += ""; // Generate Verifier; - code_ += "impl flatbuffers::Verifiable for {{STRUCT_NAME}}<'_> {"; + code_ += "impl flatbuffers::Verifiable for {{STRUCT_TY}}<'_> {"; code_ += " #[inline]"; code_ += " fn run_verifier("; code_ += " v: &mut flatbuffers::Verifier, pos: usize"; @@ -1850,29 +1970,33 @@ class RustGenerator : public BaseGenerator { // All types besides unions. code_.SetValue("TY", FollowType(field.value.type, "'_")); code_ += - "\n .visit_field::<{{TY}}>(&\"{{FIELD_NAME}}\", " + "\n .visit_field::<{{TY}}>(\"{{FIELD}}\", " "Self::{{OFFSET_NAME}}, {{IS_REQ}})?\\"; return; } // Unions. - EnumDef &union_def = *field.value.type.enum_def; + const EnumDef &union_def = *field.value.type.enum_def; code_.SetValue("UNION_TYPE", WrapInNameSpace(union_def)); + code_.SetValue("UNION_TYPE_OFFSET_NAME", + namer_.LegacyRustUnionTypeOffsetName(field)); + code_.SetValue("UNION_TYPE_METHOD", + namer_.LegacyRustUnionTypeMethod(field)); code_ += "\n .visit_union::<{{UNION_TYPE}}, _>(" - "&\"{{FIELD_NAME}}_type\", Self::{{OFFSET_NAME}}_TYPE, " - "&\"{{FIELD_NAME}}\", Self::{{OFFSET_NAME}}, {{IS_REQ}}, " + "\"{{UNION_TYPE_METHOD}}\", Self::{{UNION_TYPE_OFFSET_NAME}}, " + "\"{{FIELD}}\", Self::{{OFFSET_NAME}}, {{IS_REQ}}, " "|key, v, pos| {"; - code_ += " match key {"; + code_ += " match key {"; ForAllUnionVariantsBesidesNone(union_def, [&](const EnumVal &unused) { (void)unused; code_ += - " {{U_ELEMENT_ENUM_TYPE}} => v.verify_union_variant::" + " {{U_ELEMENT_ENUM_TYPE}} => v.verify_union_variant::" "<flatbuffers::ForwardsUOffset<{{U_ELEMENT_TABLE_TYPE}}>>(" "\"{{U_ELEMENT_ENUM_TYPE}}\", pos),"; }); - code_ += " _ => Ok(()),"; - code_ += " }"; - code_ += " })?\\"; + code_ += " _ => Ok(()),"; + code_ += " }"; + code_ += " })?\\"; }); code_ += "\n .finish();"; code_ += " Ok(())"; @@ -1882,28 +2006,98 @@ class RustGenerator : public BaseGenerator { // Generate an args struct: code_.SetValue("MAYBE_LT", TableBuilderArgsNeedsLifetime(struct_def) ? "<'a>" : ""); - code_ += "pub struct {{STRUCT_NAME}}Args{{MAYBE_LT}} {"; + code_ += "{{ACCESS_TYPE}} struct {{STRUCT_TY}}Args{{MAYBE_LT}} {"; ForAllTableFields(struct_def, [&](const FieldDef &field) { code_.SetValue("PARAM_TYPE", TableBuilderArgsDefnType(field, "'a")); - code_ += " pub {{FIELD_NAME}}: {{PARAM_TYPE}},"; + code_ += " pub {{FIELD}}: {{PARAM_TYPE}},"; }); code_ += "}"; // Generate an impl of Default for the *Args type: - code_ += "impl<'a> Default for {{STRUCT_NAME}}Args{{MAYBE_LT}} {"; - code_ += " #[inline]"; - code_ += " fn default() -> Self {"; - code_ += " {{STRUCT_NAME}}Args {"; + code_ += "impl<'a> Default for {{STRUCT_TY}}Args{{MAYBE_LT}} {"; + code_ += " #[inline]"; + code_ += " fn default() -> Self {"; + code_ += " {{STRUCT_TY}}Args {"; ForAllTableFields(struct_def, [&](const FieldDef &field) { - code_ += " {{FIELD_NAME}}: {{BLDR_DEF_VAL}},\\"; + code_ += " {{FIELD}}: {{BLDR_DEF_VAL}},\\"; code_ += field.IsRequired() ? " // required field" : ""; }); - code_ += " }"; code_ += " }"; + code_ += " }"; code_ += "}"; + code_ += ""; + + // Implement serde::Serialize + if (parser_.opts.rust_serialize) { + const auto numFields = struct_def.fields.vec.size(); + code_.SetValue("NUM_FIELDS", NumToString(numFields)); + code_ += "impl Serialize for {{STRUCT_TY}}<'_> {"; + code_ += + " fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>"; + code_ += " where"; + code_ += " S: Serializer,"; + code_ += " {"; + if (numFields == 0) { + code_ += + " let s = serializer.serialize_struct(\"{{STRUCT_TY}}\", 0)?;"; + } else { + code_ += + " let mut s = serializer.serialize_struct(\"{{STRUCT_TY}}\", " + "{{NUM_FIELDS}})?;"; + } + ForAllTableFields(struct_def, [&](const FieldDef &field) { + const Type &type = field.value.type; + if (IsUnion(type)) { + if (type.base_type == BASE_TYPE_UNION) { + const auto &enum_def = *type.enum_def; + code_.SetValue("ENUM_TY", WrapInNameSpace(enum_def)); + code_.SetValue("FIELD", namer_.Field(field)); + code_.SetValue("UNION_TYPE_METHOD", + namer_.LegacyRustUnionTypeMethod(field)); + + code_ += " match self.{{UNION_TYPE_METHOD}}() {"; + code_ += " {{ENUM_TY}}::NONE => (),"; + ForAllUnionObjectVariantsBesidesNone(enum_def, [&] { + code_.SetValue("FIELD", namer_.Field(field)); + code_ += " {{ENUM_TY}}::{{VARIANT_NAME}} => {"; + code_ += + " let f = " + "self.{{FIELD}}_as_{{U_ELEMENT_NAME}}()"; + code_ += + " .expect(\"Invalid union table, expected " + "`{{ENUM_TY}}::{{VARIANT_NAME}}`.\");"; + code_ += " s.serialize_field(\"{{FIELD}}\", &f)?;"; + code_ += " }"; + }); + code_ += " _ => unimplemented!(),"; + code_ += " }"; + } else { + code_ += + " s.serialize_field(\"{{FIELD}}\", " + "&self.{{FIELD}}())?;"; + } + } else { + if (field.IsOptional()) { + code_ += " if let Some(f) = self.{{FIELD}}() {"; + code_ += " s.serialize_field(\"{{FIELD}}\", &f)?;"; + code_ += " } else {"; + code_ += " s.skip_field(\"{{FIELD}}\")?;"; + code_ += " }"; + } else { + code_ += + " s.serialize_field(\"{{FIELD}}\", " + "&self.{{FIELD}}())?;"; + } + } + }); + code_ += " s.end()"; + code_ += " }"; + code_ += "}"; + code_ += ""; + } // Generate a builder struct: - code_ += "pub struct {{STRUCT_NAME}}Builder<'a: 'b, 'b> {"; + code_ += "{{ACCESS_TYPE}} struct {{STRUCT_TY}}Builder<'a: 'b, 'b> {"; code_ += " fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>,"; code_ += " start_: flatbuffers::WIPOffset<" @@ -1911,10 +2105,10 @@ class RustGenerator : public BaseGenerator { code_ += "}"; // Generate builder functions: - code_ += "impl<'a: 'b, 'b> {{STRUCT_NAME}}Builder<'a, 'b> {"; + code_ += "impl<'a: 'b, 'b> {{STRUCT_TY}}Builder<'a, 'b> {"; ForAllTableFields(struct_def, [&](const FieldDef &field) { const bool is_scalar = IsScalar(field.value.type.base_type); - std::string offset = GetFieldOffsetName(field); + std::string offset = namer_.LegacyRustFieldOffsetName(field); // Generate functions to add data, which take one of two forms. // // If a value has a default: @@ -1926,31 +2120,31 @@ class RustGenerator : public BaseGenerator { // fn add_x(x_: type) { // fbb_.push_slot_always::<type>(offset, x_); // } - code_.SetValue("FIELD_OFFSET", Name(struct_def) + "::" + offset); + code_.SetValue("FIELD_OFFSET", namer_.Type(struct_def) + "::" + offset); code_.SetValue("FIELD_TYPE", TableBuilderArgsAddFuncType(field, "'b ")); code_.SetValue("FUNC_BODY", TableBuilderArgsAddFuncBody(field)); - code_ += " #[inline]"; + code_ += "#[inline]"; code_ += - " pub fn add_{{FIELD_NAME}}(&mut self, {{FIELD_NAME}}: " + "pub fn add_{{FIELD}}(&mut self, {{FIELD}}: " "{{FIELD_TYPE}}) {"; if (is_scalar && !field.IsOptional()) { code_ += - " {{FUNC_BODY}}({{FIELD_OFFSET}}, {{FIELD_NAME}}, " + " {{FUNC_BODY}}({{FIELD_OFFSET}}, {{FIELD}}, " "{{BLDR_DEF_VAL}});"; } else { - code_ += " {{FUNC_BODY}}({{FIELD_OFFSET}}, {{FIELD_NAME}});"; + code_ += " {{FUNC_BODY}}({{FIELD_OFFSET}}, {{FIELD}});"; } - code_ += " }"; + code_ += "}"; }); // Struct initializer (all fields required); code_ += " #[inline]"; code_ += " pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> " - "{{STRUCT_NAME}}Builder<'a, 'b> {"; + "{{STRUCT_TY}}Builder<'a, 'b> {"; code_.SetValue("NUM_FIELDS", NumToString(struct_def.fields.vec.size())); code_ += " let start = _fbb.start_table();"; - code_ += " {{STRUCT_NAME}}Builder {"; + code_ += " {{STRUCT_TY}}Builder {"; code_ += " fbb_: _fbb,"; code_ += " start_: start,"; code_ += " }"; @@ -1960,58 +2154,56 @@ class RustGenerator : public BaseGenerator { code_ += " #[inline]"; code_ += " pub fn finish(self) -> " - "flatbuffers::WIPOffset<{{STRUCT_NAME}}<'a>> {"; + "flatbuffers::WIPOffset<{{STRUCT_TY}}<'a>> {"; code_ += " let o = self.fbb_.end_table(self.start_);"; ForAllTableFields(struct_def, [&](const FieldDef &field) { if (!field.IsRequired()) return; code_ += - " self.fbb_.required(o, {{STRUCT_NAME}}::{{OFFSET_NAME}}," - "\"{{FIELD_NAME}}\");"; + " self.fbb_.required(o, {{STRUCT_TY}}::{{OFFSET_NAME}}," + "\"{{FIELD}}\");"; }); code_ += " flatbuffers::WIPOffset::new(o.value())"; code_ += " }"; code_ += "}"; code_ += ""; - code_ += "impl std::fmt::Debug for {{STRUCT_NAME}}<'_> {"; + code_ += "impl core::fmt::Debug for {{STRUCT_TY}}<'_> {"; code_ += - " fn fmt(&self, f: &mut std::fmt::Formatter<'_>" - ") -> std::fmt::Result {"; - code_ += " let mut ds = f.debug_struct(\"{{STRUCT_NAME}}\");"; + " fn fmt(&self, f: &mut core::fmt::Formatter<'_>" + ") -> core::fmt::Result {"; + code_ += " let mut ds = f.debug_struct(\"{{STRUCT_TY}}\");"; ForAllTableFields(struct_def, [&](const FieldDef &field) { if (GetFullType(field.value.type) == ftUnionValue) { // Generate a match statement to handle unions properly. code_.SetValue("KEY_TYPE", GenTableAccessorFuncReturnType(field, "")); - code_.SetValue("FIELD_TYPE_FIELD_NAME", field.name); code_.SetValue("UNION_ERR", "&\"InvalidFlatbuffer: Union discriminant" " does not match value.\""); - code_ += " match self.{{FIELD_NAME}}_type() {"; + code_ += " match self.{{DISCRIMINANT}}() {"; ForAllUnionVariantsBesidesNone( *field.value.type.enum_def, [&](const EnumVal &unused) { (void)unused; - code_ += " {{U_ELEMENT_ENUM_TYPE}} => {"; + code_ += " {{U_ELEMENT_ENUM_TYPE}} => {"; code_ += - " if let Some(x) = " - "self.{{FIELD_TYPE_FIELD_NAME}}_as_" + " if let Some(x) = " + "self.{{FIELD}}_as_" "{{U_ELEMENT_NAME}}() {"; - code_ += " ds.field(\"{{FIELD_NAME}}\", &x)"; - code_ += " } else {"; - code_ += - " ds.field(\"{{FIELD_NAME}}\", {{UNION_ERR}})"; - code_ += " }"; - code_ += " },"; + code_ += " ds.field(\"{{FIELD}}\", &x)"; + code_ += " } else {"; + code_ += " ds.field(\"{{FIELD}}\", {{UNION_ERR}})"; + code_ += " }"; + code_ += " },"; }); - code_ += " _ => {"; - code_ += " let x: Option<()> = None;"; - code_ += " ds.field(\"{{FIELD_NAME}}\", &x)"; - code_ += " },"; - code_ += " };"; + code_ += " _ => {"; + code_ += " let x: Option<()> = None;"; + code_ += " ds.field(\"{{FIELD}}\", &x)"; + code_ += " },"; + code_ += " };"; } else { // Most fields. - code_ += " ds.field(\"{{FIELD_NAME}}\", &self.{{FIELD_NAME}}());"; + code_ += " ds.field(\"{{FIELD}}\", &self.{{FIELD}}());"; } }); code_ += " ds.finish()"; @@ -2020,28 +2212,28 @@ class RustGenerator : public BaseGenerator { } void GenTableObject(const StructDef &table) { - code_.SetValue("OBJECT_NAME", NativeName(table)); - code_.SetValue("STRUCT_NAME", Name(table)); + code_.SetValue("STRUCT_OTY", namer_.ObjectType(table)); + code_.SetValue("STRUCT_TY", namer_.Type(table)); // Generate the native object. code_ += "#[non_exhaustive]"; code_ += "#[derive(Debug, Clone, PartialEq)]"; - code_ += "pub struct {{OBJECT_NAME}} {"; + code_ += "{{ACCESS_TYPE}} struct {{STRUCT_OTY}} {"; ForAllObjectTableFields(table, [&](const FieldDef &field) { // Union objects combine both the union discriminant and value, so we // skip making a field for the discriminant. if (field.value.type.base_type == BASE_TYPE_UTYPE) return; - code_ += " pub {{FIELD_NAME}}: {{FIELD_OBJECT_TYPE}},"; + code_ += "pub {{FIELD}}: {{FIELD_OTY}},"; }); code_ += "}"; - code_ += "impl Default for {{OBJECT_NAME}} {"; + code_ += "impl Default for {{STRUCT_OTY}} {"; code_ += " fn default() -> Self {"; code_ += " Self {"; ForAllObjectTableFields(table, [&](const FieldDef &field) { if (field.value.type.base_type == BASE_TYPE_UTYPE) return; std::string default_value = GetDefaultValue(field, kObject); - code_ += " {{FIELD_NAME}}: " + default_value + ","; + code_ += " {{FIELD}}: " + default_value + ","; }); code_ += " }"; code_ += " }"; @@ -2051,11 +2243,11 @@ class RustGenerator : public BaseGenerator { // may be required, they, and therefore enums need defaults. // Generate pack function. - code_ += "impl {{OBJECT_NAME}} {"; + code_ += "impl {{STRUCT_OTY}} {"; code_ += " pub fn pack<'b>("; code_ += " &self,"; code_ += " _fbb: &mut flatbuffers::FlatBufferBuilder<'b>"; - code_ += " ) -> flatbuffers::WIPOffset<{{STRUCT_NAME}}<'b>> {"; + code_ += " ) -> flatbuffers::WIPOffset<{{STRUCT_TY}}<'b>> {"; // First we generate variables for each field and then later assemble them // using "StructArgs" to more easily manage ownership of the builder. ForAllObjectTableFields(table, [&](const FieldDef &field) { @@ -2065,17 +2257,18 @@ class RustGenerator : public BaseGenerator { case ftBool: case ftFloat: case ftEnumKey: { - code_ += " let {{FIELD_NAME}} = self.{{FIELD_NAME}};"; + code_ += " let {{FIELD}} = self.{{FIELD}};"; return; } case ftUnionKey: return; // Generate union type with union value. case ftUnionValue: { - code_.SetValue("SNAKE_CASE_ENUM_NAME", - MakeSnakeCase(Name(*field.value.type.enum_def))); + code_.SetValue("ENUM_METHOD", + namer_.Method(*field.value.type.enum_def)); + code_.SetValue("DISCRIMINANT", namer_.LegacyRustUnionTypeMethod(field)); code_ += - " let {{FIELD_NAME}}_type = " - "self.{{FIELD_NAME}}.{{SNAKE_CASE_ENUM_NAME}}_type();"; - code_ += " let {{FIELD_NAME}} = self.{{FIELD_NAME}}.pack(_fbb);"; + " let {{DISCRIMINANT}} = " + "self.{{FIELD}}.{{ENUM_METHOD}}_type();"; + code_ += " let {{FIELD}} = self.{{FIELD}}.pack(_fbb);"; return; } // The rest of the types require special casing around optionalness @@ -2087,15 +2280,13 @@ class RustGenerator : public BaseGenerator { case ftStruct: { // Hold the struct in a variable so we can reference it. if (field.IsRequired()) { - code_ += - " let {{FIELD_NAME}}_tmp = " - "Some(self.{{FIELD_NAME}}.pack());"; + code_ += " let {{FIELD}}_tmp = Some(self.{{FIELD}}.pack());"; } else { code_ += - " let {{FIELD_NAME}}_tmp = self.{{FIELD_NAME}}" + " let {{FIELD}}_tmp = self.{{FIELD}}" ".as_ref().map(|x| x.pack());"; } - code_ += " let {{FIELD_NAME}} = {{FIELD_NAME}}_tmp.as_ref();"; + code_ += " let {{FIELD}} = {{FIELD}}_tmp.as_ref();"; return; } @@ -2123,8 +2314,8 @@ class RustGenerator : public BaseGenerator { MapNativeTableField( field, - "let w: Vec<_> = x.iter().map(|s| s.as_ref()).collect();" - "_fbb.create_vector_of_strings(&w)"); + "let w: Vec<_> = x.iter().map(|s| _fbb.create_string(s)).collect();" + "_fbb.create_vector(&w)"); return; } case ftVectorOfTable: { @@ -2146,10 +2337,10 @@ class RustGenerator : public BaseGenerator { } } }); - code_ += " {{STRUCT_NAME}}::create(_fbb, &{{STRUCT_NAME}}Args{"; + code_ += " {{STRUCT_TY}}::create(_fbb, &{{STRUCT_TY}}Args{"; ForAllObjectTableFields(table, [&](const FieldDef &field) { (void)field; // Unused. - code_ += " {{FIELD_NAME}},"; + code_ += " {{FIELD}},"; }); code_ += " })"; code_ += " }"; @@ -2161,23 +2352,25 @@ class RustGenerator : public BaseGenerator { for (auto it = v.begin(); it != v.end(); it++) { const FieldDef &field = **it; if (field.deprecated) continue; - code_.SetValue("FIELD_NAME", Name(field)); - code_.SetValue("FIELD_OBJECT_TYPE", ObjectFieldType(field, true)); + code_.SetValue("FIELD", namer_.Field(field)); + code_.SetValue("FIELD_OTY", ObjectFieldType(field, true)); + code_.IncrementIdentLevel(); cb(field); + code_.DecrementIdentLevel(); } } void MapNativeTableField(const FieldDef &field, const std::string &expr) { if (field.IsOptional()) { - code_ += " let {{FIELD_NAME}} = self.{{FIELD_NAME}}.as_ref().map(|x|{"; - code_ += " " + expr; - code_ += " });"; + code_ += " let {{FIELD}} = self.{{FIELD}}.as_ref().map(|x|{"; + code_ += " " + expr; + code_ += " });"; } else { // For some reason Args has optional types for required fields. // TODO(cneo): Fix this... but its a breaking change? - code_ += " let {{FIELD_NAME}} = Some({"; - code_ += " let x = &self.{{FIELD_NAME}};"; - code_ += " " + expr; - code_ += " });"; + code_ += " let {{FIELD}} = Some({"; + code_ += " let x = &self.{{FIELD}};"; + code_ += " " + expr; + code_ += " });"; } } @@ -2187,189 +2380,161 @@ class RustGenerator : public BaseGenerator { FLATBUFFERS_ASSERT(field.key); code_.SetValue("KEY_TYPE", GenTableAccessorFuncReturnType(field, "")); + code_.SetValue("REF", IsString(field.value.type) ? "" : "&"); - code_ += " #[inline]"; + code_ += "#[inline]"; code_ += - " pub fn key_compare_less_than(&self, o: &{{STRUCT_NAME}}) -> " - " bool {"; - code_ += " self.{{FIELD_NAME}}() < o.{{FIELD_NAME}}()"; - code_ += " }"; + "pub fn key_compare_less_than(&self, o: &{{STRUCT_TY}}) -> " + "bool {"; + code_ += " self.{{FIELD}}() < o.{{FIELD}}()"; + code_ += "}"; code_ += ""; - code_ += " #[inline]"; + code_ += "#[inline]"; code_ += - " pub fn key_compare_with_value(&self, val: {{KEY_TYPE}}) -> " - " ::std::cmp::Ordering {"; - code_ += " let key = self.{{FIELD_NAME}}();"; - code_ += " key.cmp(&val)"; - code_ += " }"; + "pub fn key_compare_with_value(&self, val: {{KEY_TYPE}}) -> " + "::core::cmp::Ordering {"; + code_ += " let key = self.{{FIELD}}();"; + code_ += " key.cmp({{REF}}val)"; + code_ += "}"; } // Generate functions for accessing the root table object. This function // must only be called if the root table is defined. void GenRootTableFuncs(const StructDef &struct_def) { FLATBUFFERS_ASSERT(parser_.root_struct_def_ && "root table not defined"); - auto name = Name(struct_def); - - code_.SetValue("STRUCT_NAME", name); - code_.SetValue("STRUCT_NAME_SNAKECASE", MakeSnakeCase(name)); - code_.SetValue("STRUCT_NAME_CAPS", MakeUpper(MakeSnakeCase(name))); + code_.SetValue("STRUCT_TY", namer_.Type(struct_def)); + code_.SetValue("STRUCT_FN", namer_.Function(struct_def)); + code_.SetValue("STRUCT_CONST", namer_.Constant(struct_def.name)); - // The root datatype accessors: - code_ += "#[inline]"; - code_ += - "#[deprecated(since=\"2.0.0\", " - "note=\"Deprecated in favor of `root_as...` methods.\")]"; - code_ += - "pub fn get_root_as_{{STRUCT_NAME_SNAKECASE}}<'a>(buf: &'a [u8])" - " -> {{STRUCT_NAME}}<'a> {"; - code_ += - " unsafe { flatbuffers::root_unchecked::<{{STRUCT_NAME}}" - "<'a>>(buf) }"; - code_ += "}"; - code_ += ""; - - code_ += "#[inline]"; - code_ += - "#[deprecated(since=\"2.0.0\", " - "note=\"Deprecated in favor of `root_as...` methods.\")]"; - code_ += - "pub fn get_size_prefixed_root_as_{{STRUCT_NAME_SNAKECASE}}" - "<'a>(buf: &'a [u8]) -> {{STRUCT_NAME}}<'a> {"; - code_ += - " unsafe { flatbuffers::size_prefixed_root_unchecked::<{{STRUCT_NAME}}" - "<'a>>(buf) }"; - code_ += "}"; - code_ += ""; // Default verifier root fns. code_ += "#[inline]"; - code_ += "/// Verifies that a buffer of bytes contains a `{{STRUCT_NAME}}`"; + code_ += "/// Verifies that a buffer of bytes contains a `{{STRUCT_TY}}`"; code_ += "/// and returns it."; code_ += "/// Note that verification is still experimental and may not"; code_ += "/// catch every error, or be maximally performant. For the"; code_ += "/// previous, unchecked, behavior use"; - code_ += "/// `root_as_{{STRUCT_NAME_SNAKECASE}}_unchecked`."; + code_ += "/// `root_as_{{STRUCT_FN}}_unchecked`."; code_ += - "pub fn root_as_{{STRUCT_NAME_SNAKECASE}}(buf: &[u8]) " - "-> Result<{{STRUCT_NAME}}, flatbuffers::InvalidFlatbuffer> {"; - code_ += " flatbuffers::root::<{{STRUCT_NAME}}>(buf)"; + "pub fn root_as_{{STRUCT_FN}}(buf: &[u8]) " + "-> Result<{{STRUCT_TY}}, flatbuffers::InvalidFlatbuffer> {"; + code_ += " flatbuffers::root::<{{STRUCT_TY}}>(buf)"; code_ += "}"; code_ += "#[inline]"; code_ += "/// Verifies that a buffer of bytes contains a size prefixed"; - code_ += "/// `{{STRUCT_NAME}}` and returns it."; + code_ += "/// `{{STRUCT_TY}}` and returns it."; code_ += "/// Note that verification is still experimental and may not"; code_ += "/// catch every error, or be maximally performant. For the"; code_ += "/// previous, unchecked, behavior use"; - code_ += "/// `size_prefixed_root_as_{{STRUCT_NAME_SNAKECASE}}_unchecked`."; + code_ += "/// `size_prefixed_root_as_{{STRUCT_FN}}_unchecked`."; code_ += - "pub fn size_prefixed_root_as_{{STRUCT_NAME_SNAKECASE}}" - "(buf: &[u8]) -> Result<{{STRUCT_NAME}}, " + "pub fn size_prefixed_root_as_{{STRUCT_FN}}" + "(buf: &[u8]) -> Result<{{STRUCT_TY}}, " "flatbuffers::InvalidFlatbuffer> {"; - code_ += " flatbuffers::size_prefixed_root::<{{STRUCT_NAME}}>(buf)"; + code_ += " flatbuffers::size_prefixed_root::<{{STRUCT_TY}}>(buf)"; code_ += "}"; // Verifier with options root fns. code_ += "#[inline]"; code_ += "/// Verifies, with the given options, that a buffer of bytes"; - code_ += "/// contains a `{{STRUCT_NAME}}` and returns it."; + code_ += "/// contains a `{{STRUCT_TY}}` and returns it."; code_ += "/// Note that verification is still experimental and may not"; code_ += "/// catch every error, or be maximally performant. For the"; code_ += "/// previous, unchecked, behavior use"; - code_ += "/// `root_as_{{STRUCT_NAME_SNAKECASE}}_unchecked`."; - code_ += "pub fn root_as_{{STRUCT_NAME_SNAKECASE}}_with_opts<'b, 'o>("; + code_ += "/// `root_as_{{STRUCT_FN}}_unchecked`."; + code_ += "pub fn root_as_{{STRUCT_FN}}_with_opts<'b, 'o>("; code_ += " opts: &'o flatbuffers::VerifierOptions,"; code_ += " buf: &'b [u8],"; code_ += - ") -> Result<{{STRUCT_NAME}}<'b>, flatbuffers::InvalidFlatbuffer>" + ") -> Result<{{STRUCT_TY}}<'b>, flatbuffers::InvalidFlatbuffer>" " {"; - code_ += " flatbuffers::root_with_opts::<{{STRUCT_NAME}}<'b>>(opts, buf)"; + code_ += " flatbuffers::root_with_opts::<{{STRUCT_TY}}<'b>>(opts, buf)"; code_ += "}"; code_ += "#[inline]"; code_ += "/// Verifies, with the given verifier options, that a buffer of"; - code_ += "/// bytes contains a size prefixed `{{STRUCT_NAME}}` and returns"; + code_ += "/// bytes contains a size prefixed `{{STRUCT_TY}}` and returns"; code_ += "/// it. Note that verification is still experimental and may not"; code_ += "/// catch every error, or be maximally performant. For the"; code_ += "/// previous, unchecked, behavior use"; - code_ += "/// `root_as_{{STRUCT_NAME_SNAKECASE}}_unchecked`."; + code_ += "/// `root_as_{{STRUCT_FN}}_unchecked`."; code_ += - "pub fn size_prefixed_root_as_{{STRUCT_NAME_SNAKECASE}}_with_opts" + "pub fn size_prefixed_root_as_{{STRUCT_FN}}_with_opts" "<'b, 'o>("; code_ += " opts: &'o flatbuffers::VerifierOptions,"; code_ += " buf: &'b [u8],"; code_ += - ") -> Result<{{STRUCT_NAME}}<'b>, flatbuffers::InvalidFlatbuffer>" + ") -> Result<{{STRUCT_TY}}<'b>, flatbuffers::InvalidFlatbuffer>" " {"; code_ += - " flatbuffers::size_prefixed_root_with_opts::<{{STRUCT_NAME}}" + " flatbuffers::size_prefixed_root_with_opts::<{{STRUCT_TY}}" "<'b>>(opts, buf)"; code_ += "}"; // Unchecked root fns. code_ += "#[inline]"; code_ += "/// Assumes, without verification, that a buffer of bytes " - "contains a {{STRUCT_NAME}} and returns it."; + "contains a {{STRUCT_TY}} and returns it."; code_ += "/// # Safety"; code_ += "/// Callers must trust the given bytes do indeed contain a valid" - " `{{STRUCT_NAME}}`."; + " `{{STRUCT_TY}}`."; code_ += - "pub unsafe fn root_as_{{STRUCT_NAME_SNAKECASE}}_unchecked" - "(buf: &[u8]) -> {{STRUCT_NAME}} {"; - code_ += " flatbuffers::root_unchecked::<{{STRUCT_NAME}}>(buf)"; + "pub unsafe fn root_as_{{STRUCT_FN}}_unchecked" + "(buf: &[u8]) -> {{STRUCT_TY}} {"; + code_ += " flatbuffers::root_unchecked::<{{STRUCT_TY}}>(buf)"; code_ += "}"; code_ += "#[inline]"; code_ += "/// Assumes, without verification, that a buffer of bytes " - "contains a size prefixed {{STRUCT_NAME}} and returns it."; + "contains a size prefixed {{STRUCT_TY}} and returns it."; code_ += "/// # Safety"; code_ += "/// Callers must trust the given bytes do indeed contain a valid" - " size prefixed `{{STRUCT_NAME}}`."; + " size prefixed `{{STRUCT_TY}}`."; code_ += - "pub unsafe fn size_prefixed_root_as_{{STRUCT_NAME_SNAKECASE}}" - "_unchecked(buf: &[u8]) -> {{STRUCT_NAME}} {"; + "pub unsafe fn size_prefixed_root_as_{{STRUCT_FN}}" + "_unchecked(buf: &[u8]) -> {{STRUCT_TY}} {"; code_ += - " flatbuffers::size_prefixed_root_unchecked::<{{STRUCT_NAME}}>" + " flatbuffers::size_prefixed_root_unchecked::<{{STRUCT_TY}}>" "(buf)"; code_ += "}"; if (parser_.file_identifier_.length()) { // Declare the identifier // (no lifetime needed as constants have static lifetimes by default) - code_ += "pub const {{STRUCT_NAME_CAPS}}_IDENTIFIER: &str\\"; + code_ += "pub const {{STRUCT_CONST}}_IDENTIFIER: &str\\"; code_ += " = \"" + parser_.file_identifier_ + "\";"; code_ += ""; // Check if a buffer has the identifier. code_ += "#[inline]"; - code_ += "pub fn {{STRUCT_NAME_SNAKECASE}}_buffer_has_identifier\\"; + code_ += "pub fn {{STRUCT_FN}}_buffer_has_identifier\\"; code_ += "(buf: &[u8]) -> bool {"; code_ += " flatbuffers::buffer_has_identifier(buf, \\"; - code_ += "{{STRUCT_NAME_CAPS}}_IDENTIFIER, false)"; + code_ += "{{STRUCT_CONST}}_IDENTIFIER, false)"; code_ += "}"; code_ += ""; code_ += "#[inline]"; - code_ += "pub fn {{STRUCT_NAME_SNAKECASE}}_size_prefixed\\"; + code_ += "pub fn {{STRUCT_FN}}_size_prefixed\\"; code_ += "_buffer_has_identifier(buf: &[u8]) -> bool {"; code_ += " flatbuffers::buffer_has_identifier(buf, \\"; - code_ += "{{STRUCT_NAME_CAPS}}_IDENTIFIER, true)"; + code_ += "{{STRUCT_CONST}}_IDENTIFIER, true)"; code_ += "}"; code_ += ""; } if (parser_.file_extension_.length()) { // Return the extension - code_ += "pub const {{STRUCT_NAME_CAPS}}_EXTENSION: &str = \\"; + code_ += "pub const {{STRUCT_CONST}}_EXTENSION: &str = \\"; code_ += "\"" + parser_.file_extension_ + "\";"; code_ += ""; } // Finish a buffer with a given root object: - code_.SetValue("OFFSET_TYPELABEL", Name(struct_def) + "Offset"); code_ += "#[inline]"; - code_ += "pub fn finish_{{STRUCT_NAME_SNAKECASE}}_buffer<'a, 'b>("; + code_ += "pub fn finish_{{STRUCT_FN}}_buffer<'a, 'b>("; code_ += " fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>,"; - code_ += " root: flatbuffers::WIPOffset<{{STRUCT_NAME}}<'a>>) {"; + code_ += " root: flatbuffers::WIPOffset<{{STRUCT_TY}}<'a>>) {"; if (parser_.file_identifier_.length()) { - code_ += " fbb.finish(root, Some({{STRUCT_NAME_CAPS}}_IDENTIFIER));"; + code_ += " fbb.finish(root, Some({{STRUCT_CONST}}_IDENTIFIER));"; } else { code_ += " fbb.finish(root, None);"; } @@ -2377,14 +2542,14 @@ class RustGenerator : public BaseGenerator { code_ += ""; code_ += "#[inline]"; code_ += - "pub fn finish_size_prefixed_{{STRUCT_NAME_SNAKECASE}}_buffer" + "pub fn finish_size_prefixed_{{STRUCT_FN}}_buffer" "<'a, 'b>(" "fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, " - "root: flatbuffers::WIPOffset<{{STRUCT_NAME}}<'a>>) {"; + "root: flatbuffers::WIPOffset<{{STRUCT_TY}}<'a>>) {"; if (parser_.file_identifier_.length()) { code_ += " fbb.finish_size_prefixed(root, " - "Some({{STRUCT_NAME_CAPS}}_IDENTIFIER));"; + "Some({{STRUCT_CONST}}_IDENTIFIER));"; } else { code_ += " fbb.finish_size_prefixed(root, None);"; } @@ -2421,25 +2586,30 @@ class RustGenerator : public BaseGenerator { it != struct_def.fields.vec.end(); ++it) { const auto &field = **it; code_.SetValue("FIELD_TYPE", GetTypeGet(field.value.type)); - code_.SetValue("FIELD_OBJECT_TYPE", ObjectFieldType(field, false)); - code_.SetValue("FIELD_NAME", Name(field)); + code_.SetValue("FIELD_OTY", ObjectFieldType(field, false)); + code_.SetValue("FIELD", namer_.Field(field)); code_.SetValue("FIELD_OFFSET", NumToString(offset_to_field)); code_.SetValue( "REF", IsStruct(field.value.type) || IsArray(field.value.type) ? "&" : ""); + code_.IncrementIdentLevel(); cb(field); + code_.DecrementIdentLevel(); const size_t size = InlineSize(field.value.type); offset_to_field += size + field.padding; } } // Generate an accessor struct with constructor for a flatbuffers struct. void GenStruct(const StructDef &struct_def) { + const bool is_private = parser_.opts.no_leak_private_annotations && + (struct_def.attributes.Lookup("private") != nullptr); + code_.SetValue("ACCESS_TYPE", is_private ? "pub(crate)" : "pub"); // Generates manual padding and alignment. // Variables are private because they contain little endian data on all // platforms. GenComment(struct_def.doc_comment); code_.SetValue("ALIGN", NumToString(struct_def.minalign)); - code_.SetValue("STRUCT_NAME", Name(struct_def)); + code_.SetValue("STRUCT_TY", namer_.Type(struct_def)); code_.SetValue("STRUCT_SIZE", NumToString(struct_def.bytesize)); // We represent Flatbuffers-structs in Rust-u8-arrays since the data may be @@ -2448,25 +2618,25 @@ class RustGenerator : public BaseGenerator { // PartialEq is useful to derive because we can correctly compare structs // for equality by just comparing their underlying byte data. This doesn't // hold for PartialOrd/Ord. - code_ += "// struct {{STRUCT_NAME}}, aligned to {{ALIGN}}"; + code_ += "// struct {{STRUCT_TY}}, aligned to {{ALIGN}}"; code_ += "#[repr(transparent)]"; code_ += "#[derive(Clone, Copy, PartialEq)]"; - code_ += "pub struct {{STRUCT_NAME}}(pub [u8; {{STRUCT_SIZE}}]);"; - code_ += "impl Default for {{STRUCT_NAME}} { "; + code_ += "{{ACCESS_TYPE}} struct {{STRUCT_TY}}(pub [u8; {{STRUCT_SIZE}}]);"; + code_ += "impl Default for {{STRUCT_TY}} { "; code_ += " fn default() -> Self { "; code_ += " Self([0; {{STRUCT_SIZE}}])"; code_ += " }"; code_ += "}"; // Debug for structs. - code_ += "impl std::fmt::Debug for {{STRUCT_NAME}} {"; + code_ += "impl core::fmt::Debug for {{STRUCT_TY}} {"; code_ += - " fn fmt(&self, f: &mut std::fmt::Formatter" - ") -> std::fmt::Result {"; - code_ += " f.debug_struct(\"{{STRUCT_NAME}}\")"; + " fn fmt(&self, f: &mut core::fmt::Formatter" + ") -> core::fmt::Result {"; + code_ += " f.debug_struct(\"{{STRUCT_TY}}\")"; ForAllStructFields(struct_def, [&](const FieldDef &unused) { (void)unused; - code_ += " .field(\"{{FIELD_NAME}}\", &self.{{FIELD_NAME}}())"; + code_ += " .field(\"{{FIELD}}\", &self.{{FIELD}}())"; }); code_ += " .finish()"; code_ += " }"; @@ -2476,44 +2646,26 @@ class RustGenerator : public BaseGenerator { // Generate impls for SafeSliceAccess (because all structs are endian-safe), // Follow for the value type, Follow for the reference type, Push for the // value type, and Push for the reference type. - code_ += "impl flatbuffers::SimpleToVerifyInSlice for {{STRUCT_NAME}} {}"; - code_ += "impl flatbuffers::SafeSliceAccess for {{STRUCT_NAME}} {}"; - code_ += "impl<'a> flatbuffers::Follow<'a> for {{STRUCT_NAME}} {"; - code_ += " type Inner = &'a {{STRUCT_NAME}};"; + code_ += "impl flatbuffers::SimpleToVerifyInSlice for {{STRUCT_TY}} {}"; + code_ += "impl<'a> flatbuffers::Follow<'a> for {{STRUCT_TY}} {"; + code_ += " type Inner = &'a {{STRUCT_TY}};"; code_ += " #[inline]"; - code_ += " fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {"; - code_ += " <&'a {{STRUCT_NAME}}>::follow(buf, loc)"; + code_ += " unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {"; + code_ += " <&'a {{STRUCT_TY}}>::follow(buf, loc)"; code_ += " }"; code_ += "}"; - code_ += "impl<'a> flatbuffers::Follow<'a> for &'a {{STRUCT_NAME}} {"; - code_ += " type Inner = &'a {{STRUCT_NAME}};"; + code_ += "impl<'a> flatbuffers::Follow<'a> for &'a {{STRUCT_TY}} {"; + code_ += " type Inner = &'a {{STRUCT_TY}};"; code_ += " #[inline]"; - code_ += " fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {"; - code_ += " flatbuffers::follow_cast_ref::<{{STRUCT_NAME}}>(buf, loc)"; + code_ += " unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {"; + code_ += " flatbuffers::follow_cast_ref::<{{STRUCT_TY}}>(buf, loc)"; code_ += " }"; code_ += "}"; - code_ += "impl<'b> flatbuffers::Push for {{STRUCT_NAME}} {"; - code_ += " type Output = {{STRUCT_NAME}};"; + code_ += "impl<'b> flatbuffers::Push for {{STRUCT_TY}} {"; + code_ += " type Output = {{STRUCT_TY}};"; code_ += " #[inline]"; - code_ += " fn push(&self, dst: &mut [u8], _rest: &[u8]) {"; - code_ += " let src = unsafe {"; - code_ += - " ::std::slice::from_raw_parts(" - "self as *const {{STRUCT_NAME}} as *const u8, Self::size())"; - code_ += " };"; - code_ += " dst.copy_from_slice(src);"; - code_ += " }"; - code_ += "}"; - code_ += "impl<'b> flatbuffers::Push for &'b {{STRUCT_NAME}} {"; - code_ += " type Output = {{STRUCT_NAME}};"; - code_ += ""; - code_ += " #[inline]"; - code_ += " fn push(&self, dst: &mut [u8], _rest: &[u8]) {"; - code_ += " let src = unsafe {"; - code_ += - " ::std::slice::from_raw_parts(" - "*self as *const {{STRUCT_NAME}} as *const u8, Self::size())"; - code_ += " };"; + code_ += " unsafe fn push(&self, dst: &mut [u8], _written_len: usize) {"; + code_ += " let src = ::core::slice::from_raw_parts(self as *const {{STRUCT_TY}} as *const u8, Self::size());"; code_ += " dst.copy_from_slice(src);"; code_ += " }"; code_ += "}"; @@ -2521,7 +2673,7 @@ class RustGenerator : public BaseGenerator { // Generate verifier: Structs are simple so presence and alignment are // all that need to be checked. - code_ += "impl<'a> flatbuffers::Verifiable for {{STRUCT_NAME}} {"; + code_ += "impl<'a> flatbuffers::Verifiable for {{STRUCT_TY}} {"; code_ += " #[inline]"; code_ += " fn run_verifier("; code_ += " v: &mut flatbuffers::Verifier, pos: usize"; @@ -2530,20 +2682,51 @@ class RustGenerator : public BaseGenerator { code_ += " v.in_buffer::<Self>(pos)"; code_ += " }"; code_ += "}"; + code_ += ""; + + // Implement serde::Serialize + if (parser_.opts.rust_serialize) { + const auto numFields = struct_def.fields.vec.size(); + code_.SetValue("NUM_FIELDS", NumToString(numFields)); + code_ += "impl Serialize for {{STRUCT_TY}} {"; + code_ += + " fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>"; + code_ += " where"; + code_ += " S: Serializer,"; + code_ += " {"; + if (numFields == 0) { + code_ += + " let s = serializer.serialize_struct(\"{{STRUCT_TY}}\", 0)?;"; + } else { + code_ += + " let mut s = serializer.serialize_struct(\"{{STRUCT_TY}}\", " + "{{NUM_FIELDS}})?;"; + } + ForAllStructFields(struct_def, [&](const FieldDef &unused) { + (void)unused; + code_ += + " s.serialize_field(\"{{FIELD}}\", " + "&self.{{FIELD}}())?;"; + }); + code_ += " s.end()"; + code_ += " }"; + code_ += "}"; + code_ += ""; + } // Generate a constructor that takes all fields as arguments. - code_ += "impl<'a> {{STRUCT_NAME}} {"; + code_ += "impl<'a> {{STRUCT_TY}} {"; code_ += " #[allow(clippy::too_many_arguments)]"; code_ += " pub fn new("; ForAllStructFields(struct_def, [&](const FieldDef &unused) { (void)unused; - code_ += " {{FIELD_NAME}}: {{REF}}{{FIELD_TYPE}},"; + code_ += " {{FIELD}}: {{REF}}{{FIELD_TYPE}},"; }); code_ += " ) -> Self {"; code_ += " let mut s = Self([0; {{STRUCT_SIZE}}]);"; ForAllStructFields(struct_def, [&](const FieldDef &unused) { (void)unused; - code_ += " s.set_{{FIELD_NAME}}({{REF}}{{FIELD_NAME}});"; + code_ += " s.set_{{FIELD}}({{FIELD}});"; }); code_ += " s"; code_ += " }"; @@ -2555,12 +2738,15 @@ class RustGenerator : public BaseGenerator { // Generate accessor methods for the struct. ForAllStructFields(struct_def, [&](const FieldDef &field) { - this->GenComment(field.doc_comment, " "); + this->GenComment(field.doc_comment); // Getter. if (IsStruct(field.value.type)) { - code_ += " pub fn {{FIELD_NAME}}(&self) -> &{{FIELD_TYPE}} {"; + code_ += "pub fn {{FIELD}}(&self) -> &{{FIELD_TYPE}} {"; + code_ += " // Safety:"; + code_ += " // Created from a valid Table for this object"; + code_ += " // Which contains a valid struct in this slot"; code_ += - " unsafe {" + " unsafe {" " &*(self.0[{{FIELD_OFFSET}}..].as_ptr() as *const" " {{FIELD_TYPE}}) }"; } else if (IsArray(field.value.type)) { @@ -2568,30 +2754,37 @@ class RustGenerator : public BaseGenerator { NumToString(field.value.type.fixed_length)); code_.SetValue("ARRAY_ITEM", GetTypeGet(field.value.type.VectorType())); code_ += - " pub fn {{FIELD_NAME}}(&'a self) -> " + "pub fn {{FIELD}}(&'a self) -> " "flatbuffers::Array<'a, {{ARRAY_ITEM}}, {{ARRAY_SIZE}}> {"; - code_ += " flatbuffers::Array::follow(&self.0, {{FIELD_OFFSET}})"; + code_ += " // Safety:"; + code_ += " // Created from a valid Table for this object"; + code_ += " // Which contains a valid array in this slot"; + code_ += " unsafe { flatbuffers::Array::follow(&self.0, {{FIELD_OFFSET}}) }"; } else { - code_ += " pub fn {{FIELD_NAME}}(&self) -> {{FIELD_TYPE}} {"; + code_ += "pub fn {{FIELD}}(&self) -> {{FIELD_TYPE}} {"; code_ += - " let mut mem = core::mem::MaybeUninit::" - "<{{FIELD_TYPE}}>::uninit();"; - code_ += " unsafe {"; - code_ += " core::ptr::copy_nonoverlapping("; - code_ += " self.0[{{FIELD_OFFSET}}..].as_ptr(),"; - code_ += " mem.as_mut_ptr() as *mut u8,"; - code_ += " core::mem::size_of::<{{FIELD_TYPE}}>(),"; - code_ += " );"; - code_ += " mem.assume_init()"; - code_ += " }.from_little_endian()"; - } - code_ += " }\n"; + " let mut mem = core::mem::MaybeUninit::" + "<<{{FIELD_TYPE}} as EndianScalar>::Scalar>::uninit();"; + code_ += " // Safety:"; + code_ += " // Created from a valid Table for this object"; + code_ += " // Which contains a valid value in this slot"; + code_ += " EndianScalar::from_little_endian(unsafe {"; + code_ += " core::ptr::copy_nonoverlapping("; + code_ += " self.0[{{FIELD_OFFSET}}..].as_ptr(),"; + code_ += " mem.as_mut_ptr() as *mut u8,"; + code_ += " core::mem::size_of::<<{{FIELD_TYPE}} as EndianScalar>::Scalar>(),"; + code_ += " );"; + code_ += " mem.assume_init()"; + code_ += " })"; + } + code_ += "}\n"; // Setter. if (IsStruct(field.value.type)) { code_.SetValue("FIELD_SIZE", NumToString(InlineSize(field.value.type))); - code_ += " pub fn set_{{FIELD_NAME}}(&mut self, x: &{{FIELD_TYPE}}) {"; + code_ += "#[allow(clippy::identity_op)]"; // If FIELD_OFFSET=0. + code_ += "pub fn set_{{FIELD}}(&mut self, x: &{{FIELD_TYPE}}) {"; code_ += - " self.0[{{FIELD_OFFSET}}..{{FIELD_OFFSET}}+{{FIELD_SIZE}}]" + " self.0[{{FIELD_OFFSET}}..{{FIELD_OFFSET}} + {{FIELD_SIZE}}]" ".copy_from_slice(&x.0)"; } else if (IsArray(field.value.type)) { if (GetFullType(field.value.type) == ftArrayOfBuiltin) { @@ -2601,36 +2794,44 @@ class RustGenerator : public BaseGenerator { "ARRAY_ITEM_SIZE", NumToString(InlineSize(field.value.type.VectorType()))); code_ += - " pub fn set_{{FIELD_NAME}}(&mut self, items: &{{FIELD_TYPE}}) " + "pub fn set_{{FIELD}}(&mut self, items: &{{FIELD_TYPE}}) " "{"; + code_ += " // Safety:"; + code_ += " // Created from a valid Table for this object"; + code_ += " // Which contains a valid array in this slot"; code_ += - " flatbuffers::emplace_scalar_array(&mut self.0, " - "{{FIELD_OFFSET}}, items);"; + " unsafe { flatbuffers::emplace_scalar_array(&mut self.0, " + "{{FIELD_OFFSET}}, items) };"; } else { code_.SetValue("FIELD_SIZE", NumToString(InlineSize(field.value.type))); - code_ += - " pub fn set_{{FIELD_NAME}}(&mut self, x: &{{FIELD_TYPE}}) {"; - code_ += " unsafe {"; - code_ += " std::ptr::copy("; - code_ += " x.as_ptr() as *const u8,"; - code_ += " self.0.as_mut_ptr().add({{FIELD_OFFSET}}),"; - code_ += " {{FIELD_SIZE}},"; - code_ += " );"; - code_ += " }"; + code_ += "pub fn set_{{FIELD}}(&mut self, x: &{{FIELD_TYPE}}) {"; + code_ += " // Safety:"; + code_ += " // Created from a valid Table for this object"; + code_ += " // Which contains a valid array in this slot"; + code_ += " unsafe {"; + code_ += " core::ptr::copy("; + code_ += " x.as_ptr() as *const u8,"; + code_ += " self.0.as_mut_ptr().add({{FIELD_OFFSET}}),"; + code_ += " {{FIELD_SIZE}},"; + code_ += " );"; + code_ += " }"; } } else { - code_ += " pub fn set_{{FIELD_NAME}}(&mut self, x: {{FIELD_TYPE}}) {"; - code_ += " let x_le = x.to_little_endian();"; - code_ += " unsafe {"; - code_ += " core::ptr::copy_nonoverlapping("; - code_ += " &x_le as *const {{FIELD_TYPE}} as *const u8,"; - code_ += " self.0[{{FIELD_OFFSET}}..].as_mut_ptr(),"; - code_ += " core::mem::size_of::<{{FIELD_TYPE}}>(),"; - code_ += " );"; - code_ += " }"; - } - code_ += " }\n"; + code_ += "pub fn set_{{FIELD}}(&mut self, x: {{FIELD_TYPE}}) {"; + code_ += " let x_le = x.to_little_endian();"; + code_ += " // Safety:"; + code_ += " // Created from a valid Table for this object"; + code_ += " // Which contains a valid value in this slot"; + code_ += " unsafe {"; + code_ += " core::ptr::copy_nonoverlapping("; + code_ += " &x_le as *const _ as *const u8,"; + code_ += " self.0[{{FIELD_OFFSET}}..].as_mut_ptr(),"; + code_ += " core::mem::size_of::<<{{FIELD_TYPE}} as EndianScalar>::Scalar>(),"; + code_ += " );"; + code_ += " }"; + } + code_ += "}\n"; // Generate a comparison function for this field if it is a key. if (field.key) { GenKeyFieldMethods(field); } @@ -2638,22 +2839,22 @@ class RustGenerator : public BaseGenerator { // Generate Object API unpack method. if (parser_.opts.generate_object_based_api) { - code_.SetValue("NATIVE_STRUCT_NAME", NativeName(struct_def)); - code_ += " pub fn unpack(&self) -> {{NATIVE_STRUCT_NAME}} {"; - code_ += " {{NATIVE_STRUCT_NAME}} {"; + code_.SetValue("STRUCT_OTY", namer_.ObjectType(struct_def)); + code_ += " pub fn unpack(&self) -> {{STRUCT_OTY}} {"; + code_ += " {{STRUCT_OTY}} {"; ForAllStructFields(struct_def, [&](const FieldDef &field) { if (IsArray(field.value.type)) { if (GetFullType(field.value.type) == ftArrayOfStruct) { code_ += - " {{FIELD_NAME}}: { let {{FIELD_NAME}} = " - "self.{{FIELD_NAME}}(); flatbuffers::array_init(|i| " - "{{FIELD_NAME}}.get(i).unpack()) },"; + " {{FIELD}}: { let {{FIELD}} = " + "self.{{FIELD}}(); flatbuffers::array_init(|i| " + "{{FIELD}}.get(i).unpack()) },"; } else { - code_ += " {{FIELD_NAME}}: self.{{FIELD_NAME}}().into(),"; + code_ += " {{FIELD}}: self.{{FIELD}}().into(),"; } } else { std::string unpack = IsStruct(field.value.type) ? ".unpack()" : ""; - code_ += " {{FIELD_NAME}}: self.{{FIELD_NAME}}()" + unpack + ","; + code_ += " {{FIELD}}: self.{{FIELD}}()" + unpack + ","; } }); code_ += " }"; @@ -2667,30 +2868,30 @@ class RustGenerator : public BaseGenerator { if (parser_.opts.generate_object_based_api) { // Struct declaration code_ += "#[derive(Debug, Clone, PartialEq, Default)]"; - code_ += "pub struct {{NATIVE_STRUCT_NAME}} {"; + code_ += "{{ACCESS_TYPE}} struct {{STRUCT_OTY}} {"; ForAllStructFields(struct_def, [&](const FieldDef &field) { (void)field; // unused. - code_ += " pub {{FIELD_NAME}}: {{FIELD_OBJECT_TYPE}},"; + code_ += "pub {{FIELD}}: {{FIELD_OTY}},"; }); code_ += "}"; // The `pack` method that turns the native struct into its Flatbuffers // counterpart. - code_ += "impl {{NATIVE_STRUCT_NAME}} {"; - code_ += " pub fn pack(&self) -> {{STRUCT_NAME}} {"; - code_ += " {{STRUCT_NAME}}::new("; + code_ += "impl {{STRUCT_OTY}} {"; + code_ += " pub fn pack(&self) -> {{STRUCT_TY}} {"; + code_ += " {{STRUCT_TY}}::new("; ForAllStructFields(struct_def, [&](const FieldDef &field) { if (IsStruct(field.value.type)) { - code_ += " &self.{{FIELD_NAME}}.pack(),"; + code_ += " &self.{{FIELD}}.pack(),"; } else if (IsArray(field.value.type)) { if (GetFullType(field.value.type) == ftArrayOfStruct) { code_ += - " &flatbuffers::array_init(|i| " - "self.{{FIELD_NAME}}[i].pack()),"; + " &flatbuffers::array_init(|i| " + "self.{{FIELD}}[i].pack()),"; } else { - code_ += " &self.{{FIELD_NAME}},"; + code_ += " &self.{{FIELD}},"; } } else { - code_ += " self.{{FIELD_NAME}},"; + code_ += " self.{{FIELD}},"; } }); code_ += " )"; @@ -2725,9 +2926,16 @@ class RustGenerator : public BaseGenerator { } } } - code_ += indent + "use std::mem;"; - code_ += indent + "use std::cmp::Ordering;"; + code_ += indent + "use core::mem;"; + code_ += indent + "use core::cmp::Ordering;"; code_ += ""; + if (parser_.opts.rust_serialize) { + code_ += indent + "extern crate serde;"; + code_ += + indent + + "use self::serde::ser::{Serialize, Serializer, SerializeStruct};"; + code_ += ""; + } code_ += indent + "extern crate flatbuffers;"; code_ += indent + "use self::flatbuffers::{EndianScalar, Follow};"; } @@ -2766,7 +2974,7 @@ class RustGenerator : public BaseGenerator { // in the previous example, E, then F, then G are opened for (auto j = common_prefix_size; j != new_size; ++j) { code_ += "#[allow(unused_imports, dead_code)]"; - code_ += "pub mod " + MakeSnakeCase(ns->components[j]) + " {"; + code_ += "pub mod " + namer_.Namespace(ns->components[j]) + " {"; // Generate local namespace imports. GenNamespaceImports(2); } @@ -2774,6 +2982,9 @@ class RustGenerator : public BaseGenerator { cur_name_space_ = ns; } + + private: + IdlNamer namer_; }; } // namespace rust @@ -2799,6 +3010,59 @@ std::string RustMakeRule(const Parser &parser, const std::string &path, return make_rule; } +namespace { + +class RustCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateRust(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + output = RustMakeRule(parser, path, filename); + return Status::OK; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + if (!GenerateRustModuleRootFile(parser, path)) { return Status::ERROR; } + return Status::OK; + } + + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return true; } + + IDLOptions::Language Language() const override { return IDLOptions::kRust; } + + std::string LanguageName() const override { return "Rust"; } +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewRustCodeGenerator() { + return std::unique_ptr<RustCodeGenerator>(new RustCodeGenerator()); +} + } // namespace flatbuffers // TODO(rw): Generated code should import other generated files. diff --git a/contrib/libs/flatbuffers/src/idl_gen_rust.h b/contrib/libs/flatbuffers/src/idl_gen_rust.h new file mode 100644 index 00000000000..ef17ed8ebfe --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_rust.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_RUST_H_ +#define FLATBUFFERS_IDL_GEN_RUST_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Rust code generator. +std::unique_ptr<CodeGenerator> NewRustCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_RUST_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_swift.cpp b/contrib/libs/flatbuffers/src/idl_gen_swift.cpp index 3fffd39455e..695f5a0d753 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_swift.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_swift.cpp @@ -14,6 +14,8 @@ * limitations under the License. */ +#include "idl_gen_swift.h" + #include <cctype> #include <unordered_set> @@ -21,21 +23,133 @@ #include "flatbuffers/flatbuffers.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" +#include "idl_namer.h" namespace flatbuffers { namespace swift { -inline std::string GenIndirect(const std::string &reading) { +namespace { + +static Namer::Config SwiftDefaultConfig() { + return { /*types=*/Case::kKeep, + /*constants=*/Case::kLowerCamel, + /*methods=*/Case::kLowerCamel, + /*functions=*/Case::kLowerCamel, + /*fields=*/Case::kLowerCamel, + /*variables=*/Case::kLowerCamel, + /*variants=*/Case::kLowerCamel, + /*enum_variant_seperator=*/".", + /*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase, + /*namespaces=*/Case::kKeep, + /*namespace_seperator=*/"_", + /*object_prefix=*/"", + /*object_suffix=*/"T", + /*keyword_prefix=*/"", + /*keyword_suffix=*/"_", + /*filenames=*/Case::kKeep, + /*directories=*/Case::kKeep, + /*output_path=*/"", + /*filename_suffix=*/"_generated", + /*filename_extension=*/".swift" }; +} + +static std::set<std::string> SwiftKeywords() { + return { + "associatedtype", + "class", + "deinit", + "enum", + "extension", + "fileprivate", + "func", + "import", + "init", + "inout", + "internal", + "let", + "open", + "operator", + "private", + "protocol", + "public", + "rethrows", + "static", + "struct", + "subscript", + "typealias", + "var", + "break", + "case", + "continue", + "default", + "defer", + "do", + "else", + "fallthrough", + "for", + "guard", + "if", + "in", + "repeat", + "return", + "switch", + "where", + "while", + "Any", + "catch", + "false", + "is", + "nil", + "super", + "self", + "Self", + "throw", + "throws", + "true", + "try", + "associativity", + "convenience", + "dynamic", + "didSet", + "final", + "get", + "infix", + "indirect", + "lazy", + "left", + "mutating", + "none", + "nonmutating", + "optional", + "override", + "postfix", + "precedence", + "prefix", + "Protocol", + "required", + "right", + "set", + "Type", + "unowned", + "weak", + "willSet", + "Void", + }; +} + +static std::string GenIndirect(const std::string &reading) { return "{{ACCESS}}.indirect(" + reading + ")"; } -inline std::string GenArrayMainBody(const std::string &optional) { - return "{{ACCESS_TYPE}} func {{VALUENAME}}(at index: Int32) -> " +static std::string GenArrayMainBody(const std::string &optional) { + return "{{ACCESS_TYPE}} func {{FIELDMETHOD}}(at index: Int32) -> " "{{VALUETYPE}}" + optional + " { "; } +} // namespace + class SwiftGenerator : public BaseGenerator { private: CodeWriter code_; @@ -45,92 +159,11 @@ class SwiftGenerator : public BaseGenerator { public: SwiftGenerator(const Parser &parser, const std::string &path, const std::string &file_name) - : BaseGenerator(parser, path, file_name, "", "_", "swift") { + : BaseGenerator(parser, path, file_name, "", "_", "swift"), + namer_(WithFlagOptions(SwiftDefaultConfig(), parser.opts, path), + SwiftKeywords()) { namespace_depth = 0; code_.SetPadding(" "); - static const char *const keywords[] = { - "associatedtype", - "class", - "deinit", - "enum", - "extension", - "fileprivate", - "func", - "import", - "init", - "inout", - "internal", - "let", - "open", - "operator", - "private", - "protocol", - "public", - "rethrows", - "static", - "struct", - "subscript", - "typealias", - "var", - "break", - "case", - "continue", - "default", - "defer", - "do", - "else", - "fallthrough", - "for", - "guard", - "if", - "in", - "repeat", - "return", - "switch", - "where", - "while", - "Any", - "catch", - "false", - "is", - "nil", - "super", - "self", - "Self", - "throw", - "throws", - "true", - "try", - "associativity", - "convenience", - "dynamic", - "didSet", - "final", - "get", - "infix", - "indirect", - "lazy", - "left", - "mutating", - "none", - "nonmutating", - "optional", - "override", - "postfix", - "precedence", - "prefix", - "Protocol", - "required", - "right", - "set", - "Type", - "unowned", - "weak", - "willSet", - "Void", - nullptr, - }; - for (auto kw = keywords; *kw; kw++) keywords_.insert(*kw); } bool generate() { @@ -140,9 +173,14 @@ class SwiftGenerator : public BaseGenerator { code_ += "// " + std::string(FlatBuffersGeneratedWarning()); code_ += "// swiftlint:disable all"; code_ += "// swiftformat:disable all\n"; - code_ += "import FlatBuffers\n"; - // Generate code for all the enum declarations. + if (parser_.opts.include_dependence_headers || parser_.opts.generate_all) { + if (parser_.opts.swift_implementation_only) + code_ += "@_implementationOnly \\"; + code_ += "import FlatBuffers\n"; + } + + // Generate code for all the enum declarations. for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end(); ++it) { const auto &enum_def = **it; @@ -183,11 +221,15 @@ class SwiftGenerator : public BaseGenerator { // Generates the reader for swift void GenStructReader(const StructDef &struct_def) { - auto is_private_access = struct_def.attributes.Lookup("private"); + const bool is_private_access = + parser_.opts.swift_implementation_only || + struct_def.attributes.Lookup("private") != nullptr; code_.SetValue("ACCESS_TYPE", is_private_access ? "internal" : "public"); GenComment(struct_def.doc_comment); - code_.SetValue("STRUCTNAME", NameWrappedInNameSpace(struct_def)); - code_ += "{{ACCESS_TYPE}} struct {{STRUCTNAME}}: NativeStruct\\"; + code_.SetValue("STRUCTNAME", namer_.NamespacedType(struct_def)); + code_ += + "{{ACCESS_TYPE}} struct {{STRUCTNAME}}: NativeStruct, Verifiable, " + "FlatbuffersInitializable\\"; if (parser_.opts.generate_object_based_api) code_ += ", NativeObject\\"; code_ += " {"; code_ += ""; @@ -201,14 +243,14 @@ class SwiftGenerator : public BaseGenerator { for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { - auto &field = **it; + const auto &field = **it; if (field.deprecated) continue; if (!constructor.empty()) constructor += ", "; - auto name = Name(field); - auto type = GenType(field.value.type); - code_.SetValue("VALUENAME", name); + const auto field_var = namer_.Variable(field); + const auto type = GenType(field.value.type); + code_.SetValue("FIELDVAR", field_var); if (IsEnum(field.value.type)) { code_.SetValue("BASEVALUE", GenTypeBasic(field.value.type, false)); } @@ -216,18 +258,21 @@ class SwiftGenerator : public BaseGenerator { GenComment(field.doc_comment); std::string valueType = IsEnum(field.value.type) ? "{{BASEVALUE}}" : "{{VALUETYPE}}"; - code_ += "private var _{{VALUENAME}}: " + valueType; - auto accessing_value = IsEnum(field.value.type) ? ".value" : ""; - auto base_value = - IsStruct(field.value.type) ? (type + "()") : field.value.constant; + code_ += "private var _{{FIELDVAR}}: " + valueType; + const auto accessing_value = IsEnum(field.value.type) ? ".value" : ""; + const auto base_value = + IsStruct(field.value.type) ? (type + "()") + : SwiftConstant(field); - main_constructor.push_back("_" + name + " = " + name + accessing_value); - base_constructor.push_back("_" + name + " = " + base_value); + main_constructor.push_back("_" + field_var + " = " + field_var + + accessing_value); + base_constructor.push_back("_" + field_var + " = " + base_value); if (field.padding) { GenPadding(field, &padding_id); } - constructor += name + ": " + type; + constructor += field_var + ": " + type; } code_ += ""; + BuildStructConstructor(struct_def); BuildObjectConstructor(main_constructor, constructor); BuildObjectConstructor(base_constructor, ""); @@ -236,18 +281,55 @@ class SwiftGenerator : public BaseGenerator { for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { - auto &field = **it; + const auto &field = **it; if (field.deprecated) continue; - auto name = Name(field); - auto type = GenType(field.value.type); - code_.SetValue("VALUENAME", name); - code_.SetValue("VALUETYPE", type); + code_.SetValue("FIELDVAR", namer_.Variable(field)); + code_.SetValue("VALUETYPE", GenType(field.value.type)); GenComment(field.doc_comment); if (!IsEnum(field.value.type)) { - code_ += GenReaderMainBody() + "_{{VALUENAME}} }"; + code_ += GenReaderMainBody() + "_{{FIELDVAR}} }"; } else if (IsEnum(field.value.type)) { code_ += - GenReaderMainBody() + "{{VALUETYPE}}(rawValue: _{{VALUENAME}})! }"; + GenReaderMainBody() + "{{VALUETYPE}}(rawValue: _{{FIELDVAR}})! }"; + } + } + code_ += ""; + code_ += + "{{ACCESS_TYPE}} static func verify<T>(_ verifier: inout Verifier, at " + "position: " + "Int, of type: T.Type) throws where T: Verifiable {"; + Indent(); + code_ += + "try verifier.inBuffer(position: position, of: {{STRUCTNAME}}.self)"; + Outdent(); + code_ += "}"; + Outdent(); + code_ += "}\n"; + if (parser_.opts.gen_json_coders) GenerateJSONEncodingAPIs(struct_def); + } + + void BuildStructConstructor(const StructDef &struct_def) { + code_ += "{{ACCESS_TYPE}} init(_ bb: ByteBuffer, o: Int32) {"; + Indent(); + code_ += "let {{ACCESS}} = Struct(bb: bb, position: o)"; + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + const auto &field = **it; + if (field.deprecated) continue; + const auto type = field.value.type; + code_.SetValue("FIELDVAR", namer_.Variable(field)); + code_.SetValue("VALUETYPE", GenType(type)); + code_.SetValue("OFFSET", NumToString(field.value.offset)); + if (IsScalar(type.base_type)) { + if (IsEnum(type)) + code_.SetValue("VALUETYPE", GenTypeBasic(field.value.type, false)); + code_ += + "_{{FIELDVAR}} = {{ACCESS}}.readBuffer(of: {{VALUETYPE}}.self, " + "at: {{OFFSET}})"; + } else { + code_ += + "_{{FIELDVAR}} = {{VALUETYPE}}({{ACCESS}}.bb, o: " + "{{ACCESS}}.postion + {{OFFSET}})"; } } Outdent(); @@ -259,12 +341,11 @@ class SwiftGenerator : public BaseGenerator { for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { - auto &field = **it; + const auto &field = **it; if (field.deprecated) continue; - auto offset = NumToString(field.value.offset); - auto name = Name(field); - auto type = GenType(field.value.type); - code_.SetValue("VALUENAME", name); + const auto offset = NumToString(field.value.offset); + const auto type = GenType(field.value.type); + code_.SetValue("FIELDVAR", namer_.Variable(field)); if (IsEnum(field.value.type)) { code_.SetValue("BASEVALUE", GenTypeBasic(field.value.type, false)); } @@ -288,7 +369,7 @@ class SwiftGenerator : public BaseGenerator { } if (parser_.opts.generate_object_based_api) { - GenerateObjectAPIExtensionHeader(NameWrappedInNameSpace(struct_def)); + GenerateObjectAPIExtensionHeader(namer_.NamespacedType(struct_def)); code_ += "return builder.create(struct: obj)"; Outdent(); code_ += "}"; @@ -297,33 +378,6 @@ class SwiftGenerator : public BaseGenerator { code_ += "}\n"; } - // Generates the create function for swift - void GenStructWriter(const StructDef &struct_def) { - auto is_private_access = struct_def.attributes.Lookup("private"); - code_.SetValue("ACCESS_TYPE", is_private_access ? "internal" : "public"); - code_.SetValue("STRUCTNAME", NameWrappedInNameSpace(struct_def)); - code_.SetValue("SHORT_STRUCTNAME", Name(struct_def)); - code_ += "extension {{STRUCTNAME}} {"; - Indent(); - code_ += "@discardableResult"; - code_ += - "{{ACCESS_TYPE}} static func create{{SHORT_STRUCTNAME}}(builder: inout " - "FlatBufferBuilder, \\"; - std::string func_header = ""; - GenerateStructArgs(struct_def, &func_header, "", ""); - code_ += func_header.substr(0, func_header.size() - 2) + "\\"; - code_ += ") -> Offset {"; - Indent(); - code_ += - "builder.createStructOf(size: {{STRUCTNAME}}.size, alignment: " - "{{STRUCTNAME}}.alignment)"; - code_ += "return builder.endStruct()"; - Outdent(); - code_ += "}\n"; - Outdent(); - code_ += "}\n"; - } - void GenerateStructArgs(const StructDef &struct_def, std::string *code_ptr, const std::string &nameprefix, const std::string &object_name, @@ -332,7 +386,7 @@ class SwiftGenerator : public BaseGenerator { auto &code = *code_ptr; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { - auto &field = **it; + const auto &field = **it; if (field.deprecated) continue; const auto &field_type = field.value.type; if (IsStruct(field.value.type)) { @@ -340,23 +394,20 @@ class SwiftGenerator : public BaseGenerator { *field_type.struct_def, code_ptr, (nameprefix + field.name), (object_name + "." + field.name), obj_api_named, is_obj_api); } else { - auto name = Name(field); - auto type = GenType(field.value.type); + const auto field_var = namer_.Variable(field); + const auto field_field = namer_.Field(field); + const auto type = GenType(field.value.type); if (!is_obj_api) { - code += nameprefix + name + ": " + type; + code += nameprefix + field_var + ": " + type; if (!IsEnum(field.value.type)) { code += " = "; - auto is_bool = IsBool(field.value.type.base_type); - auto constant = - is_bool ? ("0" == field.value.constant ? "false" : "true") - : field.value.constant; - code += constant; + code += SwiftConstant(field); } code += ", "; continue; } - code += - nameprefix + name + ": " + obj_api_named + object_name + "." + name; + code += nameprefix + field_var + ": " + obj_api_named + object_name + + "." + field_field; code += ", "; } } @@ -366,17 +417,21 @@ class SwiftGenerator : public BaseGenerator { // Generates the reader for swift void GenTable(const StructDef &struct_def) { - auto is_private_access = struct_def.attributes.Lookup("private"); + const bool is_private_access = + parser_.opts.swift_implementation_only || + struct_def.attributes.Lookup("private") != nullptr; code_.SetValue("ACCESS_TYPE", is_private_access ? "internal" : "public"); - GenObjectHeader(struct_def); GenTableAccessors(struct_def); GenTableReader(struct_def); GenTableWriter(struct_def); if (parser_.opts.generate_object_based_api) GenerateObjectAPITableExtension(struct_def); + code_ += ""; + GenerateVerifier(struct_def); Outdent(); code_ += "}\n"; + if (parser_.opts.gen_json_coders) GenerateJSONEncodingAPIs(struct_def); } // Generates the reader for swift @@ -389,7 +444,7 @@ class SwiftGenerator : public BaseGenerator { it != struct_def.fields.vec.end(); ++it) { const auto &field = **it; if (field.deprecated) { continue; } - code_.SetValue("OFFSET_NAME", Name(field)); + code_.SetValue("OFFSET_NAME", namer_.Variable(field)); code_.SetValue("OFFSET_VALUE", NumToString(field.value.offset)); code_ += "case {{OFFSET_NAME}} = {{OFFSET_VALUE}}"; } @@ -404,12 +459,13 @@ class SwiftGenerator : public BaseGenerator { void GenObjectHeader(const StructDef &struct_def) { GenComment(struct_def.doc_comment); - code_.SetValue("SHORT_STRUCTNAME", Name(struct_def)); - code_.SetValue("STRUCTNAME", NameWrappedInNameSpace(struct_def)); + code_.SetValue("SHORT_STRUCTNAME", namer_.Type(struct_def)); + code_.SetValue("STRUCTNAME", namer_.NamespacedType(struct_def)); code_.SetValue("OBJECTTYPE", struct_def.fixed ? "Struct" : "Table"); code_.SetValue("MUTABLE", struct_def.fixed ? Mutable() : ""); code_ += "{{ACCESS_TYPE}} struct {{STRUCTNAME}}{{MUTABLE}}: FlatBufferObject\\"; + if (!struct_def.fixed) code_ += ", Verifiable\\"; if (!struct_def.fixed && parser_.opts.generate_object_based_api) code_ += ", ObjectAPIPacker\\"; code_ += " {\n"; @@ -421,19 +477,14 @@ class SwiftGenerator : public BaseGenerator { if (!struct_def.fixed) { if (parser_.file_identifier_.length()) { code_.SetValue("FILENAME", parser_.file_identifier_); + code_ += "{{ACCESS_TYPE}} static var id: String { \"{{FILENAME}}\" } "; code_ += "{{ACCESS_TYPE}} static func finish(_ fbb: inout " "FlatBufferBuilder, end: " "Offset, prefix: Bool = false) { fbb.finish(offset: end, " "fileId: " - "\"{{FILENAME}}\", addPrefix: prefix) }"; + "{{STRUCTNAME}}.id, addPrefix: prefix) }"; } - code_ += - "{{ACCESS_TYPE}} static func getRootAs{{SHORT_STRUCTNAME}}(bb: " - "ByteBuffer) -> " - "{{STRUCTNAME}} { return {{STRUCTNAME}}(Table(bb: bb, position: " - "Int32(bb.read(def: UOffset.self, position: bb.reader)) + " - "Int32(bb.reader))) }\n"; code_ += "private init(_ t: Table) { {{ACCESS}} = t }"; } code_ += @@ -448,7 +499,7 @@ class SwiftGenerator : public BaseGenerator { std::vector<std::string> require_fields; std::vector<std::string> create_func_body; std::vector<std::string> create_func_header; - auto should_generate_create = struct_def.fields.vec.size() != 0; + const auto should_generate_create = struct_def.fields.vec.size() != 0; code_.SetValue("NUMBEROFFIELDS", NumToString(struct_def.fields.vec.size())); code_ += @@ -508,14 +559,12 @@ class SwiftGenerator : public BaseGenerator { std::string spacing = ""; if (key_field != nullptr && !struct_def.fixed && struct_def.has_key) { - code_.SetValue("VALUENAME", NameWrappedInNameSpace(struct_def)); - code_.SetValue("SHORT_VALUENAME", Name(struct_def)); code_.SetValue("VOFFSET", NumToString(key_field->value.offset)); - code_ += - "{{ACCESS_TYPE}} static func " - "sortVectorOf{{SHORT_VALUENAME}}(offsets:[Offset], " - "_ fbb: inout FlatBufferBuilder) -> Offset {"; + code_ += "{{ACCESS_TYPE}} static func " + + namer_.Method("sort_vector_of", struct_def) + + "(offsets:[Offset], " + "_ fbb: inout FlatBufferBuilder) -> Offset {"; Indent(); code_ += spacing + "var off = offsets"; code_ += @@ -526,7 +575,7 @@ class SwiftGenerator : public BaseGenerator { code_ += spacing + "return fbb.createVector(ofOffsets: off)"; Outdent(); code_ += "}"; - GenLookup(*key_field); + GenLookup(*key_field, namer_.NamespacedType(struct_def)); } } @@ -536,93 +585,96 @@ class SwiftGenerator : public BaseGenerator { std::string builder_string = ", _ fbb: inout FlatBufferBuilder) { "; auto &create_func_body = *create_body; auto &create_func_header = *create_header; - auto name = Name(field); - auto type = GenType(field.value.type); - auto opt_scalar = + const auto field_field = namer_.Field(field); + const auto field_var = namer_.Variable(field); + const auto type = GenType(field.value.type); + const auto opt_scalar = field.IsOptional() && IsScalar(field.value.type.base_type); - auto nullable_type = opt_scalar ? type + "?" : type; - code_.SetValue("VALUENAME", name); + const auto nullable_type = opt_scalar ? type + "?" : type; + code_.SetValue("FIELDVAR", namer_.Variable(field)); code_.SetValue("VALUETYPE", nullable_type); - code_.SetValue("OFFSET", name); - code_.SetValue("CONSTANT", field.value.constant); + code_.SetValue("OFFSET", namer_.Field(field)); + code_.SetValue("CONSTANT", SwiftConstant(field)); std::string check_if_vector = (IsVector(field.value.type) || IsArray(field.value.type)) ? "VectorOf(" : "("; - auto body = "add" + check_if_vector + name + ": "; + const auto body = "add" + check_if_vector + field_field + ": "; code_ += "{{ACCESS_TYPE}} static func " + body + "\\"; - create_func_body.push_back("{{STRUCTNAME}}." + body + name + ", &fbb)"); + create_func_body.push_back("{{STRUCTNAME}}." + body + field_field + + ", &fbb)"); if (IsScalar(field.value.type.base_type) && !IsBool(field.value.type.base_type)) { - std::string is_enum = IsEnum(field.value.type) ? ".rawValue" : ""; - std::string optional_enum = + const std::string is_enum = IsEnum(field.value.type) ? ".rawValue" : ""; + const std::string optional_enum = IsEnum(field.value.type) ? ("?" + is_enum) : ""; code_ += - "{{VALUETYPE}}" + builder_string + "fbb.add(element: {{VALUENAME}}\\"; + "{{VALUETYPE}}" + builder_string + "fbb.add(element: {{FIELDVAR}}\\"; code_ += field.IsOptional() ? (optional_enum + "\\") : (is_enum + ", def: {{CONSTANT}}\\"); code_ += ", at: {{TABLEOFFSET}}.{{OFFSET}}.p) }"; - auto default_value = + const auto default_value = IsEnum(field.value.type) ? (field.IsOptional() ? "nil" : GenEnumDefaultValue(field)) - : field.value.constant; + : SwiftConstant(field); create_func_header.push_back( - "" + name + ": " + nullable_type + " = " + + "" + field_field + ": " + nullable_type + " = " + (field.IsOptional() ? "nil" : default_value)); return; } if (IsBool(field.value.type.base_type)) { - std::string default_value = - "0" == field.value.constant ? "false" : "true"; + std::string default_value = SwiftConstant(field); code_.SetValue("CONSTANT", default_value); code_.SetValue("VALUETYPE", field.IsOptional() ? "Bool?" : "Bool"); - code_ += "{{VALUETYPE}}" + builder_string + - "fbb.add(element: {{VALUENAME}},\\"; + code_ += + "{{VALUETYPE}}" + builder_string + "fbb.add(element: {{FIELDVAR}},\\"; code_ += field.IsOptional() ? "\\" : " def: {{CONSTANT}},"; code_ += " at: {{TABLEOFFSET}}.{{OFFSET}}.p) }"; create_func_header.push_back( - name + ": " + nullable_type + " = " + + field_var + ": " + nullable_type + " = " + (field.IsOptional() ? "nil" : default_value)); return; } if (IsStruct(field.value.type)) { - auto create_struct = - "guard let {{VALUENAME}} = {{VALUENAME}} else { return };" - " fbb.create(struct: {{VALUENAME}}, position: " + const auto create_struct = + "guard let {{FIELDVAR}} = {{FIELDVAR}} else { return };" + " fbb.create(struct: {{FIELDVAR}}, position: " "{{TABLEOFFSET}}.{{OFFSET}}.p) }"; code_ += type + "?" + builder_string + create_struct; /// Optional hard coded since structs are always optional - create_func_header.push_back(name + ": " + type + "? = nil"); + create_func_header.push_back(field_var + ": " + type + + (field.IsOptional() ? "? = nil" : "")); return; } - auto camel_case_name = - MakeCamel(name, false) + + const auto arg_label = + namer_.Variable(field) + (IsVector(field.value.type) || IsArray(field.value.type) ? "VectorOffset" : "Offset"); - create_func_header.push_back(camel_case_name + " " + name + ": " + - "Offset = Offset()"); - auto reader_type = + create_func_header.push_back(arg_label + " " + field_var + ": " + "Offset" + + (field.IsRequired() ? "" : " = Offset()")); + const auto reader_type = IsStruct(field.value.type) && field.value.type.struct_def->fixed ? "structOffset: {{TABLEOFFSET}}.{{OFFSET}}.p) }" - : "offset: {{VALUENAME}}, at: {{TABLEOFFSET}}.{{OFFSET}}.p) }"; + : "offset: {{FIELDVAR}}, at: {{TABLEOFFSET}}.{{OFFSET}}.p) }"; code_ += "Offset" + builder_string + "fbb.add(" + reader_type; - auto vectortype = field.value.type.VectorType(); + const auto vectortype = field.value.type.VectorType(); if ((vectortype.base_type == BASE_TYPE_STRUCT && field.value.type.struct_def->fixed) && (IsVector(field.value.type) || IsArray(field.value.type))) { - auto field_name = NameWrappedInNameSpace(*vectortype.struct_def); - code_ += "public static func startVectorOf" + MakeCamel(name, true) + + const auto field_name = namer_.NamespacedType(*vectortype.struct_def); + code_ += "{{ACCESS_TYPE}} static func " + + namer_.Method("start_vector_of", field_var) + "(_ size: Int, in builder: inout " "FlatBufferBuilder) {"; Indent(); @@ -637,23 +689,26 @@ class SwiftGenerator : public BaseGenerator { void GenTableReader(const StructDef &struct_def) { for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { - auto &field = **it; + const auto &field = **it; if (field.deprecated) continue; GenTableReaderFields(field); } } void GenTableReaderFields(const FieldDef &field) { - auto offset = NumToString(field.value.offset); - auto name = Name(field); - auto type = GenType(field.value.type); - code_.SetValue("VALUENAME", name); + const auto offset = NumToString(field.value.offset); + const auto field_field = namer_.Field(field); + const auto type = GenType(field.value.type); + code_.SetValue("FIELDVAR", namer_.Variable(field)); + code_.SetValue("FIELDMETHOD", namer_.Method(field)); code_.SetValue("VALUETYPE", type); - code_.SetValue("OFFSET", name); - code_.SetValue("CONSTANT", field.value.constant); - std::string def_Val = field.IsDefault() ? "{{CONSTANT}}" : "nil"; - std::string optional = field.IsOptional() ? "?" : ""; - auto const_string = "return o == 0 ? " + def_Val + " : "; + code_.SetValue("OFFSET", namer_.Constant(field.name)); + code_.SetValue("CONSTANT", SwiftConstant(field)); + bool opt_scalar = + field.IsOptional() && IsScalar(field.value.type.base_type); + std::string def_Val = opt_scalar ? "nil" : "{{CONSTANT}}"; + std::string optional = opt_scalar ? "?" : ""; + const auto const_string = "return o == 0 ? " + def_Val + " : "; GenComment(field.doc_comment); if (IsScalar(field.value.type.base_type) && !IsEnum(field.value.type) && !IsBool(field.value.type.base_type)) { @@ -665,19 +720,20 @@ class SwiftGenerator : public BaseGenerator { if (IsBool(field.value.type.base_type)) { std::string default_value = - "0" == field.value.constant ? "false" : "true"; + field.IsOptional() ? "nil" + : SwiftConstant(field); code_.SetValue("CONSTANT", default_value); code_.SetValue("VALUETYPE", "Bool"); code_ += GenReaderMainBody(optional) + "\\"; - code_.SetValue("VALUETYPE", "Byte"); - code_ += GenOffset() + "return o == 0 ? {{CONSTANT}} : 0 != " + - GenReader("VALUETYPE", "o") + " }"; + code_ += GenOffset() + + "return o == 0 ? {{CONSTANT}} : " + GenReader("VALUETYPE", "o") + + " }"; if (parser_.opts.mutable_buffer) code_ += GenMutate("o", GenOffset()); return; } if (IsEnum(field.value.type)) { - auto default_value = + const auto default_value = field.IsOptional() ? "nil" : GenEnumDefaultValue(field); code_.SetValue("BASEVALUE", GenTypeBasic(field.value.type, false)); code_ += GenReaderMainBody(optional) + "\\"; @@ -688,15 +744,15 @@ class SwiftGenerator : public BaseGenerator { return; } - std::string is_required = field.IsRequired() ? "!" : "?"; - auto required_reader = field.IsRequired() ? "return " : const_string; + const std::string is_required = field.IsRequired() ? "!" : "?"; + const auto required_reader = field.IsRequired() ? "return " : const_string; if (IsStruct(field.value.type) && field.value.type.struct_def->fixed) { code_.SetValue("VALUETYPE", GenType(field.value.type)); code_.SetValue("CONSTANT", "nil"); code_ += GenReaderMainBody(is_required) + GenOffset() + required_reader + "{{ACCESS}}.readBuffer(of: {{VALUETYPE}}.self, at: o) }"; - code_.SetValue("VALUENAME", "mutable" + MakeCamel(name)); + code_.SetValue("FIELDVAR", namer_.Variable("mutable", field_field)); code_.SetValue("VALUETYPE", GenType(field.value.type) + Mutable()); code_.SetValue("CONSTANT", "nil"); code_ += GenReaderMainBody(is_required) + GenOffset() + required_reader + @@ -713,12 +769,12 @@ class SwiftGenerator : public BaseGenerator { break; case BASE_TYPE_STRING: { - auto default_string = "\"" + field.value.constant + "\""; + const auto default_string = "\"" + SwiftConstant(field) + "\""; code_.SetValue("VALUETYPE", GenType(field.value.type)); code_.SetValue("CONSTANT", field.IsDefault() ? default_string : "nil"); code_ += GenReaderMainBody(is_required) + GenOffset() + required_reader + "{{ACCESS}}.string(at: o) }"; - code_ += "{{ACCESS_TYPE}} var {{VALUENAME}}SegmentArray: [UInt8]" + + code_ += "{{ACCESS_TYPE}} var {{FIELDVAR}}SegmentArray: [UInt8]" + is_required + " { return " "{{ACCESS}}.getVector(at: {{TABLEOFFSET}}.{{OFFSET}}.v) }"; @@ -729,7 +785,7 @@ class SwiftGenerator : public BaseGenerator { case BASE_TYPE_UNION: code_.SetValue("CONSTANT", "nil"); code_ += - "{{ACCESS_TYPE}} func {{VALUENAME}}<T: " + "{{ACCESS_TYPE}} func {{FIELDVAR}}<T: " "FlatbuffersInitializable>(type: " "T.Type) -> T" + is_required + " { " + GenOffset() + required_reader + @@ -741,20 +797,22 @@ class SwiftGenerator : public BaseGenerator { void GenTableReaderVectorFields(const FieldDef &field) { std::string const_string = "return o == 0 ? {{CONSTANT}} : "; - auto vectortype = field.value.type.VectorType(); + const auto vectortype = field.value.type.VectorType(); code_.SetValue("SIZE", NumToString(InlineSize(vectortype))); - code_ += "{{ACCESS_TYPE}} var {{VALUENAME}}Count: Int32 { " + GenOffset() + + code_.SetValue("HAS_FIELDVAR", namer_.Variable("has", field)); + code_ += "{{ACCESS_TYPE}} var {{HAS_FIELDVAR}}: Bool { " + GenOffset() + + "return o == 0 ? false : true }"; + code_ += "{{ACCESS_TYPE}} var {{FIELDVAR}}Count: Int32 { " + GenOffset() + "return o == 0 ? 0 : {{ACCESS}}.vector(count: o) }"; - code_.SetValue("CONSTANT", - IsScalar(vectortype.base_type) == true ? "0" : "nil"); - auto nullable = IsScalar(vectortype.base_type) == true ? "" : "?"; - nullable = IsEnum(vectortype) == true ? "?" : nullable; + code_.SetValue("CONSTANT", IsScalar(vectortype.base_type) ? "0" : "nil"); + const auto nullable = + IsScalar(vectortype.base_type) && !IsEnum(vectortype) ? "" : "?"; if (vectortype.base_type != BASE_TYPE_UNION) { code_ += GenArrayMainBody(nullable) + GenOffset() + "\\"; } else { code_ += - "{{ACCESS_TYPE}} func {{VALUENAME}}<T: FlatbuffersInitializable>(at " + "{{ACCESS_TYPE}} func {{FIELDVAR}}<T: FlatbuffersInitializable>(at " "index: " "Int32, type: T.Type) -> T? { " + GenOffset() + "\\"; @@ -773,7 +831,7 @@ class SwiftGenerator : public BaseGenerator { "{{ACCESS}}.directRead(of: {{VALUETYPE}}.self, offset: " "{{ACCESS}}.vector(at: o) + index * {{SIZE}}) }"; code_ += - "{{ACCESS_TYPE}} var {{VALUENAME}}: [{{VALUETYPE}}] { return " + "{{ACCESS_TYPE}} var {{FIELDVAR}}: [{{VALUETYPE}}] { return " "{{ACCESS}}.getVector(at: {{TABLEOFFSET}}.{{OFFSET}}.v) ?? [] }"; if (parser_.opts.mutable_buffer) code_ += GenMutateArray(); return; @@ -784,7 +842,7 @@ class SwiftGenerator : public BaseGenerator { code_ += "{{ACCESS}}.directRead(of: {{VALUETYPE}}.self, offset: " "{{ACCESS}}.vector(at: o) + index * {{SIZE}}) }"; - code_.SetValue("VALUENAME", "mutable" + MakeCamel(Name(field))); + code_.SetValue("FIELDMETHOD", namer_.Method("mutable", field)); code_.SetValue("VALUETYPE", GenType(field.value.type) + Mutable()); code_ += GenArrayMainBody(nullable) + GenOffset() + const_string + GenConstructor("{{ACCESS}}.vector(at: o) + index * {{SIZE}}"); @@ -819,10 +877,10 @@ class SwiftGenerator : public BaseGenerator { code_ += GenConstructor( "{{ACCESS}}.indirect({{ACCESS}}.vector(at: o) + index * " "{{SIZE}})"); - auto &sd = *field.value.type.struct_def; - auto &fields = sd.fields.vec; + const auto &sd = *field.value.type.struct_def; + const auto &fields = sd.fields.vec; for (auto kit = fields.begin(); kit != fields.end(); ++kit) { - auto &key_field = **kit; + const auto &key_field = **kit; if (key_field.key) { GenByKeyFunctions(key_field); break; @@ -831,10 +889,267 @@ class SwiftGenerator : public BaseGenerator { } } + void GenerateCodingKeys(const StructDef &struct_def) { + code_ += "enum CodingKeys: String, CodingKey {"; + Indent(); + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + const auto &field = **it; + if (field.deprecated) continue; + + code_.SetValue("RAWVALUENAME", field.name); + code_.SetValue("FIELDVAR", namer_.Variable(field)); + code_ += "case {{FIELDVAR}} = \"{{RAWVALUENAME}}\""; + } + Outdent(); + code_ += "}"; + } + + void GenerateEncoderUnionBody(const FieldDef &field) { + EnumDef &union_def = *field.value.type.enum_def; + const auto is_vector = field.value.type.base_type == BASE_TYPE_VECTOR || + field.value.type.base_type == BASE_TYPE_ARRAY; + if (field.value.type.base_type == BASE_TYPE_UTYPE || + (is_vector && + field.value.type.VectorType().base_type == BASE_TYPE_UTYPE)) + return; + if (is_vector) { + code_ += + "var enumsEncoder = container.nestedUnkeyedContainer(forKey: " + ".{{FIELDVAR}}Type)"; + code_ += + "var contentEncoder = container.nestedUnkeyedContainer(forKey: " + ".{{FIELDVAR}})"; + code_ += "for index in 0..<{{FIELDVAR}}Count {"; + Indent(); + code_ += "guard let type = {{FIELDVAR}}Type(at: index) else { continue }"; + code_ += "try enumsEncoder.encode(type)"; + code_ += "switch type {"; + for (auto it = union_def.Vals().begin(); it != union_def.Vals().end(); + ++it) { + const auto &ev = **it; + const auto type = GenType(ev.union_type); + code_.SetValue("KEY", namer_.LegacySwiftVariant(ev)); + code_.SetValue("VALUETYPE", type); + if (ev.union_type.base_type == BASE_TYPE_NONE) { continue; } + code_ += "case .{{KEY}}:"; + Indent(); + code_ += "let _v = {{FIELDVAR}}(at: index, type: {{VALUETYPE}}.self)"; + code_ += "try contentEncoder.encode(_v)"; + Outdent(); + } + code_ += "default: break;"; + code_ += "}"; + Outdent(); + code_ += "}"; + return; + } + + code_ += "switch {{FIELDVAR}}Type {"; + for (auto it = union_def.Vals().begin(); it != union_def.Vals().end(); + ++it) { + const auto &ev = **it; + const auto type = GenType(ev.union_type); + code_.SetValue("KEY", namer_.LegacySwiftVariant(ev)); + code_.SetValue("VALUETYPE", type); + if (ev.union_type.base_type == BASE_TYPE_NONE) { continue; } + code_ += "case .{{KEY}}:"; + Indent(); + code_ += "let _v = {{FIELDVAR}}(type: {{VALUETYPE}}.self)"; + code_ += "try container.encodeIfPresent(_v, forKey: .{{FIELDVAR}})"; + Outdent(); + } + code_ += "default: break;"; + code_ += "}"; + } + + void GenerateEncoderBody(const StructDef &struct_def) { + code_ += "var container = encoder.container(keyedBy: CodingKeys.self)"; + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + const auto &field = **it; + if (field.deprecated) continue; + const auto type = field.value.type; + + const auto is_non_union_vector = + (field.value.type.base_type == BASE_TYPE_ARRAY || + field.value.type.base_type == BASE_TYPE_VECTOR) && + field.value.type.VectorType().base_type != BASE_TYPE_UTYPE; + + code_.SetValue("FIELDVAR", namer_.Variable(field)); + code_.SetValue("CONSTANT", SwiftConstant(field)); + bool should_indent = true; + if (is_non_union_vector) { + code_ += "if {{FIELDVAR}}Count > 0 {"; + } else if (IsEnum(type) && !field.IsOptional()) { + code_.SetValue("CONSTANT", GenEnumDefaultValue(field)); + code_ += "if {{FIELDVAR}} != {{CONSTANT}} {"; + } else if (IsFloat(type.base_type) && StringIsFlatbufferNan(field.value.constant)) { + code_ += "if !{{FIELDVAR}}.isNaN {"; + } else if (IsScalar(type.base_type) && !IsEnum(type) && + !IsBool(type.base_type) && !field.IsOptional()) { + code_ += "if {{FIELDVAR}} != {{CONSTANT}} {"; + } else if (IsBool(type.base_type) && !field.IsOptional()) { + code_.SetValue("CONSTANT", SwiftConstant(field)); + code_ += "if {{FIELDVAR}} != {{CONSTANT}} {"; + } else { + should_indent = false; + } + if (should_indent) Indent(); + + if (IsUnion(type) && !IsEnum(type)) { + GenerateEncoderUnionBody(field); + } else if (is_non_union_vector && + (!IsScalar(type.VectorType().base_type) || + IsEnum(type.VectorType()))) { + code_ += + "var contentEncoder = container.nestedUnkeyedContainer(forKey: " + ".{{FIELDVAR}})"; + code_ += "for index in 0..<{{FIELDVAR}}Count {"; + Indent(); + code_ += "guard let type = {{FIELDVAR}}(at: index) else { continue }"; + code_ += "try contentEncoder.encode(type)"; + Outdent(); + code_ += "}"; + } else { + code_ += + "try container.encodeIfPresent({{FIELDVAR}}, forKey: " + ".{{FIELDVAR}})"; + } + if (should_indent) Outdent(); + + if (is_non_union_vector || + (IsScalar(type.base_type) && !field.IsOptional())) { + code_ += "}"; + } + } + } + + void GenerateJSONEncodingAPIs(const StructDef &struct_def) { + code_ += "extension {{STRUCTNAME}}: Encodable {"; + Indent(); + code_ += ""; + if (struct_def.fields.vec.empty() == false) GenerateCodingKeys(struct_def); + + code_ += "{{ACCESS_TYPE}} func encode(to encoder: Encoder) throws {"; + Indent(); + if (struct_def.fields.vec.empty() == false) GenerateEncoderBody(struct_def); + Outdent(); + code_ += "}"; + Outdent(); + code_ += "}"; + code_ += ""; + } + + void GenerateVerifier(const StructDef &struct_def) { + code_ += + "{{ACCESS_TYPE}} static func verify<T>(_ verifier: inout Verifier, at " + "position: " + "Int, of type: T.Type) throws where T: Verifiable {"; + Indent(); + code_ += "var _v = try verifier.visitTable(at: position)"; + for (auto it = struct_def.fields.vec.begin(); + it != struct_def.fields.vec.end(); ++it) { + const auto &field = **it; + if (field.deprecated) continue; + const auto offset = NumToString(field.value.offset); + + code_.SetValue("FIELDVAR", namer_.Variable(field)); + code_.SetValue("VALUETYPE", GenerateVerifierType(field)); + code_.SetValue("OFFSET", namer_.Field(field)); + code_.SetValue("ISREQUIRED", field.IsRequired() ? "true" : "false"); + + if (IsUnion(field.value.type)) { + GenerateUnionTypeVerifier(field); + continue; + } + + code_ += + "try _v.visit(field: {{TABLEOFFSET}}.{{OFFSET}}.p, fieldName: " + "\"{{FIELDVAR}}\", required: {{ISREQUIRED}}, type: " + "{{VALUETYPE}}.self)"; + } + code_ += "_v.finish()"; + Outdent(); + code_ += "}"; + } + + void GenerateUnionTypeVerifier(const FieldDef &field) { + const auto is_vector = + IsVector(field.value.type) || IsArray(field.value.type); + if (field.value.type.base_type == BASE_TYPE_UTYPE || + (is_vector && + field.value.type.VectorType().base_type == BASE_TYPE_UTYPE)) + return; + EnumDef &union_def = *field.value.type.enum_def; + code_.SetValue("VALUETYPE", namer_.NamespacedType(union_def)); + code_.SetValue("FUNCTION_NAME", is_vector ? "visitUnionVector" : "visit"); + code_ += + "try _v.{{FUNCTION_NAME}}(unionKey: {{TABLEOFFSET}}.{{OFFSET}}Type.p, " + "unionField: {{TABLEOFFSET}}.{{OFFSET}}.p, unionKeyName: " + "\"{{FIELDVAR}}Type\", fieldName: \"{{FIELDVAR}}\", required: " + "{{ISREQUIRED}}, completion: { (verifier, key: {{VALUETYPE}}, pos) in"; + Indent(); + code_ += "switch key {"; + for (auto it = union_def.Vals().begin(); it != union_def.Vals().end(); + ++it) { + const auto &ev = **it; + const auto type = GenType(ev.union_type); + code_.SetValue("KEY", namer_.LegacySwiftVariant(ev)); + code_.SetValue("VALUETYPE", type); + code_ += "case .{{KEY}}:"; + Indent(); + if (ev.union_type.base_type == BASE_TYPE_NONE) { + code_ += "break // NOTE - SWIFT doesnt support none"; + } else if (ev.union_type.base_type == BASE_TYPE_STRING) { + code_ += + "try ForwardOffset<String>.verify(&verifier, at: pos, of: " + "String.self)"; + } else { + code_.SetValue("MAINTYPE", ev.union_type.struct_def->fixed + ? type + : "ForwardOffset<" + type + ">"); + code_ += + "try {{MAINTYPE}}.verify(&verifier, at: pos, of: " + "{{VALUETYPE}}.self)"; + } + Outdent(); + } + code_ += "}"; + Outdent(); + code_ += "})"; + } + + std::string GenerateVerifierType(const FieldDef &field) { + const auto type = field.value.type; + const auto is_vector = IsVector(type) || IsArray(type); + + if (is_vector) { + const auto vector_type = field.value.type.VectorType(); + return "ForwardOffset<Vector<" + + GenerateNestedVerifierTypes(vector_type) + ", " + + GenType(vector_type) + ">>"; + } + + return GenerateNestedVerifierTypes(field.value.type); + } + + std::string GenerateNestedVerifierTypes(const Type &type) { + const auto string_type = GenType(type); + + if (IsScalar(type.base_type)) { return string_type; } + + if (IsString(type)) { return "ForwardOffset<" + string_type + ">"; } + + if (type.struct_def && type.struct_def->fixed) { return string_type; } + + return "ForwardOffset<" + string_type + ">"; + } + void GenByKeyFunctions(const FieldDef &key_field) { code_.SetValue("TYPE", GenType(key_field.value.type)); code_ += - "{{ACCESS_TYPE}} func {{VALUENAME}}By(key: {{TYPE}}) -> {{VALUETYPE}}? " + "{{ACCESS_TYPE}} func {{FIELDVAR}}By(key: {{TYPE}}) -> {{VALUETYPE}}? " "{ \\"; code_ += GenOffset() + "return o == 0 ? nil : {{VALUETYPE}}.lookupByKey(vector: " @@ -843,14 +1158,26 @@ class SwiftGenerator : public BaseGenerator { void GenEnum(const EnumDef &enum_def) { if (enum_def.generated) return; - auto is_private_access = enum_def.attributes.Lookup("private"); + const bool is_private_access = parser_.opts.swift_implementation_only || + enum_def.attributes.Lookup("private") != nullptr; + code_.SetValue("ENUM_TYPE", + enum_def.is_union ? "UnionEnum" : "Enum, Verifiable"); code_.SetValue("ACCESS_TYPE", is_private_access ? "internal" : "public"); - code_.SetValue("ENUM_NAME", NameWrappedInNameSpace(enum_def)); + code_.SetValue("ENUM_NAME", namer_.NamespacedType(enum_def)); code_.SetValue("BASE_TYPE", GenTypeBasic(enum_def.underlying_type, false)); GenComment(enum_def.doc_comment); - code_ += "{{ACCESS_TYPE}} enum {{ENUM_NAME}}: {{BASE_TYPE}}, Enum {"; + code_ += + "{{ACCESS_TYPE}} enum {{ENUM_NAME}}: {{BASE_TYPE}}, {{ENUM_TYPE}} {"; Indent(); code_ += "{{ACCESS_TYPE}} typealias T = {{BASE_TYPE}}"; + if (enum_def.is_union) { + code_ += ""; + code_ += "{{ACCESS_TYPE}} init?(value: T) {"; + Indent(); + code_ += "self.init(rawValue: value)"; + Outdent(); + code_ += "}\n"; + } code_ += "{{ACCESS_TYPE}} static var byteSize: Int { return " "MemoryLayout<{{BASE_TYPE}}>.size " @@ -859,17 +1186,20 @@ class SwiftGenerator : public BaseGenerator { "{{ACCESS_TYPE}} var value: {{BASE_TYPE}} { return self.rawValue }"; for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { const auto &ev = **it; - auto name = Name(ev); - code_.SetValue("KEY", name); + code_.SetValue("KEY", namer_.LegacySwiftVariant(ev)); code_.SetValue("VALUE", enum_def.ToString(ev)); GenComment(ev.doc_comment); code_ += "case {{KEY}} = {{VALUE}}"; } - code_ += "\n"; - AddMinOrMaxEnumValue(Name(*enum_def.MaxValue()), "max"); - AddMinOrMaxEnumValue(Name(*enum_def.MinValue()), "min"); + code_ += ""; + AddMinOrMaxEnumValue(namer_.LegacySwiftVariant(*enum_def.MaxValue()), + "max"); + AddMinOrMaxEnumValue(namer_.LegacySwiftVariant(*enum_def.MinValue()), + "min"); Outdent(); code_ += "}\n"; + if (parser_.opts.gen_json_coders) EnumEncoder(enum_def); + code_ += ""; if (parser_.opts.generate_object_based_api && enum_def.is_union) { code_ += "{{ACCESS_TYPE}} struct {{ENUM_NAME}}Union {"; Indent(); @@ -894,20 +1224,40 @@ class SwiftGenerator : public BaseGenerator { } } + void EnumEncoder(const EnumDef &enum_def) { + code_ += "extension {{ENUM_NAME}}: Encodable {"; + Indent(); + code_ += "{{ACCESS_TYPE}} func encode(to encoder: Encoder) throws {"; + Indent(); + code_ += "var container = encoder.singleValueContainer()"; + code_ += "switch self {"; + for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { + const auto &ev = **it; + code_.SetValue("KEY", namer_.LegacySwiftVariant(ev)); + code_.SetValue("RAWKEY", ev.name); + code_ += "case .{{KEY}}: try container.encode(\"{{RAWKEY}}\")"; + } + code_ += "}"; + Outdent(); + code_ += "}"; + Outdent(); + code_ += "}"; + } + // MARK: - Object API - void GenerateObjectAPIExtensionHeader(std::string name) { + void GenerateObjectAPIExtensionHeader(std::string type_name) { code_ += "\n"; - code_ += "{{ACCESS_TYPE}} mutating func unpack() -> " + name + " {"; + code_ += "{{ACCESS_TYPE}} mutating func unpack() -> " + type_name + " {"; Indent(); - code_ += "return " + name + "(&self)"; + code_ += "return " + type_name + "(&self)"; Outdent(); code_ += "}"; code_ += "{{ACCESS_TYPE}} static func pack(_ builder: inout FlatBufferBuilder, " "obj: " "inout " + - name + "?) -> Offset {"; + type_name + "?) -> Offset {"; Indent(); code_ += "guard var obj = obj else { return Offset() }"; code_ += "return pack(&builder, obj: &obj)"; @@ -918,7 +1268,7 @@ class SwiftGenerator : public BaseGenerator { "{{ACCESS_TYPE}} static func pack(_ builder: inout FlatBufferBuilder, " "obj: " "inout " + - name + ") -> Offset {"; + type_name + ") -> Offset {"; Indent(); } @@ -928,40 +1278,39 @@ class SwiftGenerator : public BaseGenerator { Indent(); for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { - auto &field = **it; + const auto &field = **it; if (field.deprecated) continue; - auto name = Name(field); - auto type = GenType(field.value.type); - code_.SetValue("VALUENAME", name); + const auto type = GenType(field.value.type); + code_.SetValue("FIELDVAR", namer_.Variable(field)); if (IsStruct(field.value.type)) { - code_ += "var _v{{VALUENAME}} = _t.{{VALUENAME}}"; - code_ += "_{{VALUENAME}} = _v{{VALUENAME}}.unpack()"; + code_ += "var _v{{FIELDVAR}} = _t.{{FIELDVAR}}"; + code_ += "_{{FIELDVAR}} = _v{{FIELDVAR}}.unpack()"; continue; } std::string is_enum = IsEnum(field.value.type) ? ".value" : ""; - code_ += "_{{VALUENAME}} = _t.{{VALUENAME}}" + is_enum; + code_ += "_{{FIELDVAR}} = _t.{{FIELDVAR}}" + is_enum; } Outdent(); code_ += "}\n"; } void GenObjectAPI(const StructDef &struct_def) { - code_ += "{{ACCESS_TYPE}} class " + ObjectAPIName("{{STRUCTNAME}}") + - ": NativeObject {\n"; + code_ += "{{ACCESS_TYPE}} class " + + namer_.NamespacedObjectType(struct_def) + ": NativeObject {\n"; std::vector<std::string> buffer_constructor; std::vector<std::string> base_constructor; Indent(); for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { - auto &field = **it; + const auto &field = **it; if (field.deprecated) continue; BuildObjectAPIConstructorBody(field, struct_def.fixed, buffer_constructor, base_constructor); } code_ += ""; BuildObjectConstructor(buffer_constructor, - "_ _t: inout " + NameWrappedInNameSpace(struct_def)); + "_ _t: inout " + namer_.NamespacedType(struct_def)); BuildObjectConstructor(base_constructor); if (!struct_def.fixed) code_ += @@ -973,35 +1322,37 @@ class SwiftGenerator : public BaseGenerator { } void GenerateObjectAPITableExtension(const StructDef &struct_def) { - GenerateObjectAPIExtensionHeader(ObjectAPIName("{{STRUCTNAME}}")); + GenerateObjectAPIExtensionHeader(namer_.NamespacedObjectType(struct_def)); std::vector<std::string> unpack_body; std::string builder = ", &builder)"; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { - auto &field = **it; + const auto &field = **it; if (field.deprecated) continue; - auto name = Name(field); - auto type = GenType(field.value.type); + const auto field_var = namer_.Variable(field); + const auto field_field = namer_.Field(field); + const auto field_method = namer_.Method(field); + const auto type = GenType(field.value.type); std::string check_if_vector = (IsVector(field.value.type) || IsArray(field.value.type)) ? "VectorOf(" : "("; - std::string body = "add" + check_if_vector + name + ": "; + std::string body = "add" + check_if_vector + field_method + ": "; switch (field.value.type.base_type) { case BASE_TYPE_ARRAY: FLATBUFFERS_FALLTHROUGH(); case BASE_TYPE_VECTOR: { - GenerateVectorObjectAPITableExtension(field, name, type); - unpack_body.push_back("{{STRUCTNAME}}." + body + "__" + name + + GenerateVectorObjectAPITableExtension(field); + unpack_body.push_back("{{STRUCTNAME}}." + body + "__" + field_var + builder); break; } case BASE_TYPE_UNION: { - code_ += "let __" + name + " = obj." + name + + code_ += "let __" + field_var + " = obj." + field_var + "?.pack(builder: &builder) ?? Offset()"; - unpack_body.push_back("if let o = obj." + name + "?.type {"); - unpack_body.push_back(" {{STRUCTNAME}}.add(" + name + "Type: o" + - builder); - unpack_body.push_back(" {{STRUCTNAME}}." + body + "__" + name + + unpack_body.push_back("if let o = obj." + field_var + "?.type {"); + unpack_body.push_back(" {{STRUCTNAME}}.add(" + field_var + + "Type: o" + builder); + unpack_body.push_back(" {{STRUCTNAME}}." + body + "__" + field_var + builder); unpack_body.push_back("}\n"); break; @@ -1015,31 +1366,31 @@ class SwiftGenerator : public BaseGenerator { GenerateStructArgs(*field.value.type.struct_def, &code, "", "", "$0", true); code = code.substr(0, code.size() - 2); - unpack_body.push_back("{{STRUCTNAME}}." + body + "obj." + name + - builder); + unpack_body.push_back("{{STRUCTNAME}}." + body + "obj." + + field_field + builder); } else { - code_ += "let __" + name + " = " + type + - ".pack(&builder, obj: &obj." + name + ")"; - unpack_body.push_back("{{STRUCTNAME}}." + body + "__" + name + + code_ += "let __" + field_var + " = " + type + + ".pack(&builder, obj: &obj." + field_field + ")"; + unpack_body.push_back("{{STRUCTNAME}}." + body + "__" + field_var + builder); } break; } case BASE_TYPE_STRING: { - unpack_body.push_back("{{STRUCTNAME}}." + body + "__" + name + + unpack_body.push_back("{{STRUCTNAME}}." + body + "__" + field_var + builder); if (field.IsRequired()) { - code_ += - "let __" + name + " = builder.create(string: obj." + name + ")"; + code_ += "let __" + field_var + " = builder.create(string: obj." + + field_field + ")"; } else { - BuildingOptionalObjects(name, "builder.create(string: s)"); + BuildingOptionalObjects(field_field, "builder.create(string: s)"); } break; } case BASE_TYPE_UTYPE: break; default: - unpack_body.push_back("{{STRUCTNAME}}." + body + "obj." + name + - builder); + unpack_body.push_back("{{STRUCTNAME}}." + body + "obj." + + field_field + builder); } } code_ += "let __root = {{STRUCTNAME}}.start{{SHORT_STRUCTNAME}}(&builder)"; @@ -1052,78 +1403,79 @@ class SwiftGenerator : public BaseGenerator { code_ += "}"; } - void GenerateVectorObjectAPITableExtension(const FieldDef &field, - const std::string &name, - const std::string &type) { - auto vectortype = field.value.type.VectorType(); + void GenerateVectorObjectAPITableExtension(const FieldDef &field_def) { + const Type &field_type = field_def.value.type; + const auto type = GenType(field_type); + const auto var = namer_.Variable(field_def); + const auto field = namer_.Field(field_def); + + const auto vectortype = field_type.VectorType(); switch (vectortype.base_type) { case BASE_TYPE_UNION: { - code_ += "var __" + name + "__: [Offset] = []"; - code_ += "for i in obj." + name + " {"; + code_ += "var __" + var + "__: [Offset] = []"; + code_ += "for i in obj." + var + " {"; Indent(); code_ += "guard let off = i?.pack(builder: &builder) else { continue }"; - code_ += "__" + name + "__.append(off)"; + code_ += "__" + var + "__.append(off)"; Outdent(); code_ += "}"; - code_ += "let __" + name + " = builder.createVector(ofOffsets: __" + - name + "__)"; - code_ += "let __" + name + "Type = builder.createVector(obj." + name + + code_ += "let __" + var + " = builder.createVector(ofOffsets: __" + + var + "__)"; + code_ += "let __" + var + "Type = builder.createVector(obj." + field + ".compactMap { $0?.type })"; break; } case BASE_TYPE_UTYPE: break; case BASE_TYPE_STRUCT: { - if (field.value.type.struct_def && - !field.value.type.struct_def->fixed) { - code_ += "var __" + name + "__: [Offset] = []"; - code_ += "for var i in obj." + name + " {"; + if (field_type.struct_def && !field_type.struct_def->fixed) { + code_ += "var __" + var + "__: [Offset] = []"; + code_ += "for var i in obj." + var + " {"; Indent(); code_ += - "__" + name + "__.append(" + type + ".pack(&builder, obj: &i))"; + "__" + var + "__.append(" + type + ".pack(&builder, obj: &i))"; Outdent(); code_ += "}"; - code_ += "let __" + name + " = builder.createVector(ofOffsets: __" + - name + "__)"; + code_ += "let __" + var + " = builder.createVector(ofOffsets: __" + + var + "__)"; } else { - code_ += "{{STRUCTNAME}}.startVectorOf" + MakeCamel(name, true) + - "(obj." + name + ".count, in: &builder)"; + code_ += "{{STRUCTNAME}}." + namer_.Method("start_vector_of", var) + + "(obj." + field + ".count, in: &builder)"; std::string code; - GenerateStructArgs(*field.value.type.struct_def, &code, "", "", "_o", - true); + GenerateStructArgs(*field_type.struct_def, &code, "", "", "_o", true); code = code.substr(0, code.size() - 2); - code_ += "for i in obj." + name + " {"; + code_ += "for i in obj." + field + " {"; Indent(); code_ += "guard let _o = i else { continue }"; code_ += "builder.create(struct: _o)"; Outdent(); code_ += "}"; - code_ += "let __" + name + " = builder.endVector(len: obj." + name + + code_ += "let __" + var + " = builder.endVector(len: obj." + field + ".count)"; } break; } case BASE_TYPE_STRING: { - code_ += "let __" + name + " = builder.createVector(ofStrings: obj." + - name + ".compactMap({ $0 }) )"; + code_ += "let __" + var + " = builder.createVector(ofStrings: obj." + + var + ".compactMap({ $0 }) )"; break; } default: { - code_ += "let __" + name + " = builder.createVector(obj." + name + ")"; + code_ += "let __" + var + " = builder.createVector(obj." + field + ")"; break; } } } - void BuildingOptionalObjects(const std::string &name, + void BuildingOptionalObjects(const std::string &var, const std::string &body_front) { - code_ += "let __" + name + ": Offset"; - code_ += "if let s = obj." + name + " {"; + code_ += "let __" + var + ": Offset"; + code_ += "if let s = obj." + var + " {"; Indent(); - code_ += "__" + name + " = " + body_front; + code_ += "__" + var + " = " + body_front; Outdent(); code_ += "} else {"; Indent(); - code_ += "__" + name + " = Offset()"; + code_ += "__" + var + " = Offset()"; Outdent(); code_ += "}"; code_ += ""; @@ -1143,120 +1495,123 @@ class SwiftGenerator : public BaseGenerator { const FieldDef &field, bool is_fixed, std::vector<std::string> &buffer_constructor, std::vector<std::string> &base_constructor) { - auto name = Name(field); - auto type = GenType(field.value.type); - code_.SetValue("VALUENAME", name); + const auto field_field = namer_.Field(field); + const auto field_var = namer_.Variable(field); + const auto type = GenType(field.value.type); + code_.SetValue("FIELDVAR", field_field); code_.SetValue("VALUETYPE", type); std::string is_required = field.IsRequired() ? "" : "?"; switch (field.value.type.base_type) { case BASE_TYPE_STRUCT: { - type = GenType(field.value.type, true); - code_.SetValue("VALUETYPE", type); - auto optional = + const auto objtype = GenType(field.value.type, true); + code_.SetValue("VALUETYPE", objtype); + const auto optional = (field.value.type.struct_def && field.value.type.struct_def->fixed); std::string question_mark = (field.IsRequired() || (optional && is_fixed) ? "" : "?"); code_ += - "{{ACCESS_TYPE}} var {{VALUENAME}}: {{VALUETYPE}}" + question_mark; - base_constructor.push_back("" + name + " = " + type + "()"); + "{{ACCESS_TYPE}} var {{FIELDVAR}}: {{VALUETYPE}}" + question_mark; + base_constructor.push_back("" + field_var + " = " + objtype + "()"); if (field.value.type.struct_def->fixed) { - buffer_constructor.push_back("" + name + " = _t." + name); + buffer_constructor.push_back("" + field_var + " = _t." + field_field); } else { - buffer_constructor.push_back("var __" + name + " = _t." + name); + buffer_constructor.push_back("var __" + field_var + " = _t." + + field_field); buffer_constructor.push_back( - "" + name + " = __" + name + + "" + field_var + " = __" + field_var + (field.IsRequired() ? "!" : question_mark) + ".unpack()"); } break; } case BASE_TYPE_ARRAY: FLATBUFFERS_FALLTHROUGH(); case BASE_TYPE_VECTOR: { - BuildObjectAPIConstructorBodyVectors(field, name, buffer_constructor, + BuildObjectAPIConstructorBodyVectors(field, buffer_constructor, base_constructor, " "); break; } case BASE_TYPE_STRING: { - code_ += "{{ACCESS_TYPE}} var {{VALUENAME}}: String" + is_required; - buffer_constructor.push_back(name + " = _t." + name); + code_ += "{{ACCESS_TYPE}} var {{FIELDVAR}}: String" + is_required; + buffer_constructor.push_back(field_var + " = _t." + field_field); if (field.IsRequired()) { std::string default_value = - field.IsDefault() ? field.value.constant : ""; - base_constructor.push_back(name + " = \"" + default_value + "\""); + field.IsDefault() ? SwiftConstant(field) : ""; + base_constructor.push_back(field_var + " = \"" + default_value + + "\""); break; } if (field.IsDefault() && !field.IsRequired()) { - std::string value = field.IsDefault() ? field.value.constant : "nil"; - base_constructor.push_back(name + " = \"" + value + "\""); + std::string value = field.IsDefault() ? SwiftConstant(field) : "nil"; + base_constructor.push_back(field_var + " = \"" + value + "\""); } break; } case BASE_TYPE_UTYPE: break; case BASE_TYPE_UNION: { - BuildUnionEnumSwitchCase(*field.value.type.enum_def, name, + BuildUnionEnumSwitchCase(*field.value.type.enum_def, field_var, buffer_constructor); break; } default: { - buffer_constructor.push_back(name + " = _t." + name); + buffer_constructor.push_back(field_var + " = _t." + field_field); std::string nullable = field.IsOptional() ? "?" : ""; if (IsScalar(field.value.type.base_type) && !IsBool(field.value.type.base_type) && !IsEnum(field.value.type)) { - code_ += - "{{ACCESS_TYPE}} var {{VALUENAME}}: {{VALUETYPE}}" + nullable; + code_ += "{{ACCESS_TYPE}} var {{FIELDVAR}}: {{VALUETYPE}}" + nullable; if (!field.IsOptional()) - base_constructor.push_back(name + " = " + field.value.constant); + base_constructor.push_back(field_var + " = " + + SwiftConstant(field)); break; } if (IsEnum(field.value.type)) { - auto default_value = IsEnum(field.value.type) - ? GenEnumDefaultValue(field) - : field.value.constant; - code_ += "{{ACCESS_TYPE}} var {{VALUENAME}}: {{VALUETYPE}}"; - base_constructor.push_back(name + " = " + default_value); + const auto default_value = IsEnum(field.value.type) + ? GenEnumDefaultValue(field) + : SwiftConstant(field); + code_ += "{{ACCESS_TYPE}} var {{FIELDVAR}}: {{VALUETYPE}}"; + base_constructor.push_back(field_var + " = " + default_value); break; } if (IsBool(field.value.type.base_type)) { - code_ += "{{ACCESS_TYPE}} var {{VALUENAME}}: Bool" + nullable; - std::string default_value = - "0" == field.value.constant ? "false" : "true"; + code_ += "{{ACCESS_TYPE}} var {{FIELDVAR}}: Bool" + nullable; if (!field.IsOptional()) - base_constructor.push_back(name + " = " + default_value); + base_constructor.push_back(field_var + " = " + SwiftConstant(field)); } } } } void BuildObjectAPIConstructorBodyVectors( - const FieldDef &field, const std::string &name, - std::vector<std::string> &buffer_constructor, + const FieldDef &field, std::vector<std::string> &buffer_constructor, std::vector<std::string> &base_constructor, const std::string &indentation) { - auto vectortype = field.value.type.VectorType(); + const auto vectortype = field.value.type.VectorType(); + const auto field_var = namer_.Field(field); + const auto field_field = namer_.Field(field); if (vectortype.base_type != BASE_TYPE_UTYPE) { - buffer_constructor.push_back(name + " = []"); - buffer_constructor.push_back("for index in 0..<_t." + name + "Count {"); - base_constructor.push_back(name + " = []"); + buffer_constructor.push_back(field_var + " = []"); + buffer_constructor.push_back("for index in 0..<_t." + field_field + + "Count {"); + base_constructor.push_back(field_var + " = []"); } switch (vectortype.base_type) { case BASE_TYPE_STRUCT: { code_.SetValue("VALUETYPE", GenType(vectortype, true)); - code_ += "{{ACCESS_TYPE}} var {{VALUENAME}}: [{{VALUETYPE}}?]"; + code_ += "{{ACCESS_TYPE}} var {{FIELDVAR}}: [{{VALUETYPE}}?]"; if (!vectortype.struct_def->fixed) { - buffer_constructor.push_back(indentation + "var __v_ = _t." + name + - "(at: index)"); - buffer_constructor.push_back(indentation + name + + buffer_constructor.push_back(indentation + "var __v_ = _t." + + field_field + "(at: index)"); + buffer_constructor.push_back(indentation + field_var + ".append(__v_?.unpack())"); } else { - buffer_constructor.push_back(indentation + name + ".append(_t." + - name + "(at: index))"); + buffer_constructor.push_back(indentation + field_var + ".append(_t." + + field_var + "(at: index))"); } break; } @@ -1265,7 +1620,7 @@ class SwiftGenerator : public BaseGenerator { break; } case BASE_TYPE_UNION: { - BuildUnionEnumSwitchCase(*field.value.type.enum_def, name, + BuildUnionEnumSwitchCase(*field.value.type.enum_def, field_var, buffer_constructor, indentation, true); break; } @@ -1274,18 +1629,18 @@ class SwiftGenerator : public BaseGenerator { code_.SetValue( "VALUETYPE", (IsString(vectortype) ? "String?" : GenType(vectortype))); - code_ += "{{ACCESS_TYPE}} var {{VALUENAME}}: [{{VALUETYPE}}]"; + code_ += "{{ACCESS_TYPE}} var {{FIELDVAR}}: [{{VALUETYPE}}]"; if (IsEnum(vectortype) && vectortype.base_type != BASE_TYPE_UNION) { - auto default_value = IsEnum(field.value.type) - ? GenEnumDefaultValue(field) - : field.value.constant; - buffer_constructor.push_back(indentation + name + ".append(_t." + - name + "(at: index)!)"); + const auto default_value = IsEnum(field.value.type) + ? GenEnumDefaultValue(field) + : SwiftConstant(field); + buffer_constructor.push_back(indentation + field_var + ".append(_t." + + field_field + "(at: index)!)"); break; } - buffer_constructor.push_back(indentation + name + ".append(_t." + name + - "(at: index))"); + buffer_constructor.push_back(indentation + field_var + ".append(_t." + + field_field + "(at: index))"); break; } } @@ -1293,19 +1648,17 @@ class SwiftGenerator : public BaseGenerator { buffer_constructor.push_back("}"); } - void BuildUnionEnumSwitchCaseWritter(const EnumDef &ev) { - auto field_name = Name(ev); - code_.SetValue("VALUETYPE", field_name); + void BuildUnionEnumSwitchCaseWritter(const EnumDef &ed) { code_ += "switch type {"; - for (auto it = ev.Vals().begin(); it < ev.Vals().end(); ++it) { - auto field = **it; - auto ev_name = Name(field); - auto type = GenType(field.union_type); - auto is_struct = IsStruct(field.union_type) ? type + Mutable() : type; - if (field.union_type.base_type == BASE_TYPE_NONE) { continue; } - code_ += "case ." + ev_name + ":"; + for (auto it = ed.Vals().begin(); it < ed.Vals().end(); ++it) { + const auto ev = **it; + const auto variant = namer_.LegacySwiftVariant(ev); + const auto type = GenType(ev.union_type); + const auto is_struct = IsStruct(ev.union_type) ? type + Mutable() : type; + if (ev.union_type.base_type == BASE_TYPE_NONE) { continue; } + code_ += "case ." + variant + ":"; Indent(); - code_ += "var __obj = value as? " + GenType(field.union_type, true); + code_ += "var __obj = value as? " + GenType(ev.union_type, true); code_ += "return " + is_struct + ".pack(&builder, obj: &__obj)"; Outdent(); } @@ -1313,34 +1666,34 @@ class SwiftGenerator : public BaseGenerator { code_ += "}"; } - void BuildUnionEnumSwitchCase(const EnumDef &ev, const std::string &name, + void BuildUnionEnumSwitchCase(const EnumDef &ed, const std::string &field, std::vector<std::string> &buffer_constructor, const std::string &indentation = "", const bool is_vector = false) { - auto field_name = NameWrappedInNameSpace(ev); - code_.SetValue("VALUETYPE", field_name); - code_ += "{{ACCESS_TYPE}} var {{VALUENAME}}: \\"; + const auto ns_type = namer_.NamespacedType(ed); + code_.SetValue("VALUETYPE", ns_type); + code_ += "{{ACCESS_TYPE}} var {{FIELDVAR}}: \\"; code_ += is_vector ? "[{{VALUETYPE}}Union?]" : "{{VALUETYPE}}Union?"; - auto vector_reader = is_vector ? "(at: index" : ""; - buffer_constructor.push_back(indentation + "switch _t." + name + "Type" + + const auto vector_reader = is_vector ? "(at: index" : ""; + buffer_constructor.push_back(indentation + "switch _t." + field + "Type" + vector_reader + (is_vector ? ")" : "") + " {"); - for (auto it = ev.Vals().begin(); it < ev.Vals().end(); ++it) { - auto field = **it; - auto ev_name = Name(field); - if (field.union_type.base_type == BASE_TYPE_NONE) { continue; } - auto type = IsStruct(field.union_type) - ? GenType(field.union_type) + Mutable() - : GenType(field.union_type); - buffer_constructor.push_back(indentation + "case ." + ev_name + ":"); + for (auto it = ed.Vals().begin(); it < ed.Vals().end(); ++it) { + const auto ev = **it; + const auto variant = namer_.LegacySwiftVariant(ev); + if (ev.union_type.base_type == BASE_TYPE_NONE) { continue; } + const auto type = IsStruct(ev.union_type) + ? GenType(ev.union_type) + Mutable() + : GenType(ev.union_type); + buffer_constructor.push_back(indentation + "case ." + variant + ":"); buffer_constructor.push_back( - indentation + " var _v = _t." + name + (is_vector ? "" : "(") + + indentation + " var _v = _t." + field + (is_vector ? "" : "(") + vector_reader + (is_vector ? ", " : "") + "type: " + type + ".self)"); - auto constructor = - field_name + "Union(_v?.unpack(), type: ." + ev_name + ")"; + const auto constructor = + ns_type + "Union(_v?.unpack(), type: ." + variant + ")"; buffer_constructor.push_back( - indentation + " " + name + + indentation + " " + field + (is_vector ? ".append(" + constructor + ")" : " = " + constructor)); } buffer_constructor.push_back(indentation + "default: break"); @@ -1348,13 +1701,14 @@ class SwiftGenerator : public BaseGenerator { } void AddMinOrMaxEnumValue(const std::string &str, const std::string &type) { - auto current_value = str; + const auto current_value = str; code_.SetValue(type, current_value); code_ += "{{ACCESS_TYPE}} static var " + type + ": {{ENUM_NAME}} { return .{{" + type + "}} }"; } - void GenLookup(const FieldDef &key_field) { + void GenLookup(const FieldDef &key_field, const std::string &struct_type) { + code_.SetValue("STRUCTTYPE", struct_type); code_.SetValue("OFFSET", NumToString(key_field.value.offset)); std::string offset_reader = "Table.offset(Int32(fbb.capacity) - tableOffset, vOffset: {{OFFSET}}, " @@ -1364,7 +1718,7 @@ class SwiftGenerator : public BaseGenerator { code_ += "fileprivate static func lookupByKey(vector: Int32, key: {{TYPE}}, " "fbb: " - "ByteBuffer) -> {{VALUENAME}}? {"; + "ByteBuffer) -> {{STRUCTTYPE}}? {"; Indent(); if (IsString(key_field.value.type)) code_ += "let key = key.utf8.map { $0 }"; @@ -1394,7 +1748,7 @@ class SwiftGenerator : public BaseGenerator { Outdent(); code_ += "} else {"; Indent(); - code_ += "return {{VALUENAME}}(fbb, o: tableOffset)"; + code_ += "return {{STRUCTTYPE}}(fbb, o: tableOffset)"; Outdent(); code_ += "}"; Outdent(); @@ -1408,7 +1762,7 @@ class SwiftGenerator : public BaseGenerator { if (field.padding) { for (int i = 0; i < 4; i++) { if (static_cast<int>(field.padding) & (1 << i)) { - auto bits = (1 << i) * 8; + const auto bits = (1 << i) * 8; code_ += "private let padding" + NumToString((*id)++) + "__: UInt" + NumToString(bits) + " = 0"; } @@ -1430,8 +1784,7 @@ class SwiftGenerator : public BaseGenerator { } std::string GenReaderMainBody(const std::string &optional = "") { - return "{{ACCESS_TYPE}} var {{VALUENAME}}: {{VALUETYPE}}" + optional + - " { "; + return "{{ACCESS_TYPE}} var {{FIELDVAR}}: {{VALUETYPE}}" + optional + " { "; } std::string GenReader(const std::string &type, @@ -1445,52 +1798,58 @@ class SwiftGenerator : public BaseGenerator { std::string GenMutate(const std::string &offset, const std::string &get_offset, bool isRaw = false) { - return "@discardableResult {{ACCESS_TYPE}} func mutate({{VALUENAME}}: " + return "@discardableResult {{ACCESS_TYPE}} func mutate({{FIELDVAR}}: " "{{VALUETYPE}}) -> Bool {" + - get_offset + " return {{ACCESS}}.mutate({{VALUENAME}}" + + get_offset + " return {{ACCESS}}.mutate({{FIELDVAR}}" + (isRaw ? ".rawValue" : "") + ", index: " + offset + ") }"; } std::string GenMutateArray() { - return "{{ACCESS_TYPE}} func mutate({{VALUENAME}}: {{VALUETYPE}}, at " - "index: " - "Int32) -> Bool { " + + return "{{ACCESS_TYPE}} func mutate({{FIELDVAR}}: {{VALUETYPE}}, at " + "index: Int32) -> Bool { " + GenOffset() + - "return {{ACCESS}}.directMutate({{VALUENAME}}, index: " + "return {{ACCESS}}.directMutate({{FIELDVAR}}, index: " "{{ACCESS}}.vector(at: o) + index * {{SIZE}}) }"; } std::string GenEnumDefaultValue(const FieldDef &field) { - auto &value = field.value; + const auto &value = field.value; FLATBUFFERS_ASSERT(value.type.enum_def); - auto &enum_def = *value.type.enum_def; + const auto &enum_def = *value.type.enum_def; // Vector of enum defaults are always "[]" which never works. const std::string constant = IsVector(value.type) ? "0" : value.constant; - auto enum_val = enum_def.FindByValue(constant); - std::string name; + const auto enum_val = enum_def.FindByValue(constant); if (enum_val) { - name = Name(*enum_val); + return "." + namer_.LegacySwiftVariant(*enum_val); } else { const auto &ev = **enum_def.Vals().begin(); - name = Name(ev); + return "." + namer_.LegacySwiftVariant(ev); } - return "." + name; } + std::string SwiftConstant(const FieldDef& field) { + const auto default_value = + StringIsFlatbufferNan(field.value.constant) ? ".nan" : + StringIsFlatbufferPositiveInfinity(field.value.constant) ? ".infinity" : + StringIsFlatbufferNegativeInfinity(field.value.constant) ? "-.infinity" : + IsBool(field.value.type.base_type) ? ("0" == field.value.constant ? "false" : "true") : + field.value.constant; + return default_value; + } + std::string GenEnumConstructor(const std::string &at) { return "{{VALUETYPE}}(rawValue: " + GenReader("BASEVALUE", at) + ") "; } std::string ValidateFunc() { - return "static func validateVersion() { FlatBuffersVersion_2_0_0() }"; + return "static func validateVersion() { FlatBuffersVersion_23_5_9() }"; } std::string GenType(const Type &type, const bool should_consider_suffix = false) const { - return IsScalar(type.base_type) - ? GenTypeBasic(type) - : (IsArray(type) ? GenType(type.VectorType()) - : GenTypePointer(type, should_consider_suffix)); + return IsScalar(type.base_type) ? GenTypeBasic(type) + : IsArray(type) ? GenType(type.VectorType()) + : GenTypePointer(type, should_consider_suffix); } std::string GenTypePointer(const Type &type, @@ -1499,12 +1858,11 @@ class SwiftGenerator : public BaseGenerator { case BASE_TYPE_STRING: return "String"; case BASE_TYPE_VECTOR: return GenType(type.VectorType()); case BASE_TYPE_STRUCT: { - auto &struct_ = *type.struct_def; - if (should_consider_suffix && !struct_.fixed) { - return WrapInNameSpace(struct_.defined_namespace, - ObjectAPIName(Name(struct_))); + const auto &sd = *type.struct_def; + if (should_consider_suffix && !sd.fixed) { + return namer_.NamespacedObjectType(sd); } - return WrapInNameSpace(struct_.defined_namespace, Name(struct_)); + return namer_.NamespacedType(sd); } case BASE_TYPE_UNION: default: return "FlatbuffersInitializable"; @@ -1515,56 +1873,30 @@ class SwiftGenerator : public BaseGenerator { return GenTypeBasic(type, true); } - std::string ObjectAPIName(const std::string &name) const { - return parser_.opts.object_prefix + name + parser_.opts.object_suffix; - } - void Indent() { code_.IncrementIdentLevel(); } void Outdent() { code_.DecrementIdentLevel(); } - std::string NameWrappedInNameSpace(const EnumDef &enum_def) const { - return WrapInNameSpace(enum_def.defined_namespace, Name(enum_def)); - } - - std::string NameWrappedInNameSpace(const StructDef &struct_def) const { - return WrapInNameSpace(struct_def.defined_namespace, Name(struct_def)); - } - std::string GenTypeBasic(const Type &type, bool can_override) const { // clang-format off static const char * const swift_type[] = { #define FLATBUFFERS_TD(ENUM, IDLTYPE, \ - CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE, STYPE) \ + CTYPE, JTYPE, GTYPE, NTYPE, PTYPE, RTYPE, KTYPE, STYPE, ...) \ #STYPE, FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD }; // clang-format on if (can_override) { - if (type.enum_def) return NameWrappedInNameSpace(*type.enum_def); + if (type.enum_def) return namer_.NamespacedType(*type.enum_def); if (type.base_type == BASE_TYPE_BOOL) return "Bool"; } return swift_type[static_cast<int>(type.base_type)]; } - std::string EscapeKeyword(const std::string &name) const { - return keywords_.find(name) == keywords_.end() ? name : name + "_"; - } - std::string Mutable() const { return "_Mutable"; } - std::string Name(const EnumVal &ev) const { - auto name = ev.name; - if (isupper(name.front())) { - std::transform(name.begin(), name.end(), name.begin(), CharToLower); - } - return EscapeKeyword(MakeCamel(name, false)); - } - - std::string Name(const Definition &def) const { - return EscapeKeyword(MakeCamel(def.name, false)); - } + IdlNamer namer_; }; } // namespace swift bool GenerateSwift(const Parser &parser, const std::string &path, @@ -1572,4 +1904,60 @@ bool GenerateSwift(const Parser &parser, const std::string &path, swift::SwiftGenerator generator(parser, path, file_name); return generator.generate(); } + +namespace { + +class SwiftCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateSwift(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateSwiftGRPC(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + (void)parser; + (void)path; + (void)filename; + (void)output; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kSwift; } + + std::string LanguageName() const override { return "Swift"; } +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewSwiftCodeGenerator() { + return std::unique_ptr<SwiftCodeGenerator>(new SwiftCodeGenerator()); +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_swift.h b/contrib/libs/flatbuffers/src/idl_gen_swift.h new file mode 100644 index 00000000000..4fd8977d40b --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_swift.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_SWIFT_H_ +#define FLATBUFFERS_IDL_GEN_SWIFT_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Cpp code generator. +std::unique_ptr<CodeGenerator> NewSwiftCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_SWIFT_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_text.cpp b/contrib/libs/flatbuffers/src/idl_gen_text.cpp index 903c41ecdb5..59f68444133 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_text.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_text.cpp @@ -15,7 +15,12 @@ */ // independent from idl_parser, since this code is not needed for most clients +#include "idl_gen_text.h" +#include <algorithm> + +#include "flatbuffers/base.h" +#include "flatbuffers/code_generator.h" #include "flatbuffers/flatbuffers.h" #include "flatbuffers/flexbuffers.h" #include "flatbuffers/idl.h" @@ -50,10 +55,10 @@ struct JsonPrinter { // for a single FlatBuffer value into JSON format. // The general case for scalars: template<typename T> - bool PrintScalar(T val, const Type &type, int /*indent*/) { + void PrintScalar(T val, const Type &type, int /*indent*/) { if (IsBool(type.base_type)) { text += val != 0 ? "true" : "false"; - return true; // done + return; // done } if (opts.output_enum_identifiers && type.enum_def) { @@ -62,7 +67,7 @@ struct JsonPrinter { text += '\"'; text += ev->name; text += '\"'; - return true; // done + return; // done } else if (val && enum_def.attributes.Lookup("bit_flags")) { const auto entry_len = text.length(); const auto u64 = static_cast<uint64_t>(val); @@ -80,7 +85,7 @@ struct JsonPrinter { // Don't slice if (u64 != mask) if (mask && (u64 == mask)) { text[text.length() - 1] = '\"'; - return true; // done + return; // done } text.resize(entry_len); // restore } @@ -88,7 +93,7 @@ struct JsonPrinter { } text += NumToString(val); - return true; + return; } void AddComma() { @@ -97,36 +102,36 @@ struct JsonPrinter { // Print a vector or an array of JSON values, comma seperated, wrapped in // "[]". - template<typename Container> - bool PrintContainer(PrintScalarTag, const Container &c, size_t size, + template<typename Container, typename SizeT = typename Container::size_type> + const char *PrintContainer(PrintScalarTag, const Container &c, SizeT size, const Type &type, int indent, const uint8_t *) { const auto elem_indent = indent + Indent(); text += '['; AddNewLine(); - for (uoffset_t i = 0; i < size; i++) { + for (SizeT i = 0; i < size; i++) { if (i) { AddComma(); AddNewLine(); } AddIndent(elem_indent); - if (!PrintScalar(c[i], type, elem_indent)) { return false; } + PrintScalar(c[i], type, elem_indent); } AddNewLine(); AddIndent(indent); text += ']'; - return true; + return nullptr; } // Print a vector or an array of JSON values, comma seperated, wrapped in // "[]". - template<typename Container> - bool PrintContainer(PrintPointerTag, const Container &c, size_t size, + template<typename Container, typename SizeT = typename Container::size_type> + const char *PrintContainer(PrintPointerTag, const Container &c, SizeT size, const Type &type, int indent, const uint8_t *prev_val) { const auto is_struct = IsStruct(type); const auto elem_indent = indent + Indent(); text += '['; AddNewLine(); - for (uoffset_t i = 0; i < size; i++) { + for (SizeT i = 0; i < size; i++) { if (i) { AddComma(); AddNewLine(); @@ -135,21 +140,20 @@ struct JsonPrinter { auto ptr = is_struct ? reinterpret_cast<const void *>( c.Data() + type.struct_def->bytesize * i) : c[i]; - if (!PrintOffset(ptr, type, elem_indent, prev_val, - static_cast<soffset_t>(i))) { - return false; - } + auto err = PrintOffset(ptr, type, elem_indent, prev_val, + static_cast<soffset_t>(i)); + if (err) return err; } AddNewLine(); AddIndent(indent); text += ']'; - return true; + return nullptr; } - template<typename T> - bool PrintVector(const void *val, const Type &type, int indent, + template<typename T, typename SizeT = uoffset_t> + const char *PrintVector(const void *val, const Type &type, int indent, const uint8_t *prev_val) { - typedef Vector<T> Container; + typedef Vector<T, SizeT> Container; typedef typename PrintTag<typename Container::return_type>::type tag; auto &vec = *reinterpret_cast<const Container *>(val); return PrintContainer<Container>(tag(), vec, vec.size(), type, indent, @@ -158,14 +162,16 @@ struct JsonPrinter { // Print an array a sequence of JSON values, comma separated, wrapped in "[]". template<typename T> - bool PrintArray(const void *val, size_t size, const Type &type, int indent) { + const char *PrintArray(const void *val, uint16_t size, const Type &type, + + int indent) { typedef Array<T, 0xFFFF> Container; typedef typename PrintTag<typename Container::return_type>::type tag; auto &arr = *reinterpret_cast<const Container *>(val); return PrintContainer<Container>(tag(), arr, size, type, indent, nullptr); } - bool PrintOffset(const void *val, const Type &type, int indent, + const char *PrintOffset(const void *val, const Type &type, int indent, const uint8_t *prev_val, soffset_t vector_index) { switch (type.base_type) { case BASE_TYPE_UNION: { @@ -182,7 +188,7 @@ struct JsonPrinter { if (enum_val) { return PrintOffset(val, enum_val->union_type, indent, nullptr, -1); } else { - return false; + return "unknown enum value"; } } case BASE_TYPE_STRUCT: @@ -190,8 +196,9 @@ struct JsonPrinter { indent); case BASE_TYPE_STRING: { auto s = reinterpret_cast<const String *>(val); - return EscapeString(s->c_str(), s->size(), &text, opts.allow_non_utf8, - opts.natural_utf8); + bool ok = EscapeString(s->c_str(), s->size(), &text, opts.allow_non_utf8, + opts.natural_utf8); + return ok ? nullptr : "string contains non-utf8 bytes"; } case BASE_TYPE_VECTOR: { const auto vec_type = type.VectorType(); @@ -199,17 +206,15 @@ struct JsonPrinter { // clang-format off switch (vec_type.base_type) { #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \ - case BASE_TYPE_ ## ENUM: \ - if (!PrintVector<CTYPE>( \ - val, vec_type, indent, prev_val)) { \ - return false; \ - } \ - break; + case BASE_TYPE_ ## ENUM: { \ + auto err = PrintVector<CTYPE>(val, vec_type, indent, prev_val); \ + if (err) return err; \ + break; } FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) #undef FLATBUFFERS_TD } // clang-format on - return true; + return nullptr; } case BASE_TYPE_ARRAY: { const auto vec_type = type.VectorType(); @@ -217,12 +222,10 @@ struct JsonPrinter { // clang-format off switch (vec_type.base_type) { #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \ - case BASE_TYPE_ ## ENUM: \ - if (!PrintArray<CTYPE>( \ - val, type.fixed_length, vec_type, indent)) { \ - return false; \ - } \ - break; + case BASE_TYPE_ ## ENUM: { \ + auto err = PrintArray<CTYPE>(val, type.fixed_length, vec_type, indent); \ + if (err) return err; \ + break; } FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD) // Arrays of scalars or structs are only possible. FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD) @@ -230,14 +233,16 @@ struct JsonPrinter { case BASE_TYPE_ARRAY: FLATBUFFERS_ASSERT(0); } // clang-format on - return true; + return nullptr; } - default: FLATBUFFERS_ASSERT(0); return false; + default: + FLATBUFFERS_ASSERT(0); + return "unknown type"; } } template<typename T> static T GetFieldDefault(const FieldDef &fd) { - T val; + T val{}; auto check = StringToNumber(fd.value.constant.c_str(), &val); (void)check; FLATBUFFERS_ASSERT(check); @@ -246,17 +251,28 @@ struct JsonPrinter { // Generate text for a scalar field. template<typename T> - bool GenField(const FieldDef &fd, const Table *table, bool fixed, + void GenField(const FieldDef &fd, const Table *table, bool fixed, int indent) { - return PrintScalar( - fixed ? reinterpret_cast<const Struct *>(table)->GetField<T>( - fd.value.offset) - : table->GetField<T>(fd.value.offset, GetFieldDefault<T>(fd)), - fd.value.type, indent); + if (fixed) { + PrintScalar( + reinterpret_cast<const Struct *>(table)->GetField<T>(fd.value.offset), + fd.value.type, indent); + } else if (fd.IsOptional()) { + auto opt = table->GetOptional<T, T>(fd.value.offset); + if (opt) { + PrintScalar(*opt, fd.value.type, indent); + } else { + text += "null"; + } + } else { + PrintScalar( + table->GetField<T>(fd.value.offset, GetFieldDefault<T>(fd)), + fd.value.type, indent); + } } // Generate text for non-scalar field. - bool GenFieldOffset(const FieldDef &fd, const Table *table, bool fixed, + const char *GenFieldOffset(const FieldDef &fd, const Table *table, bool fixed, int indent, const uint8_t *prev_val) { const void *val = nullptr; if (fixed) { @@ -264,12 +280,18 @@ struct JsonPrinter { FLATBUFFERS_ASSERT(IsStruct(fd.value.type) || IsArray(fd.value.type)); val = reinterpret_cast<const Struct *>(table)->GetStruct<const void *>( fd.value.offset); - } else if (fd.flexbuffer) { + } else if (fd.flexbuffer && opts.json_nested_flexbuffers) { + // We could verify this FlexBuffer before access, but since this sits + // inside a FlatBuffer that we don't know wether it has been verified or + // not, there is little point making this part safer than the parent.. + // The caller should really be verifying the whole. + // If the whole buffer is corrupt, we likely crash before we even get + // here. auto vec = table->GetPointer<const Vector<uint8_t> *>(fd.value.offset); auto root = flexbuffers::GetRoot(vec->data(), vec->size()); root.ToString(true, opts.strict_json, text); - return true; - } else if (fd.nested_flatbuffer) { + return nullptr; + } else if (fd.nested_flatbuffer && opts.json_nested_flatbuffers) { auto vec = table->GetPointer<const Vector<uint8_t> *>(fd.value.offset); auto root = GetRoot<Table>(vec->data()); return GenStruct(*fd.nested_flatbuffer, root, indent); @@ -283,7 +305,8 @@ struct JsonPrinter { // Generate text for a struct or table, values separated by commas, indented, // and bracketed by "{}" - bool GenStruct(const StructDef &struct_def, const Table *table, int indent) { + const char *GenStruct(const StructDef &struct_def, const Table *table, + int indent) { text += '{'; int fieldout = 0; const uint8_t *prev_val = nullptr; @@ -307,11 +330,9 @@ struct JsonPrinter { // clang-format off switch (fd.value.type.base_type) { #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \ - case BASE_TYPE_ ## ENUM: \ - if (!GenField<CTYPE>(fd, table, struct_def.fixed, elem_indent)) { \ - return false; \ - } \ - break; + case BASE_TYPE_ ## ENUM: { \ + GenField<CTYPE>(fd, table, struct_def.fixed, elem_indent); \ + break; } FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD) #undef FLATBUFFERS_TD // Generate drop-thru case statements for all pointer types: @@ -320,10 +341,11 @@ struct JsonPrinter { FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD) FLATBUFFERS_GEN_TYPE_ARRAY(FLATBUFFERS_TD) #undef FLATBUFFERS_TD - if (!GenFieldOffset(fd, table, struct_def.fixed, elem_indent, prev_val)) { - return false; - } - break; + { + auto err = GenFieldOffset(fd, table, struct_def.fixed, elem_indent, prev_val); + if (err) return err; + break; + } } // clang-format on // Track prev val for use with union types. @@ -337,7 +359,7 @@ struct JsonPrinter { AddNewLine(); AddIndent(indent); text += '}'; - return true; + return nullptr; } JsonPrinter(const Parser &parser, std::string &dest) @@ -349,25 +371,26 @@ struct JsonPrinter { std::string &text; }; -static bool GenerateTextImpl(const Parser &parser, const Table *table, - const StructDef &struct_def, std::string *_text) { +static const char *GenerateTextImpl(const Parser &parser, const Table *table, + const StructDef &struct_def, std::string *_text) { JsonPrinter printer(parser, *_text); - if (!printer.GenStruct(struct_def, table, 0)) { return false; } + auto err = printer.GenStruct(struct_def, table, 0); + if (err) return err; printer.AddNewLine(); - return true; + return nullptr; } // Generate a text representation of a flatbuffer in JSON format. -bool GenerateTextFromTable(const Parser &parser, const void *table, +const char *GenerateTextFromTable(const Parser &parser, const void *table, const std::string &table_name, std::string *_text) { auto struct_def = parser.LookupStruct(table_name); - if (struct_def == nullptr) { return false; } + if (struct_def == nullptr) { return "unknown struct"; } auto root = static_cast<const Table *>(table); return GenerateTextImpl(parser, root, *struct_def, _text); } // Generate a text representation of a flatbuffer in JSON format. -bool GenerateText(const Parser &parser, const void *flatbuffer, +const char *GenerateText(const Parser &parser, const void *flatbuffer, std::string *_text) { FLATBUFFERS_ASSERT(parser.root_struct_def_); // call SetRootType() auto root = parser.opts.size_prefixed ? GetSizePrefixedRoot<Table>(flatbuffer) @@ -380,21 +403,24 @@ static std::string TextFileName(const std::string &path, return path + file_name + ".json"; } -bool GenerateTextFile(const Parser &parser, const std::string &path, - const std::string &file_name) { +const char *GenerateTextFile(const Parser &parser, const std::string &path, + const std::string &file_name) { if (parser.opts.use_flexbuffers) { std::string json; parser.flex_root_.ToString(true, parser.opts.strict_json, json); return flatbuffers::SaveFile(TextFileName(path, file_name).c_str(), - json.c_str(), json.size(), true); + json.c_str(), json.size(), true) + ? nullptr + : "SaveFile failed"; } - if (!parser.builder_.GetSize() || !parser.root_struct_def_) return true; + if (!parser.builder_.GetSize() || !parser.root_struct_def_) return nullptr; std::string text; - if (!GenerateText(parser, parser.builder_.GetBufferPointer(), &text)) { - return false; - } + auto err = GenerateText(parser, parser.builder_.GetBufferPointer(), &text); + if (err) return err; return flatbuffers::SaveFile(TextFileName(path, file_name).c_str(), text, - false); + false) + ? nullptr + : "SaveFile failed"; } std::string TextMakeRule(const Parser &parser, const std::string &path, @@ -411,4 +437,65 @@ std::string TextMakeRule(const Parser &parser, const std::string &path, return make_rule; } +namespace { + +class TextCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + auto err = GenerateTextFile(parser, path, filename); + if (err) { + status_detail = " (" + std::string(err) + ")"; + return Status::ERROR; + } + return Status::OK; + } + + // Generate code from the provided `buffer` of given `length`. The buffer is a + // serialized reflection.fbs. + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + output = TextMakeRule(parser, path, filename); + return Status::OK; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + (void)parser; + (void)path; + (void)filename; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + + bool IsSchemaOnly() const override { return false; } + + bool SupportsBfbsGeneration() const override { return false; } + + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kJson; } + + std::string LanguageName() const override { return "text"; } +}; + +} // namespace + +std::unique_ptr<CodeGenerator> NewTextCodeGenerator() { + return std::unique_ptr<TextCodeGenerator>(new TextCodeGenerator()); +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_text.h b/contrib/libs/flatbuffers/src/idl_gen_text.h new file mode 100644 index 00000000000..3179a4cfa12 --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_text.h @@ -0,0 +1,29 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_TEXT_H_ +#define FLATBUFFERS_IDL_GEN_TEXT_H_ + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Text code generator. +std::unique_ptr<CodeGenerator> NewTextCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_TEXT_H_ diff --git a/contrib/libs/flatbuffers/src/idl_gen_ts.cpp b/contrib/libs/flatbuffers/src/idl_gen_ts.cpp index 53e088fe137..ca072f1c41c 100644 --- a/contrib/libs/flatbuffers/src/idl_gen_ts.cpp +++ b/contrib/libs/flatbuffers/src/idl_gen_ts.cpp @@ -14,28 +14,89 @@ * limitations under the License. */ -// independent from idl_parser, since this code is not needed for most clients +#include "idl_gen_ts.h" + #include <algorithm> #include <cassert> +#include <cmath> +#include <iostream> #include <unordered_map> #include <unordered_set> #include "flatbuffers/code_generators.h" #include "flatbuffers/flatbuffers.h" +#include "flatbuffers/flatc.h" #include "flatbuffers/idl.h" #include "flatbuffers/util.h" +#include "idl_namer.h" namespace flatbuffers { - +namespace { struct ImportDefinition { std::string name; - std::string statement; - const Definition *dependent; - const Definition *dependency; + std::string import_statement; + std::string export_statement; + std::string bare_file_path; + std::string rel_file_path; + std::string object_name; + const Definition *dependent = nullptr; + const Definition *dependency = nullptr; }; +struct NsDefinition { + std::string path; + std::string filepath; + std::string symbolic_name; + const Namespace *ns; + std::map<std::string, const Definition *> definitions; +}; + +Namer::Config TypeScriptDefaultConfig() { + return { /*types=*/Case::kKeep, + /*constants=*/Case::kUnknown, + /*methods=*/Case::kLowerCamel, + /*functions=*/Case::kLowerCamel, + /*fields=*/Case::kLowerCamel, + /*variables=*/Case::kLowerCamel, + /*variants=*/Case::kKeep, + /*enum_variant_seperator=*/"::", + /*escape_keywords=*/Namer::Config::Escape::AfterConvertingCase, + /*namespaces=*/Case::kKeep, + /*namespace_seperator=*/"_", + /*object_prefix=*/"", + /*object_suffix=*/"T", + /*keyword_prefix=*/"", + /*keyword_suffix=*/"_", + /*filenames=*/Case::kDasher, + /*directories=*/Case::kDasher, + /*output_path=*/"", + /*filename_suffix=*/"_generated", + /*filename_extension=*/".ts" }; +} + +std::set<std::string> TypescriptKeywords() { + // List of keywords retrieved from here: + // https://github.com/microsoft/TypeScript/issues/2536 + return { + "arguments", "break", "case", "catch", "class", "const", + "continue", "debugger", "default", "delete", "do", "else", + "enum", "export", "extends", "false", "finally", "for", + "function", "if", "import", "in", "instanceof", "new", + "null", "Object", "return", "super", "switch", "this", + "throw", "true", "try", "typeof", "var", "void", + "while", "with", "as", "implements", "interface", "let", + "package", "private", "protected", "public", "static", "yield", + }; +} + enum AnnotationType { kParam = 0, kType = 1, kReturns = 2 }; +template<typename T> struct SupportsObjectAPI : std::false_type {}; + +template<> struct SupportsObjectAPI<StructDef> : std::true_type {}; + +} // namespace + namespace ts { // Iterate through all definitions we haven't generate code for (enums, structs, // and tables) and output them to a single file. @@ -45,38 +106,106 @@ class TsGenerator : public BaseGenerator { TsGenerator(const Parser &parser, const std::string &path, const std::string &file_name) - : BaseGenerator(parser, path, file_name, "", ".", "ts") {} + : BaseGenerator(parser, path, file_name, "", "_", "ts"), + namer_(WithFlagOptions(TypeScriptDefaultConfig(), parser.opts, path), + TypescriptKeywords()) {} + bool generate() { generateEnums(); generateStructs(); + generateEntry(); + if (!generateBundle()) return false; return true; } + std::string GetTypeName(const EnumDef &def, const bool = false, + const bool force_ns_wrap = false) { + if (force_ns_wrap) { return namer_.NamespacedType(def); } + return namer_.Type(def); + } + + std::string GetTypeName(const StructDef &def, const bool object_api = false, + const bool force_ns_wrap = false) { + if (object_api && parser_.opts.generate_object_based_api) { + if (force_ns_wrap) { + return namer_.NamespacedObjectType(def); + } else { + return namer_.ObjectType(def); + } + } else { + if (force_ns_wrap) { + return namer_.NamespacedType(def); + } else { + return namer_.Type(def); + } + } + } + // Save out the generated code for a single class while adding // declaration boilerplate. - bool SaveType(const Definition &definition, const std::string &classcode, - import_set &imports, import_set &bare_imports) const { - if (!classcode.length()) return true; + bool SaveType(const Definition &definition, const std::string &class_code, + import_set &imports, import_set &bare_imports) { + if (!class_code.length()) return true; + + std::string code; - std::string code = - "// " + std::string(FlatBuffersGeneratedWarning()) + "\n\n"; + code += "// " + std::string(FlatBuffersGeneratedWarning()) + "\n\n"; - for (auto it = bare_imports.begin(); it != bare_imports.end(); it++) - code += it->second.statement + "\n"; + for (auto it = bare_imports.begin(); it != bare_imports.end(); it++) { + code += it->second.import_statement + "\n"; + } if (!bare_imports.empty()) code += "\n"; - for (auto it = imports.begin(); it != imports.end(); it++) - if (it->second.dependency != &definition) // do not import itself - code += it->second.statement + "\n"; + for (auto it = imports.begin(); it != imports.end(); it++) { + if (it->second.dependency != &definition) { + code += it->second.import_statement + "\n"; + } + } if (!imports.empty()) code += "\n\n"; - code += classcode; - auto filename = NamespaceDir(*definition.defined_namespace, true) + - ToDasherizedCase(definition.name) + ".ts"; - return SaveFile(filename.c_str(), code, false); + code += class_code; + + auto dirs = namer_.Directories(*definition.defined_namespace); + EnsureDirExists(dirs); + auto basename = dirs + namer_.File(definition, SkipFile::Suffix); + + return SaveFile(basename.c_str(), code, false); + } + + void TrackNsDef(const Definition &definition, std::string type_name) { + std::string path; + std::string filepath; + std::string symbolic_name; + if (definition.defined_namespace->components.size() > 0) { + path = namer_.Directories(*definition.defined_namespace, + SkipDir::TrailingPathSeperator); + filepath = path + ".ts"; + path = namer_.Directories(*definition.defined_namespace, + SkipDir::OutputPathAndTrailingPathSeparator); + symbolic_name = definition.defined_namespace->components.back(); + } else { + auto def_mod_name = namer_.File(definition, SkipFile::SuffixAndExtension); + symbolic_name = file_name_; + filepath = path_ + symbolic_name + ".ts"; + } + if (ns_defs_.count(path) == 0) { + NsDefinition nsDef; + nsDef.path = path; + nsDef.filepath = filepath; + nsDef.ns = definition.defined_namespace; + nsDef.definitions.insert(std::make_pair(type_name, &definition)); + nsDef.symbolic_name = symbolic_name; + ns_defs_[path] = nsDef; + } else { + ns_defs_[path].definitions.insert(std::make_pair(type_name, &definition)); + } } private: + IdlNamer namer_; + + std::map<std::string, NsDefinition> ns_defs_; + // Generate code for all enums. void generateEnums() { for (auto it = parser_.enums_.vec.begin(); it != parser_.enums_.vec.end(); @@ -87,6 +216,8 @@ class TsGenerator : public BaseGenerator { auto &enum_def = **it; GenEnum(enum_def, &enumcode, imports, false); GenEnum(enum_def, &enumcode, imports, true); + std::string type_name = GetTypeName(enum_def); + TrackNsDef(enum_def, type_name); SaveType(enum_def, enumcode, imports, bare_imports); } } @@ -101,10 +232,106 @@ class TsGenerator : public BaseGenerator { auto &struct_def = **it; std::string declcode; GenStruct(parser_, struct_def, &declcode, imports); + std::string type_name = GetTypeName(struct_def); + TrackNsDef(struct_def, type_name); SaveType(struct_def, declcode, imports, bare_imports); } } + // Generate code for a single entry point module. + void generateEntry() { + std::string code; + + // add root namespace def if not already existing from defs tracking + std::string root; + if (ns_defs_.count(root) == 0) { + NsDefinition nsDef; + nsDef.path = root; + nsDef.symbolic_name = file_name_; + nsDef.filepath = path_ + file_name_ + ".ts"; + nsDef.ns = new Namespace(); + ns_defs_[nsDef.path] = nsDef; + } + + for (const auto &it : ns_defs_) { + code = "// " + std::string(FlatBuffersGeneratedWarning()) + "\n\n"; + // export all definitions in ns entry point module + int export_counter = 0; + for (const auto &def : it.second.definitions) { + std::vector<std::string> rel_components; + // build path for root level vs child level + if (it.second.ns->components.size() > 1) + std::copy(it.second.ns->components.begin() + 1, + it.second.ns->components.end(), + std::back_inserter(rel_components)); + else + std::copy(it.second.ns->components.begin(), + it.second.ns->components.end(), + std::back_inserter(rel_components)); + auto base_file_name = + namer_.File(*(def.second), SkipFile::SuffixAndExtension); + auto base_name = + namer_.Directories(it.second.ns->components, SkipDir::OutputPath) + + base_file_name; + auto ts_file_path = base_name + ".ts"; + auto base_name_rel = std::string("./"); + base_name_rel += + namer_.Directories(rel_components, SkipDir::OutputPath); + base_name_rel += base_file_name; + auto ts_file_path_rel = base_name_rel + ".ts"; + auto type_name = def.first; + auto fully_qualified_type_name = + it.second.ns->GetFullyQualifiedName(type_name); + auto is_struct = parser_.structs_.Lookup(fully_qualified_type_name); + code += "export { " + type_name; + if (parser_.opts.generate_object_based_api && is_struct) { + code += ", " + type_name + parser_.opts.object_suffix; + } + code += " } from '"; + std::string import_extension = + parser_.opts.ts_no_import_ext ? "" : ".js"; + code += base_name_rel + import_extension + "';\n"; + export_counter++; + } + + // re-export child namespace(s) in parent + const auto child_ns_level = it.second.ns->components.size() + 1; + for (const auto &it2 : ns_defs_) { + if (it2.second.ns->components.size() != child_ns_level) continue; + auto ts_file_path = it2.second.path + ".ts"; + code += "export * as " + it2.second.symbolic_name + " from './"; + std::string rel_path = it2.second.path; + code += rel_path + ".js';\n"; + export_counter++; + } + + if (export_counter > 0) SaveFile(it.second.filepath.c_str(), code, false); + } + } + + bool generateBundle() { + if (parser_.opts.ts_flat_files) { + std::string inputpath; + std::string symbolic_name = file_name_; + inputpath = path_ + file_name_ + ".ts"; + std::string bundlepath = + GeneratedFileName(path_, file_name_, parser_.opts); + bundlepath = bundlepath.substr(0, bundlepath.size() - 3) + ".js"; + std::string cmd = "esbuild"; + cmd += " "; + cmd += inputpath; + // cmd += " --minify"; + cmd += " --format=cjs --bundle --outfile="; + cmd += bundlepath; + cmd += " --external:flatbuffers"; + std::cout << "Entry point " << inputpath << " generated." << std::endl; + std::cout << "A single file bundle can be created using fx. esbuild with:" + << std::endl; + std::cout << "> " << cmd << std::endl; + } + return true; + } + // Generate a documentation comment, if available. static void GenDocComment(const std::vector<std::string> &dc, std::string *code_ptr, @@ -136,9 +363,9 @@ class TsGenerator : public BaseGenerator { if (reverse) return; // FIXME. std::string &code = *code_ptr; GenDocComment(enum_def.doc_comment, code_ptr); - std::string ns = GetNameSpace(enum_def); - std::string enum_def_name = enum_def.name + (reverse ? "Name" : ""); - code += "export enum " + enum_def.name + "{\n"; + code += "export enum "; + code += GetTypeName(enum_def); + code += " {\n"; for (auto it = enum_def.Vals().begin(); it != enum_def.Vals().end(); ++it) { auto &ev = **it; if (!ev.doc_comment.empty()) { @@ -150,11 +377,20 @@ class TsGenerator : public BaseGenerator { if (reverse) { code += " '" + enum_def.ToString(ev) + "'"; code += " = "; - code += "'" + ev.name + "'"; + code += "'" + namer_.Variant(ev) + "'"; } else { - code += " " + ev.name; + code += " " + namer_.Variant(ev); code += " = "; - code += enum_def.ToString(ev); + // Unfortunately, because typescript does not support bigint enums, + // for 64-bit enums, we instead map the enum names to strings. + switch (enum_def.underlying_type.base_type) { + case BASE_TYPE_LONG: + case BASE_TYPE_ULONG: { + code += "'" + enum_def.ToString(ev) + "'"; + break; + } + default: code += enum_def.ToString(ev); + } } code += (it + 1) != enum_def.Vals().end() ? ",\n" : "\n"; @@ -165,7 +401,7 @@ class TsGenerator : public BaseGenerator { code += GenUnionConvFunc(enum_def.underlying_type, imports); } - code += "\n\n"; + code += "\n"; } static std::string GenType(const Type &type) { @@ -200,8 +436,7 @@ class TsGenerator : public BaseGenerator { return GenBBAccess() + ".__union_with_string" + arguments; case BASE_TYPE_VECTOR: return GenGetter(type.VectorType(), arguments); default: { - auto getter = - GenBBAccess() + ".read" + MakeCamel(GenType(type)) + arguments; + auto getter = GenBBAccess() + "." + "read" + GenType(type) + arguments; if (type.base_type == BASE_TYPE_BOOL) { getter = "!!" + getter; } return getter; } @@ -210,18 +445,43 @@ class TsGenerator : public BaseGenerator { std::string GenBBAccess() const { return "this.bb!"; } - std::string GenDefaultValue(const FieldDef &field, const std::string &context, - import_set &imports) { + std::string GenDefaultValue(const FieldDef &field, import_set &imports) { if (field.IsScalarOptional()) { return "null"; } const auto &value = field.value; if (value.type.enum_def && value.type.base_type != BASE_TYPE_UNION && value.type.base_type != BASE_TYPE_VECTOR) { - if (auto val = value.type.enum_def->FindByValue(value.constant)) { - return AddImport(imports, *value.type.enum_def, *value.type.enum_def) + - "." + val->name; - } else { - return value.constant; + switch (value.type.base_type) { + case BASE_TYPE_ARRAY: { + std::string ret = "["; + for (auto i = 0; i < value.type.fixed_length; ++i) { + std::string enum_name = + AddImport(imports, *value.type.enum_def, *value.type.enum_def) + .name; + std::string enum_value = namer_.Variant( + *value.type.enum_def->FindByValue(value.constant)); + ret += enum_name + "." + enum_value + + (i < value.type.fixed_length - 1 ? ", " : ""); + } + ret += "]"; + return ret; + } + case BASE_TYPE_LONG: + case BASE_TYPE_ULONG: { + // If the value is an enum with a 64-bit base type, we have to just + // return the bigint value directly since typescript does not support + // enums with bigint backing types. + return "BigInt('" + value.constant + "')"; + } + default: { + EnumVal *val = value.type.enum_def->FindByValue(value.constant); + if (val == nullptr) + val = const_cast<EnumVal *>(value.type.enum_def->MinValue()); + return AddImport(imports, *value.type.enum_def, + *value.type.enum_def) + .name + + "." + namer_.Variant(*val); + } } } @@ -234,17 +494,24 @@ class TsGenerator : public BaseGenerator { return "null"; } + case BASE_TYPE_ARRAY: case BASE_TYPE_VECTOR: return "[]"; case BASE_TYPE_LONG: case BASE_TYPE_ULONG: { - int64_t constant = StringToInt(value.constant.c_str()); - std::string createLong = context + ".createLong"; - return createLong + "(" + NumToString(static_cast<int32_t>(constant)) + - ", " + NumToString(static_cast<int32_t>(constant >> 32)) + ")"; + return "BigInt('" + value.constant + "')"; } - default: return value.constant; + default: { + if (StringIsFlatbufferNan(value.constant)) { + return "NaN"; + } else if (StringIsFlatbufferPositiveInfinity(value.constant)) { + return "Infinity"; + } else if (StringIsFlatbufferNegativeInfinity(value.constant)) { + return "-Infinity"; + } + return value.constant; + } } } @@ -257,7 +524,7 @@ class TsGenerator : public BaseGenerator { if (IsString(type)) { name = "string|Uint8Array"; } else { - name = AddImport(imports, owner, *type.struct_def); + name = AddImport(imports, owner, *type.struct_def).name; } return allowNull ? (name + "|null") : name; } @@ -266,12 +533,28 @@ class TsGenerator : public BaseGenerator { switch (type.base_type) { case BASE_TYPE_BOOL: return allowNull ? "boolean|null" : "boolean"; case BASE_TYPE_LONG: - case BASE_TYPE_ULONG: - return allowNull ? "flatbuffers.Long|null" : "flatbuffers.Long"; + case BASE_TYPE_ULONG: return allowNull ? "bigint|null" : "bigint"; + case BASE_TYPE_ARRAY: { + std::string name; + if (type.element == BASE_TYPE_LONG || type.element == BASE_TYPE_ULONG) { + name = "bigint[]"; + } else if (type.element != BASE_TYPE_STRUCT) { + name = "number[]"; + } else { + name = "any[]"; + if (parser_.opts.generate_object_based_api) { + name = "(any|" + + GetTypeName(*type.struct_def, /*object_api =*/true) + ")[]"; + } + } + + return name + (allowNull ? "|null" : ""); + } default: if (IsScalar(type.base_type)) { if (type.enum_def) { - const auto enum_name = AddImport(imports, owner, *type.enum_def); + const auto enum_name = + AddImport(imports, owner, *type.enum_def).name; return allowNull ? (enum_name + "|null") : enum_name; } return allowNull ? "number|null" : "number"; @@ -281,7 +564,7 @@ class TsGenerator : public BaseGenerator { } // Returns the method name for use with add/put calls. - static std::string GenWriteMethod(const Type &type) { + std::string GenWriteMethod(const Type &type) { // Forward to signed versions since unsigned versions don't exist switch (type.base_type) { case BASE_TYPE_UTYPE: @@ -292,7 +575,7 @@ class TsGenerator : public BaseGenerator { default: break; } - return IsScalar(type.base_type) ? MakeCamel(GenType(type)) + return IsScalar(type.base_type) ? namer_.Type(GenType(type)) : (IsStruct(type) ? "Struct" : "Offset"); } @@ -316,15 +599,15 @@ class TsGenerator : public BaseGenerator { GenStructArgs(imports, *field.value.type.struct_def, arguments, nameprefix + field.name + "_"); } else { - *arguments += - ", " + nameprefix + field.name + ": " + - GenTypeName(imports, field, field.value.type, true, field.IsOptional()); + *arguments += ", " + nameprefix + field.name + ": " + + GenTypeName(imports, field, field.value.type, true, + field.IsOptional()); } } } - static void GenStructBody(const StructDef &struct_def, std::string *body, - const std::string &nameprefix) { + void GenStructBody(const StructDef &struct_def, std::string *body, + const std::string &nameprefix) { *body += " builder.prep("; *body += NumToString(struct_def.minalign) + ", "; *body += NumToString(struct_def.bytesize) + ");\n"; @@ -339,18 +622,97 @@ class TsGenerator : public BaseGenerator { // Generate arguments for a struct inside a struct. To ensure names // don't clash, and to make it obvious these arguments are constructing // a nested struct, prefix the name with the field name. - GenStructBody(*field.value.type.struct_def, body, - nameprefix + field.name + "_"); + GenStructBody( + *field.value.type.struct_def, body, + nameprefix.length() ? nameprefix + "_" + field.name : field.name); } else { - *body += " builder.write" + GenWriteMethod(field.value.type) + "("; - if (field.value.type.base_type == BASE_TYPE_BOOL) { *body += "+"; } - *body += nameprefix + field.name + ");\n"; + auto element_type = field.value.type.element; + + if (field.value.type.base_type == BASE_TYPE_ARRAY) { + switch (field.value.type.element) { + case BASE_TYPE_STRUCT: { + std::string str_last_item_idx = + NumToString(field.value.type.fixed_length - 1); + *body += "\n for (let i = " + str_last_item_idx + + "; i >= 0; --i" + ") {\n"; + + std::string fname = nameprefix.length() + ? nameprefix + "_" + field.name + : field.name; + + *body += " const item = " + fname + "?.[i];\n\n"; + + if (parser_.opts.generate_object_based_api) { + *body += " if (item instanceof " + + GetTypeName(*field.value.type.struct_def, + /*object_api =*/true) + + ") {\n"; + *body += " item.pack(builder);\n"; + *body += " continue;\n"; + *body += " }\n\n"; + } + + std::string class_name = + GetPrefixedName(*field.value.type.struct_def); + std::string pack_func_create_call = + class_name + ".create" + class_name + "(builder,\n"; + pack_func_create_call += + " " + + GenStructMemberValueTS(*field.value.type.struct_def, "item", + ",\n ", false) + + "\n "; + *body += " " + pack_func_create_call; + *body += " );\n }\n\n"; + + break; + } + default: { + std::string str_last_item_idx = + NumToString(field.value.type.fixed_length - 1); + std::string fname = nameprefix.length() + ? nameprefix + "_" + field.name + : field.name; + + *body += "\n for (let i = " + str_last_item_idx + + "; i >= 0; --i) {\n"; + *body += " builder.write"; + *body += GenWriteMethod( + static_cast<flatbuffers::Type>(field.value.type.element)); + *body += "("; + *body += element_type == BASE_TYPE_BOOL ? "+" : ""; + + if (element_type == BASE_TYPE_LONG || + element_type == BASE_TYPE_ULONG) { + *body += "BigInt(" + fname + "?.[i] ?? 0));\n"; + } else { + *body += "(" + fname + "?.[i] ?? 0));\n\n"; + } + *body += " }\n\n"; + break; + } + } + } else { + std::string fname = + nameprefix.length() ? nameprefix + "_" + field.name : field.name; + + *body += " builder.write" + GenWriteMethod(field.value.type) + "("; + if (field.value.type.base_type == BASE_TYPE_BOOL) { + *body += "Number(Boolean(" + fname + ")));\n"; + continue; + } else if (field.value.type.base_type == BASE_TYPE_LONG || + field.value.type.base_type == BASE_TYPE_ULONG) { + *body += "BigInt(" + fname + " ?? 0));\n"; + continue; + } + + *body += fname + ");\n"; + } } } } std::string GenerateNewExpression(const std::string &object_name) { - return "new " + object_name + "()"; + return "new " + namer_.Type(object_name) + "()"; } void GenerateRootAccessor(StructDef &struct_def, std::string *code_ptr, @@ -396,16 +758,6 @@ class TsGenerator : public BaseGenerator { } } - static std::string GetObjApiClassName(const StructDef &sd, - const IDLOptions &opts) { - return GetObjApiClassName(sd.name, opts); - } - - static std::string GetObjApiClassName(const std::string &name, - const IDLOptions &opts) { - return opts.object_prefix + name + opts.object_suffix; - } - bool UnionHasStringType(const EnumDef &union_enum) { return std::any_of(union_enum.Vals().begin(), union_enum.Vals().end(), [](const EnumVal *ev) { @@ -434,7 +786,7 @@ class TsGenerator : public BaseGenerator { if (IsString(ev.union_type)) { type = "string"; // no need to wrap string type in namespace } else if (ev.union_type.base_type == BASE_TYPE_STRUCT) { - type = AddImport(imports, union_enum, *ev.union_type.struct_def); + type = AddImport(imports, union_enum, *ev.union_type.struct_def).name; } else { FLATBUFFERS_ASSERT(false); } @@ -448,110 +800,134 @@ class TsGenerator : public BaseGenerator { return ret; } - std::string AddImport(import_set &imports, const Definition &dependent, - const StructDef &dependency) { - std::string ns; - const auto &depc_comps = dependency.defined_namespace->components; - for (auto it = depc_comps.begin(); it != depc_comps.end(); it++) ns += *it; - std::string unique_name = ns + dependency.name; - std::string import_name = dependency.name; - std::string long_import_name; - if (imports.find(unique_name) != imports.end()) - return imports.find(unique_name)->second.name; + static bool CheckIfNameClashes(const import_set &imports, + const std::string &name) { + // TODO: this would be better as a hashset. for (auto it = imports.begin(); it != imports.end(); it++) { - if (it->second.name == import_name) { - long_import_name = ns + import_name; - break; - } + if (it->second.name == name) { return true; } } - std::string import_statement; - import_statement += "import { "; - if (long_import_name.empty()) { - import_statement += import_name; - if (parser_.opts.generate_object_based_api) - import_statement += ", " + import_name + "T"; + return false; + } + + std::string GenSymbolExpression(const StructDef &struct_def, + const bool has_name_clash, + const std::string &import_name, + const std::string &name, + const std::string &object_name) { + std::string symbols_expression; + + if (has_name_clash) { + // We have a name clash + symbols_expression += import_name + " as " + name; + + if (parser_.opts.generate_object_based_api) { + symbols_expression += ", " + + GetTypeName(struct_def, /*object_api =*/true) + + " as " + object_name; + } } else { - import_statement += dependency.name + " as " + long_import_name; - if (parser_.opts.generate_object_based_api) - import_statement += - ", " + dependency.name + "T as " + long_import_name + "T"; + // No name clash, use the provided name + symbols_expression += name; + + if (parser_.opts.generate_object_based_api) { + symbols_expression += ", " + object_name; + } } - import_statement += " } from '"; - std::string file_name; - const auto &dep_comps = dependent.defined_namespace->components; - for (size_t i = 0; i < dep_comps.size(); i++) - file_name += i == 0 ? ".." : (kPathSeparator + std::string("..")); - if (dep_comps.size() == 0) file_name += "."; - for (auto it = depc_comps.begin(); it != depc_comps.end(); it++) - file_name += kPathSeparator + ToDasherizedCase(*it); - file_name += kPathSeparator + ToDasherizedCase(dependency.name); - import_statement += file_name + "';"; - ImportDefinition import; - import.name = long_import_name.empty() ? import_name : long_import_name; - import.statement = import_statement; - import.dependency = &dependency; - import.dependent = &dependent; - imports.insert(std::make_pair(unique_name, import)); - return import.name; + + return symbols_expression; } - // TODO: largely (but not identical) duplicated code from above couln't find a - // good way to refactor - std::string AddImport(import_set &imports, const Definition &dependent, - const EnumDef &dependency) { - std::string ns; - const auto &depc_comps = dependency.defined_namespace->components; - for (auto it = depc_comps.begin(); it != depc_comps.end(); it++) ns += *it; - std::string unique_name = ns + dependency.name; - std::string import_name = dependency.name; - std::string long_import_name; - if (imports.find(unique_name) != imports.end()) - return imports.find(unique_name)->second.name; - for (auto it = imports.begin(); it != imports.end(); it++) { - if (it->second.name == import_name) { - long_import_name = ns + import_name; - break; - } + std::string GenSymbolExpression(const EnumDef &enum_def, + const bool has_name_clash, + const std::string &import_name, + const std::string &name, + const std::string &) { + std::string symbols_expression; + if (has_name_clash) { + symbols_expression += import_name + " as " + name; + } else { + symbols_expression += name; } - std::string import_statement; - import_statement += "import { "; - if (long_import_name.empty()) - import_statement += import_name; - else - import_statement += dependency.name + " as " + long_import_name; - if (dependency.is_union) { - import_statement += ", unionTo" + import_name; - import_statement += ", unionListTo" + import_name; + + if (enum_def.is_union) { + symbols_expression += ", unionTo" + name; + symbols_expression += ", unionListTo" + name; } - import_statement += " } from '"; - std::string file_name; + + return symbols_expression; + } + + template<typename DefinitionT> + ImportDefinition AddImport(import_set &imports, const Definition &dependent, + const DefinitionT &dependency) { + // The unique name of the dependency, fully qualified in its namespace. + const std::string unique_name = GetTypeName( + dependency, /*object_api = */ false, /*force_ns_wrap=*/true); + + // Look if we have already added this import and return its name if found. + const auto import_pair = imports.find(unique_name); + if (import_pair != imports.end()) { return import_pair->second; } + + // Check if this name would have a name clash with another type. Just use + // the "base" name (properly escaped) without any namespacing applied. + const std::string import_name = GetTypeName(dependency); + const bool has_name_clash = CheckIfNameClashes(imports, import_name); + + // If we have a name clash, use the unique name, otherwise use simple name. + std::string name = has_name_clash ? unique_name : import_name; + + const std::string object_name = + GetTypeName(dependency, /*object_api=*/true, has_name_clash); + + const std::string symbols_expression = GenSymbolExpression( + dependency, has_name_clash, import_name, name, object_name); + + std::string bare_file_path; + std::string rel_file_path; const auto &dep_comps = dependent.defined_namespace->components; - for (size_t i = 0; i < dep_comps.size(); i++) - file_name += i == 0 ? ".." : (kPathSeparator + std::string("..")); - if (dep_comps.size() == 0) file_name += "."; - for (auto it = depc_comps.begin(); it != depc_comps.end(); it++) - file_name += kPathSeparator + ToDasherizedCase(*it); - file_name += kPathSeparator + ToDasherizedCase(dependency.name); - import_statement += file_name + "';"; + for (size_t i = 0; i < dep_comps.size(); i++) { + rel_file_path += i == 0 ? ".." : (kPathSeparator + std::string("..")); + } + if (dep_comps.size() == 0) { rel_file_path += "."; } + + bare_file_path += + kPathSeparator + + namer_.Directories(dependency.defined_namespace->components, + SkipDir::OutputPath) + + namer_.File(dependency, SkipFile::SuffixAndExtension); + rel_file_path += bare_file_path; + ImportDefinition import; - import.name = long_import_name.empty() ? import_name : long_import_name; - import.statement = import_statement; + import.name = name; + import.object_name = object_name; + import.bare_file_path = bare_file_path; + import.rel_file_path = rel_file_path; + std::string import_extension = parser_.opts.ts_no_import_ext ? "" : ".js"; + import.import_statement = "import { " + symbols_expression + " } from '" + + rel_file_path + import_extension + "';"; + import.export_statement = "export { " + symbols_expression + " } from '." + + bare_file_path + import_extension + "';"; import.dependency = &dependency; import.dependent = &dependent; + imports.insert(std::make_pair(unique_name, import)); - return import.name; + + return import; } void AddImport(import_set &imports, std::string import_name, std::string fileName) { ImportDefinition import; import.name = import_name; - import.statement = "import " + import_name + " from '" + fileName + "';"; + import.import_statement = + "import " + import_name + " from '" + fileName + "';"; imports.insert(std::make_pair(import_name, import)); } // Generate a TS union type based on a union's enum - std::string GenObjApiUnionTypeTS(import_set &imports, const IDLOptions &opts, + std::string GenObjApiUnionTypeTS(import_set &imports, + const StructDef &dependent, + const IDLOptions &, const EnumDef &union_enum) { std::string ret = ""; std::set<std::string> type_list; @@ -565,8 +941,8 @@ class TsGenerator : public BaseGenerator { if (IsString(ev.union_type)) { type = "string"; // no need to wrap string type in namespace } else if (ev.union_type.base_type == BASE_TYPE_STRUCT) { - type = GetObjApiClassName( - AddImport(imports, union_enum, *ev.union_type.struct_def), opts); + type = AddImport(imports, dependent, *ev.union_type.struct_def) + .object_name; } else { FLATBUFFERS_ASSERT(false); } @@ -583,11 +959,11 @@ class TsGenerator : public BaseGenerator { } std::string GenUnionConvFuncName(const EnumDef &enum_def) { - return "unionTo" + enum_def.name; + return namer_.Function("unionTo", enum_def); } std::string GenUnionListConvFuncName(const EnumDef &enum_def) { - return "unionListTo" + enum_def.name; + return namer_.Function("unionListTo", enum_def); } std::string GenUnionConvFunc(const Type &union_type, import_set &imports) { @@ -598,12 +974,12 @@ class TsGenerator : public BaseGenerator { const auto valid_union_type_with_null = valid_union_type + "|null"; auto ret = "\n\nexport function " + GenUnionConvFuncName(enum_def) + - "(\n type: " + enum_def.name + + "(\n type: " + GetTypeName(enum_def) + ",\n accessor: (obj:" + valid_union_type + ") => " + valid_union_type_with_null + "\n): " + valid_union_type_with_null + " {\n"; - const auto enum_type = AddImport(imports, enum_def, enum_def); + const auto enum_type = AddImport(imports, enum_def, enum_def).name; const auto union_enum_loop = [&](const std::string &accessor_str) { ret += " switch(" + enum_type + "[type]) {\n"; @@ -614,13 +990,13 @@ class TsGenerator : public BaseGenerator { const auto &ev = **it; if (ev.IsZero()) { continue; } - ret += " case '" + ev.name + "': "; + ret += " case '" + namer_.Variant(ev) + "': "; if (IsString(ev.union_type)) { ret += "return " + accessor_str + "'') as string;"; } else if (ev.union_type.base_type == BASE_TYPE_STRUCT) { const auto type = - AddImport(imports, enum_def, *ev.union_type.struct_def); + AddImport(imports, enum_def, *ev.union_type.struct_def).name; ret += "return " + accessor_str + "new " + type + "())! as " + type + ";"; } else { @@ -637,7 +1013,7 @@ class TsGenerator : public BaseGenerator { ret += "}"; ret += "\n\nexport function " + GenUnionListConvFuncName(enum_def) + - "(\n type: " + enum_def.name + + "(\n type: " + GetTypeName(enum_def) + ", \n accessor: (index: number, obj:" + valid_union_type + ") => " + valid_union_type_with_null + ", \n index: number\n): " + valid_union_type_with_null + " {\n"; @@ -653,12 +1029,13 @@ class TsGenerator : public BaseGenerator { // Used for generating a short function that returns the correct class // based on union enum type. Assume the context is inside the non object api // type - std::string GenUnionValTS(import_set &imports, const std::string &field_name, + std::string GenUnionValTS(import_set &imports, const StructDef &dependent, + const std::string &field_name, const Type &union_type, const bool is_array = false) { if (union_type.enum_def) { const auto &enum_def = *union_type.enum_def; - const auto enum_type = AddImport(imports, enum_def, enum_def); + const auto enum_type = AddImport(imports, dependent, enum_def).name; const std::string union_accessor = "this." + field_name; const auto union_has_string = UnionHasStringType(enum_def); @@ -668,11 +1045,11 @@ class TsGenerator : public BaseGenerator { if (!is_array) { const auto conversion_function = GenUnionConvFuncName(enum_def); - const auto target_enum = "this." + field_name + "Type()"; ret = "(() => {\n"; - ret += " let temp = " + conversion_function + "(" + target_enum + - ", " + field_binded_method + ");\n"; + ret += " const temp = " + conversion_function + "(this." + + namer_.Method(field_name, "Type") + "(), " + + field_binded_method + ");\n"; ret += " if(temp === null) { return null; }\n"; ret += union_has_string ? " if(typeof temp === 'string') { return temp; }\n" @@ -681,21 +1058,22 @@ class TsGenerator : public BaseGenerator { ret += " })()"; } else { const auto conversion_function = GenUnionListConvFuncName(enum_def); - const auto target_enum_accesor = "this." + field_name + "Type"; - const auto target_enum_length = target_enum_accesor + "Length()"; ret = "(() => {\n"; - ret += " let ret = [];\n"; - ret += " for(let targetEnumIndex = 0; targetEnumIndex < " + - target_enum_length + + ret += " const ret: (" + + GenObjApiUnionTypeTS(imports, *union_type.struct_def, + parser_.opts, *union_type.enum_def) + + ")[] = [];\n"; + ret += " for(let targetEnumIndex = 0; targetEnumIndex < this." + + namer_.Method(field_name, "TypeLength") + "()" + "; " "++targetEnumIndex) {\n"; - ret += " let targetEnum = " + target_enum_accesor + - "(targetEnumIndex);\n"; + ret += " const targetEnum = this." + + namer_.Method(field_name, "Type") + "(targetEnumIndex);\n"; ret += " if(targetEnum === null || " + enum_type + "[targetEnum!] === 'NONE') { " "continue; }\n\n"; - ret += " let temp = " + conversion_function + "(targetEnum, " + + ret += " const temp = " + conversion_function + "(targetEnum, " + field_binded_method + ", targetEnumIndex);\n"; ret += " if(temp === null) { continue; }\n"; ret += union_has_string ? " if(typeof temp === 'string') { " @@ -730,15 +1108,25 @@ class TsGenerator : public BaseGenerator { it != struct_def.fields.vec.end(); ++it) { auto &field = **it; - const auto curr_member_accessor = - prefix + "." + MakeCamel(field.name, false); + auto curr_member_accessor = prefix + "." + namer_.Method(field); + if (prefix != "this") { + curr_member_accessor = prefix + "?." + namer_.Method(field); + } if (IsStruct(field.value.type)) { ret += GenStructMemberValueTS(*field.value.type.struct_def, curr_member_accessor, delimiter); } else { if (nullCheck) { - ret += - "(" + prefix + " === null ? 0 : " + curr_member_accessor + "!)"; + std::string nullValue = "0"; + if (field.value.type.base_type == BASE_TYPE_BOOL) { + nullValue = "false"; + } else if (field.value.type.base_type == BASE_TYPE_LONG || + field.value.type.base_type == BASE_TYPE_ULONG) { + nullValue = "BigInt(0)"; + } else if (field.value.type.base_type == BASE_TYPE_ARRAY) { + nullValue = "[]"; + } + ret += "(" + curr_member_accessor + " ?? " + nullValue + ")"; } else { ret += curr_member_accessor; } @@ -753,7 +1141,7 @@ class TsGenerator : public BaseGenerator { void GenObjApi(const Parser &parser, StructDef &struct_def, std::string &obj_api_unpack_func, std::string &obj_api_class, import_set &imports) { - const auto class_name = GetObjApiClassName(struct_def, parser.opts); + const auto class_name = GetTypeName(struct_def, /*object_api=*/true); std::string unpack_func = "\nunpack(): " + class_name + " {\n return new " + class_name + "(" + @@ -773,7 +1161,7 @@ class TsGenerator : public BaseGenerator { std::string pack_func_offset_decl; std::string pack_func_create_call; - const auto struct_name = AddImport(imports, struct_def, struct_def); + const auto struct_name = AddImport(imports, struct_def, struct_def).name; if (has_create) { pack_func_create_call = " return " + struct_name + ".create" + @@ -796,9 +1184,10 @@ class TsGenerator : public BaseGenerator { auto &field = **it; if (field.deprecated) continue; - const auto field_name = MakeCamel(field.name, false); + const auto field_method = namer_.Method(field); + const auto field_field = namer_.Field(field); const std::string field_binded_method = - "this." + field_name + ".bind(this)"; + "this." + field_method + ".bind(this)"; std::string field_val; std::string field_type; @@ -809,8 +1198,7 @@ class TsGenerator : public BaseGenerator { // a string that contains values for things that can be created inline or // the variable name from field_offset_decl std::string field_offset_val; - const auto field_default_val = - GenDefaultValue(field, "flatbuffers", imports); + const auto field_default_val = GenDefaultValue(field, imports); // Emit a scalar field const auto is_string = IsString(field.value.type); @@ -819,14 +1207,14 @@ class TsGenerator : public BaseGenerator { field_type += GenTypeName(imports, field, field.value.type, false, has_null_default); - field_val = "this." + field_name + "()"; + field_val = "this." + namer_.Method(field) + "()"; if (field.value.type.base_type != BASE_TYPE_STRING) { - field_offset_val = "this." + field_name; + field_offset_val = "this." + namer_.Field(field); } else { field_offset_decl = GenNullCheckConditional( - "this." + field_name, - "builder.createString(this." + field_name + "!)", "0"); + "this." + namer_.Field(field), + "builder.createString(this." + field_field + "!)", "0"); } } @@ -836,14 +1224,15 @@ class TsGenerator : public BaseGenerator { switch (field.value.type.base_type) { case BASE_TYPE_STRUCT: { const auto &sd = *field.value.type.struct_def; - field_type += GetObjApiClassName(sd, parser.opts); + field_type += AddImport(imports, struct_def, sd).object_name; - const std::string field_accessor = "this." + field_name + "()"; + const std::string field_accessor = + "this." + namer_.Method(field) + "()"; field_val = GenNullCheckConditional(field_accessor, field_accessor + "!.unpack()"); auto packing = GenNullCheckConditional( - "this." + field_name, "this." + field_name + "!.pack(builder)", - "0"); + "this." + field_field, + "this." + field_field + "!.pack(builder)", "0"); if (sd.fixed) { field_offset_val = std::move(packing); @@ -854,6 +1243,95 @@ class TsGenerator : public BaseGenerator { break; } + case BASE_TYPE_ARRAY: { + auto vectortype = field.value.type.VectorType(); + auto vectortypename = + GenTypeName(imports, struct_def, vectortype, false); + is_vector = true; + + field_type = "("; + + switch (vectortype.base_type) { + case BASE_TYPE_STRUCT: { + const auto &sd = *field.value.type.struct_def; + const auto field_type_name = + GetTypeName(sd, /*object_api=*/true); + field_type += field_type_name; + field_type += ")[]"; + + field_val = GenBBAccess() + ".createObjList<" + vectortypename + + ", " + field_type_name + ">(" + + field_binded_method + ", " + + NumToString(field.value.type.fixed_length) + ")"; + + if (sd.fixed) { + field_offset_decl = + "builder.createStructOffsetList(this." + field_field + + ", " + AddImport(imports, struct_def, struct_def).name + + "." + namer_.Method("start", field, "Vector") + ")"; + } else { + field_offset_decl = + AddImport(imports, struct_def, struct_def).name + "." + + namer_.Method("create", field, "Vector") + + "(builder, builder.createObjectOffsetList(" + "this." + + field_field + "))"; + } + + break; + } + + case BASE_TYPE_STRING: { + field_type += "string)[]"; + field_val = GenBBAccess() + ".createScalarList<string>(" + + field_binded_method + ", this." + + namer_.Field(field, "Length") + "())"; + field_offset_decl = + AddImport(imports, struct_def, struct_def).name + "." + + namer_.Method("create", field, "Vector") + + "(builder, builder.createObjectOffsetList(" + "this." + + namer_.Field(field) + "))"; + break; + } + + case BASE_TYPE_UNION: { + field_type += GenObjApiUnionTypeTS( + imports, struct_def, parser.opts, *(vectortype.enum_def)); + field_type += ")[]"; + field_val = GenUnionValTS(imports, struct_def, field_method, + vectortype, true); + + field_offset_decl = + AddImport(imports, struct_def, struct_def).name + "." + + namer_.Method("create", field, "Vector") + + "(builder, builder.createObjectOffsetList(" + "this." + + namer_.Field(field) + "))"; + + break; + } + default: { + if (vectortype.enum_def) { + field_type += GenTypeName(imports, struct_def, vectortype, + false, HasNullDefault(field)); + } else { + field_type += vectortypename; + } + field_type += ")[]"; + field_val = GenBBAccess() + ".createScalarList<" + + vectortypename + ">(" + field_binded_method + ", " + + NumToString(field.value.type.fixed_length) + ")"; + + field_offset_decl = + AddImport(imports, struct_def, struct_def).name + "." + + namer_.Method("create", field, "Vector") + + "(builder, this." + field_field + ")"; + + break; + } + } + + break; + } + case BASE_TYPE_VECTOR: { auto vectortype = field.value.type.VectorType(); auto vectortypename = @@ -865,24 +1343,27 @@ class TsGenerator : public BaseGenerator { switch (vectortype.base_type) { case BASE_TYPE_STRUCT: { const auto &sd = *field.value.type.struct_def; - field_type += GetObjApiClassName(sd, parser.opts); + const auto field_type_name = + GetTypeName(sd, /*object_api=*/true); + field_type += field_type_name; field_type += ")[]"; - field_val = GenBBAccess() + ".createObjList(" + - field_binded_method + ", this." + field_name + - "Length())"; + field_val = GenBBAccess() + ".createObjList<" + vectortypename + + ", " + field_type_name + ">(" + + field_binded_method + ", this." + + namer_.Method(field, "Length") + "())"; if (sd.fixed) { field_offset_decl = - "builder.createStructOffsetList(this." + field_name + - ", " + AddImport(imports, struct_def, struct_def) + - ".start" + MakeCamel(field_name) + "Vector)"; + "builder.createStructOffsetList(this." + field_field + + ", " + AddImport(imports, struct_def, struct_def).name + + "." + namer_.Method("start", field, "Vector") + ")"; } else { field_offset_decl = - AddImport(imports, struct_def, struct_def) + ".create" + - MakeCamel(field_name) + - "Vector(builder, builder.createObjectOffsetList(" + - "this." + field_name + "))"; + AddImport(imports, struct_def, struct_def).name + "." + + namer_.Method("create", field, "Vector") + + "(builder, builder.createObjectOffsetList(" + "this." + + field_field + "))"; } break; @@ -890,29 +1371,29 @@ class TsGenerator : public BaseGenerator { case BASE_TYPE_STRING: { field_type += "string)[]"; - field_val = GenBBAccess() + ".createScalarList(" + - field_binded_method + ", this." + field_name + - "Length())"; + field_val = GenBBAccess() + ".createScalarList<string>(" + + field_binded_method + ", this." + + namer_.Field(field, "Length") + "())"; field_offset_decl = - AddImport(imports, struct_def, struct_def) + ".create" + - MakeCamel(field_name) + - "Vector(builder, builder.createObjectOffsetList(" + - "this." + field_name + "))"; + AddImport(imports, struct_def, struct_def).name + "." + + namer_.Method("create", field, "Vector") + + "(builder, builder.createObjectOffsetList(" + "this." + + namer_.Field(field) + "))"; break; } case BASE_TYPE_UNION: { - field_type += GenObjApiUnionTypeTS(imports, parser.opts, - *(vectortype.enum_def)); + field_type += GenObjApiUnionTypeTS( + imports, struct_def, parser.opts, *(vectortype.enum_def)); field_type += ")[]"; - field_val = - GenUnionValTS(imports, field_name, vectortype, true); + field_val = GenUnionValTS(imports, struct_def, field_method, + vectortype, true); field_offset_decl = - AddImport(imports, struct_def, struct_def) + ".create" + - MakeCamel(field_name) + - "Vector(builder, builder.createObjectOffsetList(" + - "this." + field_name + "))"; + AddImport(imports, struct_def, struct_def).name + "." + + namer_.Method("create", field, "Vector") + + "(builder, builder.createObjectOffsetList(" + "this." + + namer_.Field(field) + "))"; break; } @@ -924,13 +1405,14 @@ class TsGenerator : public BaseGenerator { field_type += vectortypename; } field_type += ")[]"; - field_val = GenBBAccess() + ".createScalarList(" + - field_binded_method + ", this." + field_name + - "Length())"; + field_val = GenBBAccess() + ".createScalarList<" + + vectortypename + ">(" + field_binded_method + + ", this." + namer_.Method(field, "Length") + "())"; - field_offset_decl = AddImport(imports, struct_def, struct_def) + - ".create" + MakeCamel(field_name) + - "Vector(builder, this." + field_name + ")"; + field_offset_decl = + AddImport(imports, struct_def, struct_def).name + "." + + namer_.Method("create", field, "Vector") + + "(builder, this." + field_field + ")"; break; } @@ -940,12 +1422,13 @@ class TsGenerator : public BaseGenerator { } case BASE_TYPE_UNION: { - field_type += GenObjApiUnionTypeTS(imports, parser.opts, + field_type += GenObjApiUnionTypeTS(imports, struct_def, parser.opts, *(field.value.type.enum_def)); - field_val = GenUnionValTS(imports, field_name, field.value.type); + field_val = GenUnionValTS(imports, struct_def, field_method, + field.value.type); field_offset_decl = - "builder.createObjectOffset(this." + field_name + ")"; + "builder.createObjectOffset(this." + field_field + ")"; break; } @@ -958,15 +1441,17 @@ class TsGenerator : public BaseGenerator { if (!field_offset_decl.empty()) { field_offset_decl = - " const " + field_name + " = " + field_offset_decl + ";"; + " const " + field_field + " = " + field_offset_decl + ";"; } - if (field_offset_val.empty()) { field_offset_val = field_name; } + if (field_offset_val.empty()) { field_offset_val = field_field; } unpack_func += " " + field_val; - unpack_to_func += " _o." + field_name + " = " + field_val + ";"; + unpack_to_func += " _o." + field_field + " = " + field_val + ";"; - constructor_func += " public " + field_name + ": " + field_type + " = " + - field_default_val; + // FIXME: if field_type and field_field are identical, then + // this generates invalid typescript. + constructor_func += " public " + field_field + ": " + field_type + + " = " + field_default_val; if (!struct_def.fixed) { if (!field_offset_decl.empty()) { @@ -976,8 +1461,12 @@ class TsGenerator : public BaseGenerator { if (has_create) { pack_func_create_call += field_offset_val; } else { - pack_func_create_call += " " + struct_name + ".add" + - MakeCamel(field.name) + "(builder, " + + if (field.IsScalarOptional()) { + pack_func_create_call += + " if (" + field_offset_val + " !== null)\n "; + } + pack_func_create_call += " " + struct_name + "." + + namer_.Method("add", field) + "(builder, " + field_offset_val + ");\n"; } } @@ -1011,10 +1500,10 @@ class TsGenerator : public BaseGenerator { pack_func_create_call += "return " + struct_name + ".end" + GetPrefixedName(struct_def) + "(builder);"; } - - obj_api_class = "\nexport class " + - GetObjApiClassName(struct_def, parser.opts) + " {\n"; - + obj_api_class = "\n"; + obj_api_class += "export class "; + obj_api_class += GetTypeName(struct_def, /*object_api=*/true); + obj_api_class += " implements flatbuffers.IGeneratedObject {\n"; obj_api_class += constructor_func; obj_api_class += pack_func_prototype + pack_func_offset_decl + pack_func_create_call + "\n}"; @@ -1045,21 +1534,31 @@ class TsGenerator : public BaseGenerator { if (struct_def.generated) return; std::string &code = *code_ptr; - std::string object_name; - std::string object_namespace = GetNameSpace(struct_def); + // Special case for the root struct, since no one will necessarily reference + // it, we have to explicitly add it to the import list. + if (&struct_def == parser_.root_struct_def_) { + AddImport(imports, struct_def, struct_def); + } + + const std::string object_name = GetTypeName(struct_def); + const std::string object_api_name = GetTypeName(struct_def, true); // Emit constructor - object_name = struct_def.name; GenDocComment(struct_def.doc_comment, code_ptr); - code += "export class " + struct_def.name; - code += " {\n"; + code += "export class "; + code += object_name; + if (parser.opts.generate_object_based_api) + code += " implements flatbuffers.IUnpackableObject<" + object_api_name + + "> {\n"; + else + code += " {\n"; code += " bb: flatbuffers.ByteBuffer|null = null;\n"; code += " bb_pos = 0;\n"; // Generate the __init method that sets the field in a pre-existing // accessor object. This is to allow object reuse. code += - "__init(i:number, bb:flatbuffers.ByteBuffer):" + object_name + " {\n"; + " __init(i:number, bb:flatbuffers.ByteBuffer):" + object_name + " {\n"; code += " this.bb_pos = i;\n"; code += " this.bb = bb;\n"; code += " return this;\n"; @@ -1086,9 +1585,16 @@ class TsGenerator : public BaseGenerator { it != struct_def.fields.vec.end(); ++it) { auto &field = **it; if (field.deprecated) continue; - auto offset_prefix = - " const offset = " + GenBBAccess() + ".__offset(this.bb_pos, " + - NumToString(field.value.offset) + ");\n return offset ? "; + std::string offset_prefix = ""; + + if (field.value.type.base_type == BASE_TYPE_ARRAY) { + offset_prefix = " return "; + } else { + offset_prefix = " const offset = " + GenBBAccess() + + ".__offset(this.bb_pos, " + + NumToString(field.value.offset) + ");\n"; + offset_prefix += " return offset ? "; + } // Emit a scalar field const auto is_string = IsString(field.value.type); @@ -1096,7 +1602,7 @@ class TsGenerator : public BaseGenerator { const auto has_null_default = is_string || HasNullDefault(field); GenDocComment(field.doc_comment, code_ptr); - std::string prefix = MakeCamel(field.name, false) + "("; + std::string prefix = namer_.Method(field) + "("; if (is_string) { code += prefix + "):string|null\n"; code += @@ -1128,9 +1634,11 @@ class TsGenerator : public BaseGenerator { } else { std::string index = "this.bb_pos + offset"; if (is_string) { index += ", optionalEncoding"; } - code += offset_prefix + - GenGetter(field.value.type, "(" + index + ")") + " : " + - GenDefaultValue(field, GenBBAccess(), imports); + code += + offset_prefix + GenGetter(field.value.type, "(" + index + ")"); + if (field.value.type.base_type != BASE_TYPE_ARRAY) { + code += " : " + GenDefaultValue(field, imports); + } code += ";\n"; } } @@ -1140,9 +1648,10 @@ class TsGenerator : public BaseGenerator { switch (field.value.type.base_type) { case BASE_TYPE_STRUCT: { const auto type = - AddImport(imports, struct_def, *field.value.type.struct_def); + AddImport(imports, struct_def, *field.value.type.struct_def) + .name; GenDocComment(field.doc_comment, code_ptr); - code += MakeCamel(field.name, false); + code += namer_.Method(field); code += "(obj?:" + type + "):" + type + "|null {\n"; if (struct_def.fixed) { @@ -1162,6 +1671,95 @@ class TsGenerator : public BaseGenerator { break; } + case BASE_TYPE_ARRAY: { + auto vectortype = field.value.type.VectorType(); + auto vectortypename = + GenTypeName(imports, struct_def, vectortype, false); + auto inline_size = InlineSize(vectortype); + auto index = "this.bb_pos + " + NumToString(field.value.offset) + + " + index" + MaybeScale(inline_size); + std::string ret_type; + bool is_union = false; + switch (vectortype.base_type) { + case BASE_TYPE_STRUCT: ret_type = vectortypename; break; + case BASE_TYPE_STRING: ret_type = vectortypename; break; + case BASE_TYPE_UNION: + ret_type = "?flatbuffers.Table"; + is_union = true; + break; + default: ret_type = vectortypename; + } + GenDocComment(field.doc_comment, code_ptr); + std::string prefix = namer_.Method(field); + // TODO: make it work without any + // if (is_union) { prefix += "<T extends flatbuffers.Table>"; } + if (is_union) { prefix += ""; } + prefix += "(index: number"; + if (is_union) { + const auto union_type = + GenUnionGenericTypeTS(*(field.value.type.enum_def)); + + vectortypename = union_type; + code += prefix + ", obj:" + union_type; + } else if (vectortype.base_type == BASE_TYPE_STRUCT) { + code += prefix + ", obj?:" + vectortypename; + } else if (IsString(vectortype)) { + code += prefix + "):string\n"; + code += prefix + ",optionalEncoding:flatbuffers.Encoding" + + "):" + vectortypename + "\n"; + code += prefix + ",optionalEncoding?:any"; + } else { + code += prefix; + } + code += "):" + vectortypename + "|null {\n"; + + if (vectortype.base_type == BASE_TYPE_STRUCT) { + code += offset_prefix + "(obj || " + + GenerateNewExpression(vectortypename); + code += ").__init("; + code += vectortype.struct_def->fixed + ? index + : GenBBAccess() + ".__indirect(" + index + ")"; + code += ", " + GenBBAccess() + ")"; + } else { + if (is_union) { + index = "obj, " + index; + } else if (IsString(vectortype)) { + index += ", optionalEncoding"; + } + code += offset_prefix + GenGetter(vectortype, "(" + index + ")"); + } + + switch (field.value.type.base_type) { + case BASE_TYPE_ARRAY: { + break; + } + case BASE_TYPE_BOOL: { + code += " : false"; + break; + } + case BASE_TYPE_LONG: + case BASE_TYPE_ULONG: { + code += " : BigInt(0)"; + break; + } + default: { + if (IsScalar(field.value.type.element)) { + if (field.value.type.enum_def) { + code += field.value.constant; + } else { + code += " : 0"; + } + } else { + code += ": null"; + } + break; + } + } + code += ";\n"; + break; + } + case BASE_TYPE_VECTOR: { auto vectortype = field.value.type.VectorType(); auto vectortypename = @@ -1182,7 +1780,7 @@ class TsGenerator : public BaseGenerator { default: ret_type = vectortypename; } GenDocComment(field.doc_comment, code_ptr); - std::string prefix = MakeCamel(field.name, false); + std::string prefix = namer_.Method(field); // TODO: make it work without any // if (is_union) { prefix += "<T extends flatbuffers.Table>"; } if (is_union) { prefix += ""; } @@ -1226,7 +1824,7 @@ class TsGenerator : public BaseGenerator { code += "false"; } else if (field.value.type.element == BASE_TYPE_LONG || field.value.type.element == BASE_TYPE_ULONG) { - code += GenBBAccess() + ".createLong(0, 0)"; + code += "BigInt(0)"; } else if (IsScalar(field.value.type.element)) { if (field.value.type.enum_def) { code += field.value.constant; @@ -1242,7 +1840,7 @@ class TsGenerator : public BaseGenerator { case BASE_TYPE_UNION: { GenDocComment(field.doc_comment, code_ptr); - code += MakeCamel(field.name, false); + code += namer_.Method(field); const auto &union_enum = *(field.value.type.enum_def); const auto union_type = GenUnionGenericTypeTS(union_enum); @@ -1267,11 +1865,14 @@ class TsGenerator : public BaseGenerator { std::string type = GenTypeName(imports, struct_def, field.value.type, true); - code += "mutate_" + field.name + "(value:" + type + "):boolean {\n"; + code += namer_.LegacyTsMutateMethod(field) + "(value:" + type + + "):boolean {\n"; + + const std::string write_method = + "." + namer_.Method("write", GenType(field.value.type)); if (struct_def.fixed) { - code += " " + GenBBAccess() + ".write" + - MakeCamel(GenType(field.value.type)) + "(this.bb_pos + " + + code += " " + GenBBAccess() + write_method + "(this.bb_pos + " + NumToString(field.value.offset) + ", "; } else { code += " const offset = " + GenBBAccess() + @@ -1282,9 +1883,8 @@ class TsGenerator : public BaseGenerator { code += " }\n\n"; // special case for bools, which are treated as uint8 - code += " " + GenBBAccess() + ".write" + - MakeCamel(GenType(field.value.type)) + - "(this.bb_pos + offset, "; + code += + " " + GenBBAccess() + write_method + "(this.bb_pos + offset, "; if (field.value.type.base_type == BASE_TYPE_BOOL) { code += "+"; } } @@ -1297,8 +1897,8 @@ class TsGenerator : public BaseGenerator { if (IsVector(field.value.type)) { // Emit a length helper GenDocComment(code_ptr); - code += MakeCamel(field.name, false); - code += "Length():number {\n" + offset_prefix; + code += namer_.Method(field, "Length"); + code += "():number {\n" + offset_prefix; code += GenBBAccess() + ".__vector_len(this.bb_pos + offset) : 0;\n}\n\n"; @@ -1308,9 +1908,9 @@ class TsGenerator : public BaseGenerator { if (IsScalar(vectorType.base_type) && !IsLong(vectorType.base_type)) { GenDocComment(code_ptr); - code += MakeCamel(field.name, false); - code += "Array():" + GenType(vectorType) + "Array|null {\n" + - offset_prefix; + code += namer_.Method(field, "Array"); + code += + "():" + GenType(vectorType) + "Array|null {\n" + offset_prefix; code += "new " + GenType(vectorType) + "Array(" + GenBBAccess() + ".bytes().buffer, " + GenBBAccess() + @@ -1325,7 +1925,10 @@ class TsGenerator : public BaseGenerator { if (parser_.opts.generate_name_strings) { GenDocComment(code_ptr); code += "static getFullyQualifiedName():string {\n"; - code += " return '" + WrapInNameSpace(struct_def) + "';\n"; + code += + " return '" + + struct_def.defined_namespace->GetFullyQualifiedName(struct_def.name) + + "';\n"; code += "}\n\n"; } @@ -1369,7 +1972,7 @@ class TsGenerator : public BaseGenerator { // Generate the field insertion method GenDocComment(code_ptr); - code += "static add" + MakeCamel(field.name); + code += "static " + namer_.Method("add", field); code += "(builder:flatbuffers.Builder, " + argname + ":" + GetArgType(imports, struct_def, field, false) + ") {\n"; code += " builder.addField" + GenWriteMethod(field.value.type) + "("; @@ -1380,13 +1983,13 @@ class TsGenerator : public BaseGenerator { code += "0"; } else if (HasNullDefault(field)) { if (IsLong(field.value.type.base_type)) { - code += "builder.createLong(0, 0)"; + code += "BigInt(0)"; } else { code += "0"; } } else { if (field.value.type.base_type == BASE_TYPE_BOOL) { code += "+"; } - code += GenDefaultValue(field, "builder", imports); + code += GenDefaultValue(field, imports); } code += ");\n}\n\n"; @@ -1400,8 +2003,8 @@ class TsGenerator : public BaseGenerator { GenDocComment(code_ptr); const std::string sig_begin = - "static create" + MakeCamel(field.name) + - "Vector(builder:flatbuffers.Builder, data:"; + "static " + namer_.Method("create", field, "Vector") + + "(builder:flatbuffers.Builder, data:"; const std::string sig_end = "):flatbuffers.Offset"; std::string type = GenTypeName(imports, struct_def, vector_type, true) + "[]"; @@ -1438,8 +2041,9 @@ class TsGenerator : public BaseGenerator { // after GenDocComment(code_ptr); - code += "static start" + MakeCamel(field.name); - code += "Vector(builder:flatbuffers.Builder, numElems:number) {\n"; + code += "static "; + code += namer_.Method("start", field, "Vector"); + code += "(builder:flatbuffers.Builder, numElems:number) {\n"; code += " builder.startVector(" + NumToString(elem_size); code += ", numElems, " + NumToString(alignment) + ");\n"; code += "}\n\n"; @@ -1482,10 +2086,10 @@ class TsGenerator : public BaseGenerator { } code += "):flatbuffers.Offset {\n"; - code += " " + struct_def.name + ".start" + - GetPrefixedName(struct_def) + "(builder);\n"; + code += " " + object_name + ".start" + GetPrefixedName(struct_def) + + "(builder);\n"; - std::string methodPrefix = struct_def.name; + std::string methodPrefix = object_name; for (auto it = struct_def.fields.vec.begin(); it != struct_def.fields.vec.end(); ++it) { const auto &field = **it; @@ -1497,7 +2101,7 @@ class TsGenerator : public BaseGenerator { code += " if (" + arg_name + " !== null)\n "; } - code += " " + methodPrefix + ".add" + MakeCamel(field.name) + "("; + code += " " + methodPrefix + "." + namer_.Method("add", field) + "("; code += "builder, " + arg_name + ");\n"; } @@ -1515,8 +2119,9 @@ class TsGenerator : public BaseGenerator { code += "}\n"; code += "\n"; - code += "static deserialize(buffer: Uint8Array):" + name + " {\n"; - code += " return " + AddImport(imports, struct_def, struct_def) + + code += "static deserialize(buffer: Uint8Array):" + + namer_.EscapeKeyword(name) + " {\n"; + code += " return " + AddImport(imports, struct_def, struct_def).name + ".getRootAs" + name + "(new flatbuffers.ByteBuffer(buffer))\n"; code += "}\n"; } @@ -1543,10 +2148,9 @@ class TsGenerator : public BaseGenerator { allowNull && field.IsOptional()); } - static std::string GetArgName(const FieldDef &field) { - auto argname = MakeCamel(field.name, false); + std::string GetArgName(const FieldDef &field) { + auto argname = namer_.Variable(field); if (!IsScalar(field.value.type.base_type)) { argname += "Offset"; } - return argname; } @@ -1565,8 +2169,6 @@ bool GenerateTS(const Parser &parser, const std::string &path, std::string TSMakeRule(const Parser &parser, const std::string &path, const std::string &file_name) { - FLATBUFFERS_ASSERT(parser.opts.lang <= IDLOptions::kMAX); - std::string filebase = flatbuffers::StripPath(flatbuffers::StripExtension(file_name)); ts::TsGenerator generator(parser, path, file_name); @@ -1580,4 +2182,54 @@ std::string TSMakeRule(const Parser &parser, const std::string &path, return make_rule; } +namespace { + +class TsCodeGenerator : public CodeGenerator { + public: + Status GenerateCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateTS(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateCode(const uint8_t *buffer, int64_t length) override { + (void)buffer; + (void)length; + return Status::NOT_IMPLEMENTED; + } + + Status GenerateMakeRule(const Parser &parser, const std::string &path, + const std::string &filename, + std::string &output) override { + output = TSMakeRule(parser, path, filename); + return Status::OK; + } + + Status GenerateGrpcCode(const Parser &parser, const std::string &path, + const std::string &filename) override { + if (!GenerateTSGRPC(parser, path, filename)) { return Status::ERROR; } + return Status::OK; + } + + Status GenerateRootFile(const Parser &parser, + const std::string &path) override { + (void)parser; + (void)path; + return Status::NOT_IMPLEMENTED; + } + bool IsSchemaOnly() const override { return true; } + + bool SupportsBfbsGeneration() const override { return false; } + bool SupportsRootFileGeneration() const override { return false; } + + IDLOptions::Language Language() const override { return IDLOptions::kTs; } + + std::string LanguageName() const override { return "TS"; } +}; +} // namespace + +std::unique_ptr<CodeGenerator> NewTsCodeGenerator() { + return std::unique_ptr<TsCodeGenerator>(new TsCodeGenerator()); +} + } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/idl_gen_ts.h b/contrib/libs/flatbuffers/src/idl_gen_ts.h new file mode 100644 index 00000000000..d2ece2dd962 --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_gen_ts.h @@ -0,0 +1,32 @@ +/* + * Copyright 2023 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLATBUFFERS_IDL_GEN_TS_H_ +#define FLATBUFFERS_IDL_GEN_TS_H_ + +#include <memory> +#include <string> + +#include "flatbuffers/code_generator.h" + +namespace flatbuffers { + +// Constructs a new Ts code generator. +std::unique_ptr<CodeGenerator> NewTsCodeGenerator(); + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_GEN_TS_H_ diff --git a/contrib/libs/flatbuffers/src/idl_namer.h b/contrib/libs/flatbuffers/src/idl_namer.h new file mode 100644 index 00000000000..7f89433da65 --- /dev/null +++ b/contrib/libs/flatbuffers/src/idl_namer.h @@ -0,0 +1,177 @@ +#ifndef FLATBUFFERS_IDL_NAMER +#define FLATBUFFERS_IDL_NAMER + +#include "flatbuffers/idl.h" +#include "namer.h" + +namespace flatbuffers { + +// Provides Namer capabilities to types defined in the flatbuffers IDL. +class IdlNamer : public Namer { + public: + explicit IdlNamer(Config config, std::set<std::string> keywords) + : Namer(config, std::move(keywords)) {} + + using Namer::Constant; + using Namer::Directories; + using Namer::Field; + using Namer::File; + using Namer::Function; + using Namer::Method; + using Namer::Namespace; + using Namer::NamespacedType; + using Namer::ObjectType; + using Namer::Type; + using Namer::Variable; + using Namer::Variant; + + std::string Constant(const FieldDef &d) const { return Constant(d.name); } + + // Types are always structs or enums so we can only expose these two + // overloads. + std::string Type(const StructDef &d) const { return Type(d.name); } + std::string Type(const EnumDef &d) const { return Type(d.name); } + + std::string Function(const Definition &s) const { return Function(s.name); } + std::string Function(const std::string& prefix, const Definition &s) const { + return Function(prefix + s.name); + } + + std::string Field(const FieldDef &s) const { return Field(s.name); } + std::string Field(const FieldDef &d, const std::string &s) const { + return Field(d.name + "_" + s); + } + + std::string Variable(const FieldDef &s) const { return Variable(s.name); } + + std::string Variable(const StructDef &s) const { return Variable(s.name); } + + std::string Variant(const EnumVal &s) const { return Variant(s.name); } + + std::string EnumVariant(const EnumDef &e, const EnumVal &v) const { + return Type(e) + config_.enum_variant_seperator + Variant(v); + } + + std::string ObjectType(const StructDef &d) const { + return ObjectType(d.name); + } + std::string ObjectType(const EnumDef &d) const { return ObjectType(d.name); } + + std::string Method(const FieldDef &d, const std::string &suffix) const { + return Method(d.name, suffix); + } + std::string Method(const std::string &prefix, const StructDef &d) const { + return Method(prefix, d.name); + } + std::string Method(const std::string &prefix, const FieldDef &d) const { + return Method(prefix, d.name); + } + std::string Method(const std::string &prefix, const FieldDef &d, + const std::string &suffix) const { + return Method(prefix, d.name, suffix); + } + + std::string Namespace(const struct Namespace &ns) const { + return Namespace(ns.components); + } + + std::string NamespacedEnumVariant(const EnumDef &e, const EnumVal &v) const { + return NamespacedString(e.defined_namespace, EnumVariant(e, v)); + } + + std::string NamespacedType(const Definition &def) const { + return NamespacedString(def.defined_namespace, Type(def.name)); + } + + std::string NamespacedObjectType(const Definition &def) const { + return NamespacedString(def.defined_namespace, ObjectType(def.name)); + } + + std::string Directories(const struct Namespace &ns, + SkipDir skips = SkipDir::None) const { + return Directories(ns.components, skips); + } + + // Legacy fields do not really follow the usual config and should be + // considered for deprecation. + + std::string LegacyRustNativeVariant(const EnumVal &v) const { + return ConvertCase(EscapeKeyword(v.name), Case::kUpperCamel); + } + + std::string LegacyRustFieldOffsetName(const FieldDef &field) const { + return "VT_" + ConvertCase(EscapeKeyword(field.name), Case::kAllUpper); + } + std::string LegacyRustUnionTypeOffsetName(const FieldDef &field) const { + return "VT_" + ConvertCase(EscapeKeyword(field.name + "_type"), Case::kAllUpper); + } + + + std::string LegacySwiftVariant(const EnumVal &ev) const { + auto name = ev.name; + if (isupper(name.front())) { + std::transform(name.begin(), name.end(), name.begin(), CharToLower); + } + return EscapeKeyword(ConvertCase(name, Case::kLowerCamel)); + } + + // Also used by Kotlin, lol. + std::string LegacyJavaMethod2(const std::string &prefix, const StructDef &sd, + const std::string &suffix) const { + return prefix + sd.name + suffix; + } + + std::string LegacyKotlinVariant(EnumVal &ev) const { + // Namer assumes the input case is snake case which is wrong... + return ConvertCase(EscapeKeyword(ev.name), Case::kLowerCamel); + } + // Kotlin methods escapes keywords after case conversion but before + // prefixing and suffixing. + std::string LegacyKotlinMethod(const std::string &prefix, const FieldDef &d, + const std::string &suffix) const { + return prefix + ConvertCase(EscapeKeyword(d.name), Case::kUpperCamel) + + suffix; + } + std::string LegacyKotlinMethod(const std::string &prefix, const StructDef &d, + const std::string &suffix) const { + return prefix + ConvertCase(EscapeKeyword(d.name), Case::kUpperCamel) + + suffix; + } + + // This is a mix of snake case and keep casing, when Ts should be using + // lower camel case. + std::string LegacyTsMutateMethod(const FieldDef& d) { + return "mutate_" + d.name; + } + + std::string LegacyRustUnionTypeMethod(const FieldDef &d) { + // assert d is a union + return Method(d.name + "_type"); + } + + private: + std::string NamespacedString(const struct Namespace *ns, + const std::string &str) const { + std::string ret; + if (ns != nullptr) { ret += Namespace(ns->components); } + if (!ret.empty()) ret += config_.namespace_seperator; + return ret + str; + } +}; + +// This is a temporary helper function for code generators to call until all +// flag-overriding logic into flatc.cpp +inline Namer::Config WithFlagOptions(const Namer::Config &input, + const IDLOptions &opts, + const std::string &path) { + Namer::Config result = input; + result.object_prefix = opts.object_prefix; + result.object_suffix = opts.object_suffix; + result.output_path = path; + result.filename_suffix = opts.filename_suffix; + return result; +} + +} // namespace flatbuffers + +#endif // FLATBUFFERS_IDL_NAMER diff --git a/contrib/libs/flatbuffers/src/idl_parser.cpp b/contrib/libs/flatbuffers/src/idl_parser.cpp index ad642d79a9a..bddc736a5f8 100644 --- a/contrib/libs/flatbuffers/src/idl_parser.cpp +++ b/contrib/libs/flatbuffers/src/idl_parser.cpp @@ -16,11 +16,15 @@ #include <algorithm> #include <cmath> +#include <iostream> #include <list> #include <string> #include <utility> +#include "flatbuffers/base.h" +#include "flatbuffers/buffer.h" #include "flatbuffers/idl.h" +#include "flatbuffers/reflection_generated.h" #include "flatbuffers/util.h" namespace flatbuffers { @@ -35,28 +39,14 @@ const char *FLATBUFFERS_VERSION() { // clang-format on } -const double kPi = 3.14159265358979323846; +namespace { -// clang-format off -const char *const kTypeNames[] = { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, ...) \ - IDLTYPE, - FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) - #undef FLATBUFFERS_TD - nullptr -}; - -const char kTypeSizes[] = { - #define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \ - sizeof(CTYPE), - FLATBUFFERS_GEN_TYPES(FLATBUFFERS_TD) - #undef FLATBUFFERS_TD -}; -// clang-format on +static const double kPi = 3.14159265358979323846; // The enums in the reflection schema should match the ones we use internally. // Compare the last element to check if these go out of sync. -static_assert(BASE_TYPE_UNION == static_cast<BaseType>(reflection::Union), +static_assert(BASE_TYPE_VECTOR64 == + static_cast<BaseType>(reflection::MaxBaseType - 1), "enums don't match"); // Any parsing calls have to be wrapped in this macro, which automates @@ -92,117 +82,36 @@ static bool IsLowerSnakeCase(const std::string &str) { return true; } -// Convert an underscore_based_identifier in to camelCase. -// Also uppercases the first character if first is true. -std::string MakeCamel(const std::string &in, bool first) { - std::string s; - for (size_t i = 0; i < in.length(); i++) { - if (!i && first) - s += CharToUpper(in[0]); - else if (in[i] == '_' && i + 1 < in.length()) - s += CharToUpper(in[++i]); - else - s += in[i]; - } - return s; -} - -// Convert an underscore_based_identifier in to screaming snake case. -std::string MakeScreamingCamel(const std::string &in) { - std::string s; - for (size_t i = 0; i < in.length(); i++) { - if (in[i] != '_') - s += CharToUpper(in[i]); - else - s += in[i]; - } - return s; -} - -void DeserializeDoc(std::vector<std::string> &doc, - const Vector<Offset<String>> *documentation) { +static void DeserializeDoc(std::vector<std::string> &doc, + const Vector<Offset<String>> *documentation) { if (documentation == nullptr) return; for (uoffset_t index = 0; index < documentation->size(); index++) doc.push_back(documentation->Get(index)->str()); } -void Parser::Message(const std::string &msg) { - if (!error_.empty()) error_ += "\n"; // log all warnings and errors - error_ += file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : ""; - // clang-format off - - #ifdef _WIN32 // MSVC alike - error_ += - "(" + NumToString(line_) + ", " + NumToString(CursorPosition()) + ")"; - #else // gcc alike - if (file_being_parsed_.length()) error_ += ":"; - error_ += NumToString(line_) + ": " + NumToString(CursorPosition()); - #endif - // clang-format on - error_ += ": " + msg; -} - -void Parser::Warning(const std::string &msg) { - if (!opts.no_warnings) Message("warning: " + msg); -} - -CheckedError Parser::Error(const std::string &msg) { - Message("error: " + msg); - return CheckedError(true); -} - -inline CheckedError NoError() { return CheckedError(false); } +static CheckedError NoError() { return CheckedError(false); } -CheckedError Parser::RecurseError() { - return Error("maximum parsing depth " + NumToString(parse_depth_counter_) + - " reached"); -} - -class Parser::ParseDepthGuard { - public: - explicit ParseDepthGuard(Parser *parser_not_null) - : parser_(*parser_not_null), caller_depth_(parser_.parse_depth_counter_) { - FLATBUFFERS_ASSERT(caller_depth_ <= (FLATBUFFERS_MAX_PARSING_DEPTH) && - "Check() must be called to prevent stack overflow"); - parser_.parse_depth_counter_ += 1; - } - - ~ParseDepthGuard() { parser_.parse_depth_counter_ -= 1; } - - CheckedError Check() { - return caller_depth_ >= (FLATBUFFERS_MAX_PARSING_DEPTH) - ? parser_.RecurseError() - : CheckedError(false); - } - - FLATBUFFERS_DELETE_FUNC(ParseDepthGuard(const ParseDepthGuard &)); - FLATBUFFERS_DELETE_FUNC(ParseDepthGuard &operator=(const ParseDepthGuard &)); - - private: - Parser &parser_; - const int caller_depth_; -}; - -template<typename T> std::string TypeToIntervalString() { +template<typename T> static std::string TypeToIntervalString() { return "[" + NumToString((flatbuffers::numeric_limits<T>::lowest)()) + "; " + NumToString((flatbuffers::numeric_limits<T>::max)()) + "]"; } // atot: template version of atoi/atof: convert a string to an instance of T. template<typename T> -bool atot_scalar(const char *s, T *val, bool_constant<false>) { +static bool atot_scalar(const char *s, T *val, bool_constant<false>) { return StringToNumber(s, val); } template<typename T> -bool atot_scalar(const char *s, T *val, bool_constant<true>) { +static bool atot_scalar(const char *s, T *val, bool_constant<true>) { // Normalize NaN parsed from fbs or json to unsigned NaN. if (false == StringToNumber(s, val)) return false; *val = (*val != *val) ? std::fabs(*val) : *val; return true; } -template<typename T> CheckedError atot(const char *s, Parser &parser, T *val) { +template<typename T> +static CheckedError atot(const char *s, Parser &parser, T *val) { auto done = atot_scalar(s, val, bool_constant<is_floating_point<T>::value>()); if (done) return NoError(); if (0 == *val) @@ -212,33 +121,26 @@ template<typename T> CheckedError atot(const char *s, Parser &parser, T *val) { ", constant does not fit " + TypeToIntervalString<T>()); } template<> -inline CheckedError atot<Offset<void>>(const char *s, Parser &parser, - Offset<void> *val) { +CheckedError atot<Offset<void>>(const char *s, Parser &parser, + Offset<void> *val) { (void)parser; *val = Offset<void>(atoi(s)); return NoError(); } -std::string Namespace::GetFullyQualifiedName(const std::string &name, - size_t max_components) const { - // Early exit if we don't have a defined namespace. - if (components.empty() || !max_components) { return name; } - std::string stream_str; - for (size_t i = 0; i < std::min(components.size(), max_components); i++) { - stream_str += components[i]; - stream_str += '.'; - } - if (!stream_str.empty()) stream_str.pop_back(); - if (name.length()) { - stream_str += '.'; - stream_str += name; - } - return stream_str; +template<> +CheckedError atot<Offset64<void>>(const char *s, Parser &parser, + Offset64<void> *val) { + (void)parser; + *val = Offset64<void>(atoi(s)); + return NoError(); } template<typename T> -T *LookupTableByName(const SymbolTable<T> &table, const std::string &name, - const Namespace ¤t_namespace, size_t skip_top) { +static T *LookupTableByName(const SymbolTable<T> &table, + const std::string &name, + const Namespace ¤t_namespace, + size_t skip_top) { const auto &components = current_namespace.components; if (table.dict.empty()) return nullptr; if (components.size() < skip_top) return nullptr; @@ -297,6 +199,254 @@ static std::string TokenToString(int t) { } // clang-format on +static bool IsIdentifierStart(char c) { return is_alpha(c) || (c == '_'); } + +static bool CompareSerializedScalars(const uint8_t *a, const uint8_t *b, + const FieldDef &key) { + switch (key.value.type.base_type) { +#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \ + case BASE_TYPE_##ENUM: { \ + CTYPE def = static_cast<CTYPE>(0); \ + if (!a || !b) { StringToNumber(key.value.constant.c_str(), &def); } \ + const auto av = a ? ReadScalar<CTYPE>(a) : def; \ + const auto bv = b ? ReadScalar<CTYPE>(b) : def; \ + return av < bv; \ + } + FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD) +#undef FLATBUFFERS_TD + default: { + FLATBUFFERS_ASSERT(false && "scalar type expected"); + return false; + } + } +} + +static bool CompareTablesByScalarKey(const Offset<Table> *_a, + const Offset<Table> *_b, + const FieldDef &key) { + const voffset_t offset = key.value.offset; + // Indirect offset pointer to table pointer. + auto a = reinterpret_cast<const uint8_t *>(_a) + ReadScalar<uoffset_t>(_a); + auto b = reinterpret_cast<const uint8_t *>(_b) + ReadScalar<uoffset_t>(_b); + // Fetch field address from table. + a = reinterpret_cast<const Table *>(a)->GetAddressOf(offset); + b = reinterpret_cast<const Table *>(b)->GetAddressOf(offset); + return CompareSerializedScalars(a, b, key); +} + +static bool CompareTablesByStringKey(const Offset<Table> *_a, + const Offset<Table> *_b, + const FieldDef &key) { + const voffset_t offset = key.value.offset; + // Indirect offset pointer to table pointer. + auto a = reinterpret_cast<const uint8_t *>(_a) + ReadScalar<uoffset_t>(_a); + auto b = reinterpret_cast<const uint8_t *>(_b) + ReadScalar<uoffset_t>(_b); + // Fetch field address from table. + a = reinterpret_cast<const Table *>(a)->GetAddressOf(offset); + b = reinterpret_cast<const Table *>(b)->GetAddressOf(offset); + if (a && b) { + // Indirect offset pointer to string pointer. + a += ReadScalar<uoffset_t>(a); + b += ReadScalar<uoffset_t>(b); + return *reinterpret_cast<const String *>(a) < + *reinterpret_cast<const String *>(b); + } else { + return a ? true : false; + } +} + +static void SwapSerializedTables(Offset<Table> *a, Offset<Table> *b) { + // These are serialized offsets, so are relative where they are + // stored in memory, so compute the distance between these pointers: + ptrdiff_t diff = (b - a) * sizeof(Offset<Table>); + FLATBUFFERS_ASSERT(diff >= 0); // Guaranteed by SimpleQsort. + auto udiff = static_cast<uoffset_t>(diff); + a->o = EndianScalar(ReadScalar<uoffset_t>(a) - udiff); + b->o = EndianScalar(ReadScalar<uoffset_t>(b) + udiff); + std::swap(*a, *b); +} + +// See below for why we need our own sort :( +template<typename T, typename F, typename S> +static void SimpleQsort(T *begin, T *end, size_t width, F comparator, + S swapper) { + if (end - begin <= static_cast<ptrdiff_t>(width)) return; + auto l = begin + width; + auto r = end; + while (l < r) { + if (comparator(begin, l)) { + r -= width; + swapper(l, r); + } else { + l += width; + } + } + l -= width; + swapper(begin, l); + SimpleQsort(begin, l, width, comparator, swapper); + SimpleQsort(r, end, width, comparator, swapper); +} + +template<typename T> static inline void SingleValueRepack(Value &e, T val) { + // Remove leading zeros. + if (IsInteger(e.type.base_type)) { e.constant = NumToString(val); } +} + +#if defined(FLATBUFFERS_HAS_NEW_STRTOD) && (FLATBUFFERS_HAS_NEW_STRTOD > 0) +// Normalize defaults NaN to unsigned quiet-NaN(0) if value was parsed from +// hex-float literal. +static void SingleValueRepack(Value &e, float val) { + if (val != val) e.constant = "nan"; +} +static void SingleValueRepack(Value &e, double val) { + if (val != val) e.constant = "nan"; +} +#endif + +template<typename T> static uint64_t EnumDistanceImpl(T e1, T e2) { + if (e1 < e2) { std::swap(e1, e2); } // use std for scalars + // Signed overflow may occur, use unsigned calculation. + // The unsigned overflow is well-defined by C++ standard (modulo 2^n). + return static_cast<uint64_t>(e1) - static_cast<uint64_t>(e2); +} + +static bool compareFieldDefs(const FieldDef *a, const FieldDef *b) { + auto a_id = atoi(a->attributes.Lookup("id")->constant.c_str()); + auto b_id = atoi(b->attributes.Lookup("id")->constant.c_str()); + return a_id < b_id; +} + +static Namespace *GetNamespace( + const std::string &qualified_name, std::vector<Namespace *> &namespaces, + std::map<std::string, Namespace *> &namespaces_index) { + size_t dot = qualified_name.find_last_of('.'); + std::string namespace_name = (dot != std::string::npos) + ? std::string(qualified_name.c_str(), dot) + : ""; + Namespace *&ns = namespaces_index[namespace_name]; + + if (!ns) { + ns = new Namespace(); + namespaces.push_back(ns); + + size_t pos = 0; + + for (;;) { + dot = qualified_name.find('.', pos); + if (dot == std::string::npos) { break; } + ns->components.push_back(qualified_name.substr(pos, dot - pos)); + pos = dot + 1; + } + } + + return ns; +} + +// Generate a unique hash for a file based on its name and contents (if any). +static uint64_t HashFile(const char *source_filename, const char *source) { + uint64_t hash = 0; + + if (source_filename) + hash = HashFnv1a<uint64_t>(StripPath(source_filename).c_str()); + + if (source && *source) hash ^= HashFnv1a<uint64_t>(source); + + return hash; +} + +template<typename T> static bool compareName(const T *a, const T *b) { + return a->defined_namespace->GetFullyQualifiedName(a->name) < + b->defined_namespace->GetFullyQualifiedName(b->name); +} + +template<typename T> static void AssignIndices(const std::vector<T *> &defvec) { + // Pre-sort these vectors, such that we can set the correct indices for them. + auto vec = defvec; + std::sort(vec.begin(), vec.end(), compareName<T>); + for (int i = 0; i < static_cast<int>(vec.size()); i++) vec[i]->index = i; +} + +} // namespace + +void Parser::Message(const std::string &msg) { + if (!error_.empty()) error_ += "\n"; // log all warnings and errors + error_ += file_being_parsed_.length() ? AbsolutePath(file_being_parsed_) : ""; + // clang-format off + + #ifdef _WIN32 // MSVC alike + error_ += + "(" + NumToString(line_) + ", " + NumToString(CursorPosition()) + ")"; + #else // gcc alike + if (file_being_parsed_.length()) error_ += ":"; + error_ += NumToString(line_) + ": " + NumToString(CursorPosition()); + #endif + // clang-format on + error_ += ": " + msg; +} + +void Parser::Warning(const std::string &msg) { + if (!opts.no_warnings) { + Message("warning: " + msg); + has_warning_ = true; // for opts.warnings_as_errors + } +} + +CheckedError Parser::Error(const std::string &msg) { + Message("error: " + msg); + return CheckedError(true); +} + +CheckedError Parser::RecurseError() { + return Error("maximum parsing depth " + NumToString(parse_depth_counter_) + + " reached"); +} + +const std::string &Parser::GetPooledString(const std::string &s) const { + return *(string_cache_.insert(s).first); +} + +class Parser::ParseDepthGuard { + public: + explicit ParseDepthGuard(Parser *parser_not_null) + : parser_(*parser_not_null), caller_depth_(parser_.parse_depth_counter_) { + FLATBUFFERS_ASSERT(caller_depth_ <= (FLATBUFFERS_MAX_PARSING_DEPTH) && + "Check() must be called to prevent stack overflow"); + parser_.parse_depth_counter_ += 1; + } + + ~ParseDepthGuard() { parser_.parse_depth_counter_ -= 1; } + + CheckedError Check() { + return caller_depth_ >= (FLATBUFFERS_MAX_PARSING_DEPTH) + ? parser_.RecurseError() + : CheckedError(false); + } + + FLATBUFFERS_DELETE_FUNC(ParseDepthGuard(const ParseDepthGuard &)); + FLATBUFFERS_DELETE_FUNC(ParseDepthGuard &operator=(const ParseDepthGuard &)); + + private: + Parser &parser_; + const int caller_depth_; +}; + +std::string Namespace::GetFullyQualifiedName(const std::string &name, + size_t max_components) const { + // Early exit if we don't have a defined namespace. + if (components.empty() || !max_components) { return name; } + std::string stream_str; + for (size_t i = 0; i < std::min(components.size(), max_components); i++) { + stream_str += components[i]; + stream_str += '.'; + } + if (!stream_str.empty()) stream_str.pop_back(); + if (name.length()) { + stream_str += '.'; + stream_str += name; + } + return stream_str; +} + std::string Parser::TokenToStringId(int t) const { return t == kTokenIdentifier ? attribute_ : TokenToString(t); } @@ -326,12 +476,9 @@ CheckedError Parser::SkipByteOrderMark() { return NoError(); } -static inline bool IsIdentifierStart(char c) { - return is_alpha(c) || (c == '_'); -} - CheckedError Parser::Next() { doc_comment_.clear(); + prev_cursor_ = cursor_; bool seen_newline = cursor_ == source_; attribute_.clear(); attr_is_trivial_ascii_string_ = true; @@ -356,6 +503,8 @@ CheckedError Parser::Next() { case ')': case '[': case ']': + case '<': + case '>': case ',': case ':': case ';': @@ -508,10 +657,21 @@ CheckedError Parser::Next() { } const auto has_sign = (c == '+') || (c == '-'); - if (has_sign && IsIdentifierStart(*cursor_)) { - // '-'/'+' and following identifier - it could be a predefined - // constant. Return the sign in token_, see ParseSingleValue. - return NoError(); + if (has_sign) { + // Check for +/-inf which is considered a float constant. + if (strncmp(cursor_, "inf", 3) == 0 && + !(IsIdentifierStart(cursor_[3]) || is_digit(cursor_[3]))) { + attribute_.assign(cursor_ - 1, cursor_ + 3); + token_ = kTokenFloatConstant; + cursor_ += 3; + return NoError(); + } + + if (IsIdentifierStart(*cursor_)) { + // '-'/'+' and following identifier - it could be a predefined + // constant. Return the sign in token_, see ParseSingleValue. + return NoError(); + } } auto dot_lvl = @@ -753,6 +913,15 @@ CheckedError Parser::ParseField(StructDef &struct_def) { ECHECK(ParseType(type)); if (struct_def.fixed) { + if (IsIncompleteStruct(type) || + (IsArray(type) && IsIncompleteStruct(type.VectorType()))) { + std::string type_name = IsArray(type) ? type.VectorType().struct_def->name + : type.struct_def->name; + return Error( + std::string("Incomplete type in struct is not allowed, type name: ") + + type_name); + } + auto valid = IsScalar(type.base_type) || IsStruct(type); if (!valid && IsArray(type)) { const auto &elem_type = type.VectorType(); @@ -799,6 +968,14 @@ CheckedError Parser::ParseField(StructDef &struct_def) { FieldDef *field; ECHECK(AddField(struct_def, name, type, &field)); + if (typefield) { + // We preserve the relation between the typefield + // and field, so we can easily map it in the code + // generators. + typefield->sibling_union_field = field; + field->sibling_union_field = typefield; + } + if (token_ == '=') { NEXT(); ECHECK(ParseSingleValue(&field->name, field->value, true)); @@ -808,8 +985,7 @@ CheckedError Parser::ParseField(StructDef &struct_def) { "or in structs."); if (IsString(type) || IsVector(type)) { advanced_features_ |= reflection::DefaultVectorsAndStrings; - if (field->value.constant != "0" && field->value.constant != "null" && - !SupportsDefaultVectorsAndStrings()) { + if (field->value.constant != "0" && !SupportsDefaultVectorsAndStrings()) { return Error( "Default values for strings and vectors are not supported in one " "of the specified programming languages"); @@ -872,6 +1048,65 @@ CheckedError Parser::ParseField(StructDef &struct_def) { } } + if (field->attributes.Lookup("vector64") != nullptr) { + if (!IsVector(type)) { + return Error("`vector64` attribute can only be applied on vectors."); + } + + // Upgrade the type to be a BASE_TYPE_VECTOR64, since the attributes are + // parsed after the type. + const BaseType element_base_type = type.element; + type = Type(BASE_TYPE_VECTOR64, type.struct_def, type.enum_def); + type.element = element_base_type; + + // Since the field was already added to the parent object, update the type + // in place. + field->value.type = type; + + // 64-bit vectors imply the offset64 attribute. + field->offset64 = true; + } + + // Record that this field uses 64-bit offsets. + if (field->attributes.Lookup("offset64") != nullptr) { + // TODO(derekbailey): would be nice to have this be a recommendation or hint + // instead of a warning. + if (type.base_type == BASE_TYPE_VECTOR64) { + Warning("attribute `vector64` implies `offset64` and isn't required."); + } + + field->offset64 = true; + } + + // Check for common conditions with Offset64 fields. + if (field->offset64) { + // TODO(derekbailey): this is where we can disable string support for + // offset64, as that is not a hard requirement to have. + if (!IsString(type) && !IsVector(type)) { + return Error( + "only string and vectors can have `offset64` attribute applied"); + } + + // If this is a Vector, only scalar and scalar-like (structs) items are + // allowed. + // TODO(derekbailey): allow vector of strings, just require that the strings + // are Offset64<string>. + if (IsVector(type) && + !((IsScalar(type.element) && !IsEnum(type.VectorType())) || + IsStruct(type.VectorType()))) { + return Error("only vectors of scalars are allowed to be 64-bit."); + } + + // Lastly, check if it is supported by the specified generated languages. Do + // this last so the above checks can inform the user of schema errors to fix + // first. + if (!Supports64BitOffsets()) { + return Error( + "fields using 64-bit offsets are not yet supported in at least one " + "of the specified programming languages."); + } + } + // For historical convenience reasons, string keys are assumed required. // Scalars are kDefault unless otherwise specified. // Nonscalars are kOptional unless required; @@ -894,8 +1129,16 @@ CheckedError Parser::ParseField(StructDef &struct_def) { if (field->key) { if (struct_def.has_key) return Error("only one field may be set as 'key'"); struct_def.has_key = true; - if (!IsScalar(type.base_type) && !IsString(type)) { - return Error("'key' field must be string or scalar type"); + auto is_valid = + IsScalar(type.base_type) || IsString(type) || IsStruct(type); + if (IsArray(type)) { + is_valid |= + IsScalar(type.VectorType().base_type) || IsStruct(type.VectorType()); + } + if (!is_valid) { + return Error( + "'key' field must be string, scalar type or fixed size array of " + "scalars"); } } @@ -914,7 +1157,7 @@ CheckedError Parser::ParseField(StructDef &struct_def) { } if (!SupportsOptionalScalars()) { return Error( - "Optional scalars are not yet supported in at least one the of " + "Optional scalars are not yet supported in at least one of " "the specified programming languages."); } } @@ -978,15 +1221,19 @@ CheckedError Parser::ParseField(StructDef &struct_def) { "definition"); field->native_inline = field->attributes.Lookup("native_inline") != nullptr; - if (field->native_inline && !IsStruct(field->value.type)) - return Error("native_inline can only be defined on structs"); + if (field->native_inline && !IsStruct(field->value.type) && + !IsVectorOfStruct(field->value.type) && + !IsVectorOfTable(field->value.type)) + return Error( + "'native_inline' can only be defined on structs, vector of structs or " + "vector of tables"); auto nested = field->attributes.Lookup("nested_flatbuffer"); if (nested) { if (nested->type.base_type != BASE_TYPE_STRING) return Error( "nested_flatbuffer attribute must be a string (the root type)"); - if (type.base_type != BASE_TYPE_VECTOR || type.element != BASE_TYPE_UCHAR) + if (!IsVector(type.base_type) || type.element != BASE_TYPE_UCHAR) return Error( "nested_flatbuffer attribute may only apply to a vector of ubyte"); // This will cause an error if the root type of the nested flatbuffer @@ -1055,7 +1302,7 @@ CheckedError Parser::ParseComma() { CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field, size_t parent_fieldn, const StructDef *parent_struct_def, - uoffset_t count, bool inside_vector) { + size_t count, bool inside_vector) { switch (val.type.base_type) { case BASE_TYPE_UNION: { FLATBUFFERS_ASSERT(field); @@ -1121,7 +1368,11 @@ CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field, } uint8_t enum_idx; if (vector_of_union_types) { - enum_idx = vector_of_union_types->Get(count); + if (vector_of_union_types->size() <= count) + return Error( + "union types vector smaller than union values vector for: " + + field->name); + enum_idx = vector_of_union_types->Get(static_cast<uoffset_t>(count)); } else { ECHECK(atot(constant.c_str(), *this, &enum_idx)); } @@ -1150,9 +1401,10 @@ CheckedError Parser::ParseAnyValue(Value &val, FieldDef *field, ECHECK(ParseString(val, field->shared)); break; } + case BASE_TYPE_VECTOR64: case BASE_TYPE_VECTOR: { uoffset_t off; - ECHECK(ParseVector(val.type.VectorType(), &off, field, parent_fieldn)); + ECHECK(ParseVector(val.type, &off, field, parent_fieldn)); val.constant = NumToString(off); break; } @@ -1324,6 +1576,9 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, for (size_t size = struct_def.sortbysize ? sizeof(largest_scalar_t) : 1; size; size /= 2) { // Go through elements in reverse, since we're building the data backwards. + // TODO(derekbailey): this doesn't work when there are Offset64 fields, as + // those have to be built first. So this needs to be changed to iterate over + // Offset64 then Offset32 fields. for (auto it = field_stack_.rbegin(); it != field_stack_.rbegin() + fieldn_outer; ++it) { auto &field_value = it->first; @@ -1340,10 +1595,18 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ builder_.PushElement(val); \ } else { \ - CTYPE val, valdef; \ - ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ - ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \ - builder_.AddElement(field_value.offset, val, valdef); \ + if (field->IsScalarOptional()) { \ + if (field_value.constant != "null") { \ + CTYPE val; \ + ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ + builder_.AddElement(field_value.offset, val); \ + } \ + } else { \ + CTYPE val, valdef; \ + ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ + ECHECK(atot(field->value.constant.c_str(), *this, &valdef)); \ + builder_.AddElement(field_value.offset, val, valdef); \ + } \ } \ break; FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD) @@ -1354,9 +1617,16 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, if (IsStruct(field->value.type)) { \ SerializeStruct(*field->value.type.struct_def, field_value); \ } else { \ - CTYPE val; \ - ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ - builder_.AddOffset(field_value.offset, val); \ + /* Special case for fields that use 64-bit addressing */ \ + if(field->offset64) { \ + Offset64<void> offset; \ + ECHECK(atot(field_value.constant.c_str(), *this, &offset)); \ + builder_.AddOffset(field_value.offset, offset); \ + } else { \ + CTYPE val; \ + ECHECK(atot(field_value.constant.c_str(), *this, &val)); \ + builder_.AddOffset(field_value.offset, val); \ + } \ } \ break; FLATBUFFERS_GEN_TYPES_POINTER(FLATBUFFERS_TD) @@ -1394,7 +1664,7 @@ CheckedError Parser::ParseTable(const StructDef &struct_def, std::string *value, } template<typename F> -CheckedError Parser::ParseVectorDelimiters(uoffset_t &count, F body) { +CheckedError Parser::ParseVectorDelimiters(size_t &count, F body) { EXPECT('['); for (;;) { if ((!opts.strict_json || !count) && Is(']')) break; @@ -1407,91 +1677,6 @@ CheckedError Parser::ParseVectorDelimiters(uoffset_t &count, F body) { return NoError(); } -static bool CompareSerializedScalars(const uint8_t *a, const uint8_t *b, - const FieldDef &key) { - switch (key.value.type.base_type) { -#define FLATBUFFERS_TD(ENUM, IDLTYPE, CTYPE, ...) \ - case BASE_TYPE_##ENUM: { \ - CTYPE def = static_cast<CTYPE>(0); \ - if (!a || !b) { StringToNumber(key.value.constant.c_str(), &def); } \ - const auto av = a ? ReadScalar<CTYPE>(a) : def; \ - const auto bv = b ? ReadScalar<CTYPE>(b) : def; \ - return av < bv; \ - } - FLATBUFFERS_GEN_TYPES_SCALAR(FLATBUFFERS_TD) -#undef FLATBUFFERS_TD - default: { - FLATBUFFERS_ASSERT(false && "scalar type expected"); - return false; - } - } -} - -static bool CompareTablesByScalarKey(const Offset<Table> *_a, - const Offset<Table> *_b, - const FieldDef &key) { - const voffset_t offset = key.value.offset; - // Indirect offset pointer to table pointer. - auto a = reinterpret_cast<const uint8_t *>(_a) + ReadScalar<uoffset_t>(_a); - auto b = reinterpret_cast<const uint8_t *>(_b) + ReadScalar<uoffset_t>(_b); - // Fetch field address from table. - a = reinterpret_cast<const Table *>(a)->GetAddressOf(offset); - b = reinterpret_cast<const Table *>(b)->GetAddressOf(offset); - return CompareSerializedScalars(a, b, key); -} - -static bool CompareTablesByStringKey(const Offset<Table> *_a, - const Offset<Table> *_b, - const FieldDef &key) { - const voffset_t offset = key.value.offset; - // Indirect offset pointer to table pointer. - auto a = reinterpret_cast<const uint8_t *>(_a) + ReadScalar<uoffset_t>(_a); - auto b = reinterpret_cast<const uint8_t *>(_b) + ReadScalar<uoffset_t>(_b); - // Fetch field address from table. - a = reinterpret_cast<const Table *>(a)->GetAddressOf(offset); - b = reinterpret_cast<const Table *>(b)->GetAddressOf(offset); - if (a && b) { - // Indirect offset pointer to string pointer. - a += ReadScalar<uoffset_t>(a); - b += ReadScalar<uoffset_t>(b); - return *reinterpret_cast<const String *>(a) < - *reinterpret_cast<const String *>(b); - } else { - return a ? true : false; - } -} - -static void SwapSerializedTables(Offset<Table> *a, Offset<Table> *b) { - // These are serialized offsets, so are relative where they are - // stored in memory, so compute the distance between these pointers: - ptrdiff_t diff = (b - a) * sizeof(Offset<Table>); - FLATBUFFERS_ASSERT(diff >= 0); // Guaranteed by SimpleQsort. - auto udiff = static_cast<uoffset_t>(diff); - a->o = EndianScalar(ReadScalar<uoffset_t>(a) - udiff); - b->o = EndianScalar(ReadScalar<uoffset_t>(b) + udiff); - std::swap(*a, *b); -} - -// See below for why we need our own sort :( -template<typename T, typename F, typename S> -void SimpleQsort(T *begin, T *end, size_t width, F comparator, S swapper) { - if (end - begin <= static_cast<ptrdiff_t>(width)) return; - auto l = begin + width; - auto r = end; - while (l < r) { - if (comparator(begin, l)) { - r -= width; - swapper(l, r); - } else { - l += width; - } - } - l -= width; - swapper(begin, l); - SimpleQsort(begin, l, width, comparator, swapper); - SimpleQsort(r, end, width, comparator, swapper); -} - CheckedError Parser::ParseAlignAttribute(const std::string &align_constant, size_t min_align, size_t *align) { // Use uint8_t to avoid problems with size_t==`unsigned long` on LP64. @@ -1509,10 +1694,11 @@ CheckedError Parser::ParseAlignAttribute(const std::string &align_constant, NumToString(FLATBUFFERS_MAX_ALIGNMENT)); } -CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue, +CheckedError Parser::ParseVector(const Type &vector_type, uoffset_t *ovalue, FieldDef *field, size_t fieldn) { - uoffset_t count = 0; - auto err = ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError { + Type type = vector_type.VectorType(); + size_t count = 0; + auto err = ParseVectorDelimiters(count, [&](size_t &) -> CheckedError { Value val; val.type = type; ECHECK(ParseAnyValue(val, field, fieldn, nullptr, count, true)); @@ -1521,6 +1707,7 @@ CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue, }); ECHECK(err); + const size_t alignment = InlineAlignment(type); const size_t len = count * InlineSize(type) / InlineAlignment(type); const size_t elemsize = InlineAlignment(type); const auto force_align = field->attributes.Lookup("force_align"); @@ -1530,8 +1717,15 @@ CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue, if (align > 1) { builder_.ForceVectorAlignment(len, elemsize, align); } } - builder_.StartVector(len, elemsize); - for (uoffset_t i = 0; i < count; i++) { + // TODO Fix using element alignment as size (`elemsize`)! + if (vector_type.base_type == BASE_TYPE_VECTOR64) { + // TODO(derekbailey): this requires a 64-bit builder. + // builder_.StartVector<Offset64, uoffset64_t>(len, elemsize, alignment); + builder_.StartVector(len, elemsize, alignment); + } else { + builder_.StartVector(len, elemsize, alignment); + } + for (size_t i = 0; i < count; i++) { // start at the back, since we're building the data backwards. auto &val = field_stack_.back().first; switch (val.type.base_type) { @@ -1553,7 +1747,11 @@ CheckedError Parser::ParseVector(const Type &type, uoffset_t *ovalue, } builder_.ClearOffsets(); - *ovalue = builder_.EndVector(count); + if (vector_type.base_type == BASE_TYPE_VECTOR64) { + *ovalue = builder_.EndVector<uoffset64_t>(count); + } else { + *ovalue = builder_.EndVector(count); + } if (type.base_type == BASE_TYPE_STRUCT && type.struct_def->has_key) { // We should sort this vector. Find the key first. @@ -1621,9 +1819,9 @@ CheckedError Parser::ParseArray(Value &array) { FlatBufferBuilder builder; const auto &type = array.type.VectorType(); auto length = array.type.fixed_length; - uoffset_t count = 0; - auto err = ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError { - vector_emplace_back(&stack, Value()); + size_t count = 0; + auto err = ParseVectorDelimiters(count, [&](size_t &) -> CheckedError { + stack.emplace_back(Value()); auto &val = stack.back(); val.type = type; if (IsStruct(type)) { @@ -1667,7 +1865,13 @@ CheckedError Parser::ParseNestedFlatbuffer(Value &val, FieldDef *field, size_t fieldn, const StructDef *parent_struct_def) { if (token_ == '[') { // backwards compat for 'legacy' ubyte buffers - ECHECK(ParseAnyValue(val, field, fieldn, parent_struct_def, 0)); + if (opts.json_nested_legacy_flatbuffers) { + ECHECK(ParseAnyValue(val, field, fieldn, parent_struct_def, 0)); + } else { + return Error( + "cannot parse nested_flatbuffer as bytes unless" + " --json-nested-bytes is set"); + } } else { auto cursor_at_value_begin = cursor_; ECHECK(SkipAnyJsonValue()); @@ -1812,22 +2016,6 @@ CheckedError Parser::TokenError() { return Error("cannot parse value starting with: " + TokenToStringId(token_)); } -// Re-pack helper (ParseSingleValue) to normalize defaults of scalars. -template<typename T> inline void SingleValueRepack(Value &e, T val) { - // Remove leading zeros. - if (IsInteger(e.type.base_type)) { e.constant = NumToString(val); } -} -#if defined(FLATBUFFERS_HAS_NEW_STRTOD) && (FLATBUFFERS_HAS_NEW_STRTOD > 0) -// Normalize defaults NaN to unsigned quiet-NaN(0) if value was parsed from -// hex-float literal. -static inline void SingleValueRepack(Value &e, float val) { - if (val != val) e.constant = "nan"; -} -static inline void SingleValueRepack(Value &e, double val) { - if (val != val) e.constant = "nan"; -} -#endif - CheckedError Parser::ParseFunction(const std::string *name, Value &e) { ParseDepthGuard depth_guard(this); ECHECK(depth_guard.Check()); @@ -1836,8 +2024,8 @@ CheckedError Parser::ParseFunction(const std::string *name, Value &e) { const auto functionname = attribute_; if (!IsFloat(e.type.base_type)) { return Error(functionname + ": type of argument mismatch, expecting: " + - kTypeNames[BASE_TYPE_DOUBLE] + - ", found: " + kTypeNames[e.type.base_type] + + TypeName(BASE_TYPE_DOUBLE) + + ", found: " + TypeName(e.type.base_type) + ", name: " + (name ? *name : "") + ", value: " + e.constant); } NEXT(); @@ -1883,8 +2071,7 @@ CheckedError Parser::TryTypedValue(const std::string *name, int dtoken, e.type.base_type = req; } else { return Error(std::string("type mismatch: expecting: ") + - kTypeNames[e.type.base_type] + - ", found: " + kTypeNames[req] + + TypeName(e.type.base_type) + ", found: " + TypeName(req) + ", name: " + (name ? *name : "") + ", value: " + e.constant); } } @@ -1945,7 +2132,7 @@ CheckedError Parser::ParseSingleValue(const std::string *name, Value &e, return Error( std::string("type mismatch or invalid value, an initializer of " "non-string field must be trivial ASCII string: type: ") + - kTypeNames[in_type] + ", name: " + (name ? *name : "") + + TypeName(in_type) + ", name: " + (name ? *name : "") + ", value: " + attribute_); } @@ -2017,7 +2204,7 @@ CheckedError Parser::ParseSingleValue(const std::string *name, Value &e, if (!match) { std::string msg; msg += "Cannot assign token starting with '" + TokenToStringId(token_) + - "' to value of <" + std::string(kTypeNames[in_type]) + "> type."; + "' to value of <" + std::string(TypeName(in_type)) + "> type."; return Error(msg); } const auto match_type = e.type.base_type; // may differ from in_type @@ -2099,13 +2286,6 @@ const EnumVal *EnumDef::MaxValue() const { return vals.vec.empty() ? nullptr : vals.vec.back(); } -template<typename T> static uint64_t EnumDistanceImpl(T e1, T e2) { - if (e1 < e2) { std::swap(e1, e2); } // use std for scalars - // Signed overflow may occur, use unsigned calculation. - // The unsigned overflow is well-defined by C++ standard (modulo 2^n). - return static_cast<uint64_t>(e1) - static_cast<uint64_t>(e2); -} - uint64_t EnumDef::Distance(const EnumVal *v1, const EnumVal *v2) const { return IsUInt64() ? EnumDistanceImpl(v1->GetAsUInt64(), v2->GetAsUInt64()) : EnumDistanceImpl(v1->GetAsInt64(), v2->GetAsInt64()); @@ -2148,10 +2328,14 @@ void EnumDef::SortByValue() { auto &v = vals.vec; if (IsUInt64()) std::sort(v.begin(), v.end(), [](const EnumVal *e1, const EnumVal *e2) { + if (e1->GetAsUInt64() == e2->GetAsUInt64()) { + return e1->name < e2->name; + } return e1->GetAsUInt64() < e2->GetAsUInt64(); }); else std::sort(v.begin(), v.end(), [](const EnumVal *e1, const EnumVal *e2) { + if (e1->GetAsInt64() == e2->GetAsInt64()) { return e1->name < e2->name; } return e1->GetAsInt64() < e2->GetAsInt64(); }); } @@ -2281,13 +2465,18 @@ struct EnumValBuilder { bool user_value; }; -CheckedError Parser::ParseEnum(const bool is_union, EnumDef **dest) { +CheckedError Parser::ParseEnum(const bool is_union, EnumDef **dest, + const char *filename) { std::vector<std::string> enum_comment = doc_comment_; NEXT(); std::string enum_name = attribute_; EXPECT(kTokenIdentifier); EnumDef *enum_def; ECHECK(StartEnum(enum_name, is_union, &enum_def)); + if (filename != nullptr && !opts.project_root.empty()) { + enum_def->declaration_file = + &GetPooledString(RelativeToRootPath(opts.project_root, filename)); + } enum_def->doc_comment = enum_comment; if (!is_union && !opts.proto_mode) { // Give specialized error message, since this type spec used to @@ -2314,6 +2503,9 @@ CheckedError Parser::ParseEnum(const bool is_union, EnumDef **dest) { // todo: Convert to the Error in the future? Warning("underlying type of bit_flags enum must be unsigned"); } + if (enum_def->attributes.Lookup("force_align")) { + return Error("`force_align` is not a valid attribute for Enums. "); + } EnumValBuilder evb(*this, *enum_def); EXPECT('{'); // A lot of code generatos expect that an enum is not-empty. @@ -2360,14 +2552,17 @@ CheckedError Parser::ParseEnum(const bool is_union, EnumDef **dest) { EXPECT(kTokenIntegerConstant); } - ECHECK(evb.AcceptEnumerator()); - if (opts.proto_mode && Is('[')) { NEXT(); // ignore attributes on enums. while (token_ != ']') NEXT(); NEXT(); + } else { + // parse attributes in fbs schema + ECHECK(ParseMetaData(&ev.attributes)); } + + ECHECK(evb.AcceptEnumerator()); } if (!Is(opts.proto_mode ? ';' : ',')) break; NEXT(); @@ -2407,14 +2602,18 @@ CheckedError Parser::ParseEnum(const bool is_union, EnumDef **dest) { } if (dest) *dest = enum_def; - types_.Add(current_namespace_->GetFullyQualifiedName(enum_def->name), - new Type(BASE_TYPE_UNION, nullptr, enum_def)); + const auto qualified_name = + current_namespace_->GetFullyQualifiedName(enum_def->name); + if (types_.Add(qualified_name, new Type(BASE_TYPE_UNION, nullptr, enum_def))) + return Error("datatype already exists: " + qualified_name); return NoError(); } CheckedError Parser::StartStruct(const std::string &name, StructDef **dest) { auto &struct_def = *LookupCreateStruct(name, true, true); - if (!struct_def.predecl) return Error("datatype already exists: " + name); + if (!struct_def.predecl) + return Error("datatype already exists: " + + current_namespace_->GetFullyQualifiedName(name)); struct_def.predecl = false; struct_def.name = name; struct_def.file = file_being_parsed_; @@ -2446,11 +2645,20 @@ CheckedError Parser::CheckClash(std::vector<FieldDef *> &fields, return NoError(); } +std::vector<IncludedFile> Parser::GetIncludedFiles() const { + const auto it = files_included_per_file_.find(file_being_parsed_); + if (it == files_included_per_file_.end()) { return {}; } + + return { it->second.cbegin(), it->second.cend() }; +} + bool Parser::SupportsOptionalScalars(const flatbuffers::IDLOptions &opts) { static FLATBUFFERS_CONSTEXPR unsigned long supported_langs = IDLOptions::kRust | IDLOptions::kSwift | IDLOptions::kLobster | IDLOptions::kKotlin | IDLOptions::kCpp | IDLOptions::kJava | - IDLOptions::kCSharp | IDLOptions::kTs | IDLOptions::kBinary; + IDLOptions::kCSharp | IDLOptions::kTs | IDLOptions::kBinary | + IDLOptions::kGo | IDLOptions::kPython | IDLOptions::kJson | + IDLOptions::kNim; unsigned long langs = opts.lang_to_generate; return (langs > 0 && langs < IDLOptions::kMAX) && !(langs & ~supported_langs); } @@ -2461,23 +2669,28 @@ bool Parser::SupportsOptionalScalars() const { bool Parser::SupportsDefaultVectorsAndStrings() const { static FLATBUFFERS_CONSTEXPR unsigned long supported_langs = - IDLOptions::kRust | IDLOptions::kSwift; + IDLOptions::kRust | IDLOptions::kSwift | IDLOptions::kNim; return !(opts.lang_to_generate & ~supported_langs); } bool Parser::SupportsAdvancedUnionFeatures() const { - return opts.lang_to_generate != 0 && - (opts.lang_to_generate & + return (opts.lang_to_generate & ~(IDLOptions::kCpp | IDLOptions::kTs | IDLOptions::kPhp | IDLOptions::kJava | IDLOptions::kCSharp | IDLOptions::kKotlin | - IDLOptions::kBinary | IDLOptions::kSwift)) == 0; + IDLOptions::kBinary | IDLOptions::kSwift | IDLOptions::kNim | + IDLOptions::kJson)) == 0; } bool Parser::SupportsAdvancedArrayFeatures() const { return (opts.lang_to_generate & ~(IDLOptions::kCpp | IDLOptions::kPython | IDLOptions::kJava | IDLOptions::kCSharp | IDLOptions::kJsonSchema | IDLOptions::kJson | - IDLOptions::kBinary | IDLOptions::kRust)) == 0; + IDLOptions::kBinary | IDLOptions::kRust | IDLOptions::kTs)) == 0; +} + +bool Parser::Supports64BitOffsets() const { + return (opts.lang_to_generate & + ~(IDLOptions::kCpp | IDLOptions::kJson | IDLOptions::kBinary)) == 0; } Namespace *Parser::UniqueNamespace(Namespace *ns) { @@ -2506,13 +2719,7 @@ std::string Parser::UnqualifiedName(const std::string &full_qualified_name) { return full_qualified_name.substr(previous, current - previous); } -static bool compareFieldDefs(const FieldDef *a, const FieldDef *b) { - auto a_id = atoi(a->attributes.Lookup("id")->constant.c_str()); - auto b_id = atoi(b->attributes.Lookup("id")->constant.c_str()); - return a_id < b_id; -} - -CheckedError Parser::ParseDecl() { +CheckedError Parser::ParseDecl(const char *filename) { std::vector<std::string> dc = doc_comment_; bool fixed = IsIdent("struct"); if (!fixed && !IsIdent("table")) return Error("declaration expected"); @@ -2523,6 +2730,10 @@ CheckedError Parser::ParseDecl() { ECHECK(StartStruct(name, &struct_def)); struct_def->doc_comment = dc; struct_def->fixed = fixed; + if (filename && !opts.project_root.empty()) { + struct_def->declaration_file = + &GetPooledString(RelativeToRootPath(opts.project_root, filename)); + } ECHECK(ParseMetaData(&struct_def->attributes)); struct_def->sortbysize = struct_def->attributes.Lookup("original_order") == nullptr && !fixed; @@ -2568,6 +2779,7 @@ CheckedError Parser::ParseDecl() { for (voffset_t i = 0; i < static_cast<voffset_t>(fields.size()); i++) { auto &field = *fields[i]; const auto &id_str = field.attributes.Lookup("id")->constant; + // Metadata values have a dynamic type, they can be `float`, 'int', or // 'string`. // The FieldIndexToOffset(i) expects the voffset_t so `id` is limited by @@ -2594,12 +2806,15 @@ CheckedError Parser::ParseDecl() { ECHECK(CheckClash(fields, struct_def, "_byte_vector", BASE_TYPE_STRING)); ECHECK(CheckClash(fields, struct_def, "ByteVector", BASE_TYPE_STRING)); EXPECT('}'); - types_.Add(current_namespace_->GetFullyQualifiedName(struct_def->name), - new Type(BASE_TYPE_STRUCT, struct_def, nullptr)); + const auto qualified_name = + current_namespace_->GetFullyQualifiedName(struct_def->name); + if (types_.Add(qualified_name, + new Type(BASE_TYPE_STRUCT, struct_def, nullptr))) + return Error("datatype already exists: " + qualified_name); return NoError(); } -CheckedError Parser::ParseService() { +CheckedError Parser::ParseService(const char *filename) { std::vector<std::string> service_comment = doc_comment_; NEXT(); auto service_name = attribute_; @@ -2609,6 +2824,10 @@ CheckedError Parser::ParseService() { service_def.file = file_being_parsed_; service_def.doc_comment = service_comment; service_def.defined_namespace = current_namespace_; + if (filename != nullptr && !opts.project_root.empty()) { + service_def.declaration_file = + &GetPooledString(RelativeToRootPath(opts.project_root, filename)); + } if (services_.Add(current_namespace_->GetFullyQualifiedName(service_name), &service_def)) return Error("service already exists: " + service_name); @@ -2724,7 +2943,7 @@ CheckedError Parser::ParseProtoDecl() { } else if (IsIdent("enum")) { // These are almost the same, just with different terminator: EnumDef *enum_def; - ECHECK(ParseEnum(false, &enum_def)); + ECHECK(ParseEnum(false, &enum_def, nullptr)); if (Is(';')) NEXT(); // Temp: remove any duplicates, as .fbs files can't handle them. enum_def->RemoveDuplicates(); @@ -2747,17 +2966,17 @@ CheckedError Parser::ParseProtoDecl() { return NoError(); } -CheckedError Parser::StartEnum(const std::string &enum_name, bool is_union, +CheckedError Parser::StartEnum(const std::string &name, bool is_union, EnumDef **dest) { auto &enum_def = *new EnumDef(); - enum_def.name = enum_name; + enum_def.name = name; enum_def.file = file_being_parsed_; enum_def.doc_comment = doc_comment_; enum_def.is_union = is_union; enum_def.defined_namespace = current_namespace_; - if (enums_.Add(current_namespace_->GetFullyQualifiedName(enum_name), - &enum_def)) - return Error("enum already exists: " + enum_name); + const auto qualified_name = current_namespace_->GetFullyQualifiedName(name); + if (enums_.Add(qualified_name, &enum_def)) + return Error("enum already exists: " + qualified_name); enum_def.underlying_type.base_type = is_union ? BASE_TYPE_UTYPE : BASE_TYPE_INT; enum_def.underlying_type.enum_def = &enum_def; @@ -2784,9 +3003,43 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend, ECHECK(ParseProtoOption()); EXPECT(';'); } else if (IsIdent("reserved")) { // Skip these. + /** + * Reserved proto ids can be comma seperated (e.g. 1,2,4,5;) + * or range based (e.g. 9 to 11;) + * or combination of them (e.g. 1,2,9 to 11,4,5;) + * It will be ended by a semicolon. + */ NEXT(); - while (!Is(';')) { NEXT(); } // A variety of formats, just skip. + bool range = false; + voffset_t from = 0; + + while (!Is(';')) { + if (token_ == kTokenIntegerConstant) { + voffset_t attribute = 0; + bool done = StringToNumber(attribute_.c_str(), &attribute); + if (!done) + return Error("Protobuf has non positive number in reserved ids"); + + if (range) { + for (voffset_t id = from + 1; id <= attribute; id++) + struct_def->reserved_ids.push_back(id); + + range = false; + } else { + struct_def->reserved_ids.push_back(attribute); + } + + from = attribute; + } + + if (attribute_ == "to") range = true; + + NEXT(); + } // A variety of formats, just skip. + NEXT(); + } else if (IsIdent("map")) { + ECHECK(ParseProtoMapField(struct_def)); } else { std::vector<std::string> field_comment = doc_comment_; // Parse the qualifier. @@ -2816,7 +3069,7 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend, if (IsIdent("group") || oneof) { if (!oneof) NEXT(); if (oneof && opts.proto_oneof_union) { - auto name = MakeCamel(attribute_, true) + "Union"; + auto name = ConvertCase(attribute_, Case::kUpperCamel) + "Union"; ECHECK(StartEnum(name, true, &oneof_union)); type = Type(BASE_TYPE_UNION, nullptr, oneof_union); } else { @@ -2841,11 +3094,13 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend, } std::string name = attribute_; EXPECT(kTokenIdentifier); + std::string proto_field_id; if (!oneof) { // Parse the field id. Since we're just translating schemas, not // any kind of binary compatibility, we can safely ignore these, and // assign our own. EXPECT('='); + proto_field_id = attribute_; EXPECT(kTokenIntegerConstant); } FieldDef *field = nullptr; @@ -2856,6 +3111,11 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend, } if (!field) ECHECK(AddField(*struct_def, name, type, &field)); field->doc_comment = field_comment; + if (!proto_field_id.empty() || oneof) { + auto val = new Value(); + val->constant = proto_field_id; + field->attributes.Add("id", val); + } if (!IsScalar(type.base_type) && required) { field->presence = FieldDef::kRequired; } @@ -2871,7 +3131,11 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend, if (key == "default") { // Temp: skip non-numeric and non-boolean defaults (enums). auto numeric = strpbrk(val.c_str(), "0123456789-+."); - if (IsScalar(type.base_type) && numeric == val.c_str()) { + if (IsFloat(type.base_type) && + (val == "inf" || val == "+inf" || val == "-inf")) { + // Prefer to be explicit with +inf. + field->value.constant = val == "inf" ? "+inf" : val; + } else if (IsScalar(type.base_type) && numeric == val.c_str()) { field->value.constant = val; } else if (val == "true") { field->value.constant = val; @@ -2917,6 +3181,47 @@ CheckedError Parser::ParseProtoFields(StructDef *struct_def, bool isextend, return NoError(); } +CheckedError Parser::ParseProtoMapField(StructDef *struct_def) { + NEXT(); + EXPECT('<'); + Type key_type; + ECHECK(ParseType(key_type)); + EXPECT(','); + Type value_type; + ECHECK(ParseType(value_type)); + EXPECT('>'); + auto field_name = attribute_; + NEXT(); + EXPECT('='); + std::string proto_field_id = attribute_; + EXPECT(kTokenIntegerConstant); + EXPECT(';'); + + auto entry_table_name = ConvertCase(field_name, Case::kUpperCamel) + "Entry"; + StructDef *entry_table; + ECHECK(StartStruct(entry_table_name, &entry_table)); + entry_table->has_key = true; + FieldDef *key_field; + ECHECK(AddField(*entry_table, "key", key_type, &key_field)); + key_field->key = true; + FieldDef *value_field; + ECHECK(AddField(*entry_table, "value", value_type, &value_field)); + + Type field_type; + field_type.base_type = BASE_TYPE_VECTOR; + field_type.element = BASE_TYPE_STRUCT; + field_type.struct_def = entry_table; + FieldDef *field; + ECHECK(AddField(*struct_def, field_name, field_type, &field)); + if (!proto_field_id.empty()) { + auto val = new Value(); + val->constant = proto_field_id; + field->attributes.Add("id", val); + } + + return NoError(); +} + CheckedError Parser::ParseProtoKey() { if (token_ == '(') { NEXT(); @@ -3010,16 +3315,16 @@ CheckedError Parser::SkipAnyJsonValue() { }); } case '[': { - uoffset_t count = 0; - return ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError { - return SkipAnyJsonValue(); - }); + size_t count = 0; + return ParseVectorDelimiters( + count, [&](size_t &) -> CheckedError { return SkipAnyJsonValue(); }); } case kTokenStringConstant: case kTokenIntegerConstant: case kTokenFloatConstant: NEXT(); break; default: - if (IsIdent("true") || IsIdent("false") || IsIdent("null")) { + if (IsIdent("true") || IsIdent("false") || IsIdent("null") || + IsIdent("inf")) { NEXT(); } else return TokenError(); @@ -3061,8 +3366,8 @@ CheckedError Parser::ParseFlexBufferValue(flexbuffers::Builder *builder) { } case '[': { auto start = builder->StartVector(); - uoffset_t count = 0; - ECHECK(ParseVectorDelimiters(count, [&](uoffset_t &) -> CheckedError { + size_t count = 0; + ECHECK(ParseVectorDelimiters(count, [&](size_t &) -> CheckedError { return ParseFlexBufferValue(builder); })); builder->EndVector(start, false, false); @@ -3150,6 +3455,10 @@ bool Parser::ParseJson(const char *json, const char *json_filename) { return done; } +std::ptrdiff_t Parser::BytesConsumed() const { + return std::distance(source_, prev_cursor_); +} + CheckedError Parser::StartParseFile(const char *source, const char *source_filename) { file_being_parsed_ = source_filename ? source_filename : ""; @@ -3232,30 +3541,76 @@ CheckedError Parser::ParseRoot(const char *source, const char **include_paths, for (auto val_it = enum_def.Vals().begin(); val_it != enum_def.Vals().end(); ++val_it) { auto &val = **val_it; - if (!SupportsAdvancedUnionFeatures() && + + if (!(opts.lang_to_generate != 0 && SupportsAdvancedUnionFeatures()) && (IsStruct(val.union_type) || IsString(val.union_type))) + return Error( "only tables can be union elements in the generated language: " + val.name); } } } + + auto err = CheckPrivateLeak(); + if (err.Check()) return err; + // Parse JSON object only if the scheme has been parsed. if (token_ == '{') { ECHECK(DoParseJson()); } - EXPECT(kTokenEof); return NoError(); } -// Generate a unique hash for a file based on its name and contents (if any). -static uint64_t HashFile(const char *source_filename, const char *source) { - uint64_t hash = 0; - - if (source_filename) - hash = HashFnv1a<uint64_t>(StripPath(source_filename).c_str()); - - if (source && *source) hash ^= HashFnv1a<uint64_t>(source); +CheckedError Parser::CheckPrivateLeak() { + if (!opts.no_leak_private_annotations) return NoError(); + // Iterate over all structs/tables to validate we arent leaking + // any private (structs/tables/enums) + for (auto it = structs_.vec.begin(); it != structs_.vec.end(); it++) { + auto &struct_def = **it; + for (auto fld_it = struct_def.fields.vec.begin(); + fld_it != struct_def.fields.vec.end(); ++fld_it) { + auto &field = **fld_it; + + if (field.value.type.enum_def) { + auto err = + CheckPrivatelyLeakedFields(struct_def, *field.value.type.enum_def); + if (err.Check()) { return err; } + } else if (field.value.type.struct_def) { + auto err = CheckPrivatelyLeakedFields(struct_def, + *field.value.type.struct_def); + if (err.Check()) { return err; } + } + } + } + // Iterate over all enums to validate we arent leaking + // any private (structs/tables) + for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) { + auto &enum_def = **it; + if (enum_def.is_union) { + for (auto val_it = enum_def.Vals().begin(); + val_it != enum_def.Vals().end(); ++val_it) { + auto &val = **val_it; + if (val.union_type.struct_def) { + auto err = + CheckPrivatelyLeakedFields(enum_def, *val.union_type.struct_def); + if (err.Check()) { return err; } + } + } + } + } + return NoError(); +} - return hash; +CheckedError Parser::CheckPrivatelyLeakedFields(const Definition &def, + const Definition &value_type) { + if (!opts.no_leak_private_annotations) return NoError(); + const auto is_private = def.attributes.Lookup("private"); + const auto is_field_private = value_type.attributes.Lookup("private"); + if (!is_private && is_field_private) { + return Error( + "Leaking private implementation, verify all objects have similar " + "annotations"); + } + return NoError(); } CheckedError Parser::DoParse(const char *source, const char **include_paths, @@ -3272,7 +3627,7 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths, if (included_files_.find(source_hash) == included_files_.end()) { included_files_[source_hash] = include_filename ? include_filename : ""; - files_included_per_file_[source_filename] = std::set<std::string>(); + files_included_per_file_[source_filename] = std::set<IncludedFile>(); } else { return NoError(); } @@ -3296,7 +3651,7 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths, ECHECK(ParseProtoDecl()); } else if (IsIdent("native_include")) { NEXT(); - vector_emplace_back(&native_included_files_, attribute_); + native_included_files_.emplace_back(attribute_); EXPECT(kTokenStringConstant); EXPECT(';'); } else if (IsIdent("include") || (opts.proto_mode && IsIdent("import"))) { @@ -3320,8 +3675,12 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths, } if (filepath.empty()) return Error("unable to locate include file: " + name); - if (source_filename) - files_included_per_file_[source_filename].insert(filepath); + if (source_filename) { + IncludedFile included_file; + included_file.filename = filepath; + included_file.schema_name = name; + files_included_per_file_[source_filename].insert(included_file); + } std::string contents; bool file_loaded = LoadFile(filepath.c_str(), true, &contents); @@ -3364,9 +3723,9 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths, } else if (token_ == '{') { return NoError(); } else if (IsIdent("enum")) { - ECHECK(ParseEnum(false, nullptr)); + ECHECK(ParseEnum(false, nullptr, source_filename)); } else if (IsIdent("union")) { - ECHECK(ParseEnum(true, nullptr)); + ECHECK(ParseEnum(true, nullptr, source_filename)); } else if (IsIdent("root_type")) { NEXT(); auto root_type = attribute_; @@ -3382,9 +3741,9 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths, NEXT(); file_identifier_ = attribute_; EXPECT(kTokenStringConstant); - if (file_identifier_.length() != FlatBufferBuilder::kFileIdentifierLength) + if (file_identifier_.length() != flatbuffers::kFileIdentifierLength) return Error("file_identifier must be exactly " + - NumToString(FlatBufferBuilder::kFileIdentifierLength) + + NumToString(flatbuffers::kFileIdentifierLength) + " characters"); EXPECT(';'); } else if (IsIdent("file_extension")) { @@ -3405,11 +3764,15 @@ CheckedError Parser::DoParse(const char *source, const char **include_paths, EXPECT(';'); known_attributes_[name] = false; } else if (IsIdent("rpc_service")) { - ECHECK(ParseService()); + ECHECK(ParseService(source_filename)); } else { - ECHECK(ParseDecl()); + ECHECK(ParseDecl(source_filename)); } } + EXPECT(kTokenEof); + if (opts.warnings_as_errors && has_warning_) { + return Error("treating warnings as errors, failed due to above warnings"); + } return NoError(); } @@ -3433,9 +3796,11 @@ CheckedError Parser::DoParseJson() { : nullptr); } } - // Check that JSON file doesn't contain more objects or IDL directives. - // Comments after JSON are allowed. - EXPECT(kTokenEof); + if (opts.require_json_eof) { + // Check that JSON file doesn't contain more objects or IDL directives. + // Comments after JSON are allowed. + EXPECT(kTokenEof); + } return NoError(); } @@ -3454,11 +3819,11 @@ std::set<std::string> Parser::GetIncludedFilesRecursive( // Workaround the lack of const accessor in C++98 maps. auto &new_files = - (*const_cast<std::map<std::string, std::set<std::string>> *>( + (*const_cast<std::map<std::string, std::set<IncludedFile>> *>( &files_included_per_file_))[current]; for (auto it = new_files.begin(); it != new_files.end(); ++it) { - if (included_files.find(*it) == included_files.end()) - to_process.push_back(*it); + if (included_files.find(it->filename) == included_files.end()) + to_process.push_back(it->filename); } } @@ -3467,16 +3832,42 @@ std::set<std::string> Parser::GetIncludedFilesRecursive( // Schema serialization functionality: -template<typename T> bool compareName(const T *a, const T *b) { - return a->defined_namespace->GetFullyQualifiedName(a->name) < - b->defined_namespace->GetFullyQualifiedName(b->name); +static flatbuffers::Offset< + flatbuffers::Vector<flatbuffers::Offset<reflection::KeyValue>>> +SerializeAttributesCommon(const SymbolTable<Value> &attributes, + FlatBufferBuilder *builder, const Parser &parser) { + std::vector<flatbuffers::Offset<reflection::KeyValue>> attrs; + for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) { + auto it = parser.known_attributes_.find(kv->first); + FLATBUFFERS_ASSERT(it != parser.known_attributes_.end()); + if (parser.opts.binary_schema_builtins || !it->second) { + auto key = builder->CreateString(kv->first); + auto val = builder->CreateString(kv->second->constant); + attrs.push_back(reflection::CreateKeyValue(*builder, key, val)); + } + } + if (attrs.size()) { + return builder->CreateVectorOfSortedTables(&attrs); + } else { + return 0; + } } -template<typename T> void AssignIndices(const std::vector<T *> &defvec) { - // Pre-sort these vectors, such that we can set the correct indices for them. - auto vec = defvec; - std::sort(vec.begin(), vec.end(), compareName<T>); - for (int i = 0; i < static_cast<int>(vec.size()); i++) vec[i]->index = i; +static bool DeserializeAttributesCommon( + SymbolTable<Value> &attributes, Parser &parser, + const Vector<Offset<reflection::KeyValue>> *attrs) { + if (attrs == nullptr) return true; + for (uoffset_t i = 0; i < attrs->size(); ++i) { + auto kv = attrs->Get(i); + auto value = new Value(); + if (kv->value()) { value->constant = kv->value()->str(); } + if (attributes.Add(kv->key()->str(), value)) { + delete value; + return false; + } + parser.known_attributes_[kv->key()->str()]; + } + return true; } void Parser::Serialize() { @@ -3484,32 +3875,63 @@ void Parser::Serialize() { AssignIndices(structs_.vec); AssignIndices(enums_.vec); std::vector<Offset<reflection::Object>> object_offsets; + std::set<std::string> files; for (auto it = structs_.vec.begin(); it != structs_.vec.end(); ++it) { auto offset = (*it)->Serialize(&builder_, *this); object_offsets.push_back(offset); (*it)->serialized_location = offset.o; + const std::string *file = (*it)->declaration_file; + if (file) files.insert(*file); } std::vector<Offset<reflection::Enum>> enum_offsets; for (auto it = enums_.vec.begin(); it != enums_.vec.end(); ++it) { auto offset = (*it)->Serialize(&builder_, *this); enum_offsets.push_back(offset); - (*it)->serialized_location = offset.o; + const std::string *file = (*it)->declaration_file; + if (file) files.insert(*file); } std::vector<Offset<reflection::Service>> service_offsets; for (auto it = services_.vec.begin(); it != services_.vec.end(); ++it) { auto offset = (*it)->Serialize(&builder_, *this); service_offsets.push_back(offset); - (*it)->serialized_location = offset.o; + const std::string *file = (*it)->declaration_file; + if (file) files.insert(*file); + } + + // Create Schemafiles vector of tables. + flatbuffers::Offset< + flatbuffers::Vector<flatbuffers::Offset<reflection::SchemaFile>>> + schema_files__; + if (!opts.project_root.empty()) { + std::vector<Offset<reflection::SchemaFile>> schema_files; + std::vector<Offset<flatbuffers::String>> included_files; + for (auto f = files_included_per_file_.begin(); + f != files_included_per_file_.end(); f++) { + const auto filename__ = builder_.CreateSharedString( + RelativeToRootPath(opts.project_root, f->first)); + for (auto i = f->second.begin(); i != f->second.end(); i++) { + included_files.push_back(builder_.CreateSharedString( + RelativeToRootPath(opts.project_root, i->filename))); + } + const auto included_files__ = builder_.CreateVector(included_files); + included_files.clear(); + + schema_files.push_back( + reflection::CreateSchemaFile(builder_, filename__, included_files__)); + } + schema_files__ = builder_.CreateVectorOfSortedTables(&schema_files); } - auto objs__ = builder_.CreateVectorOfSortedTables(&object_offsets); - auto enum__ = builder_.CreateVectorOfSortedTables(&enum_offsets); - auto fiid__ = builder_.CreateString(file_identifier_); - auto fext__ = builder_.CreateString(file_extension_); - auto serv__ = builder_.CreateVectorOfSortedTables(&service_offsets); - auto schema_offset = reflection::CreateSchema( + + const auto objs__ = builder_.CreateVectorOfSortedTables(&object_offsets); + const auto enum__ = builder_.CreateVectorOfSortedTables(&enum_offsets); + const auto fiid__ = builder_.CreateString(file_identifier_); + const auto fext__ = builder_.CreateString(file_extension_); + const auto serv__ = builder_.CreateVectorOfSortedTables(&service_offsets); + const auto schema_offset = reflection::CreateSchema( builder_, objs__, enum__, fiid__, fext__, (root_struct_def_ ? root_struct_def_->serialized_location : 0), serv__, - static_cast<reflection::AdvancedFeatures>(advanced_features_)); + static_cast<reflection::AdvancedFeatures>(advanced_features_), + schema_files__); if (opts.size_prefixed) { builder_.FinishSizePrefixed(schema_offset, reflection::SchemaIdentifier()); } else { @@ -3517,32 +3939,6 @@ void Parser::Serialize() { } } -static Namespace *GetNamespace( - const std::string &qualified_name, std::vector<Namespace *> &namespaces, - std::map<std::string, Namespace *> &namespaces_index) { - size_t dot = qualified_name.find_last_of('.'); - std::string namespace_name = (dot != std::string::npos) - ? std::string(qualified_name.c_str(), dot) - : ""; - Namespace *&ns = namespaces_index[namespace_name]; - - if (!ns) { - ns = new Namespace(); - namespaces.push_back(ns); - - size_t pos = 0; - - for (;;) { - dot = qualified_name.find('.', pos); - if (dot == std::string::npos) { break; } - ns->components.push_back(qualified_name.substr(pos, dot - pos)); - pos = dot + 1; - } - } - - return ns; -} - Offset<reflection::Object> StructDef::Serialize(FlatBufferBuilder *builder, const Parser &parser) const { std::vector<Offset<reflection::Field>> field_offsets; @@ -3550,16 +3946,18 @@ Offset<reflection::Object> StructDef::Serialize(FlatBufferBuilder *builder, field_offsets.push_back((*it)->Serialize( builder, static_cast<uint16_t>(it - fields.vec.begin()), parser)); } - auto qualified_name = defined_namespace->GetFullyQualifiedName(name); - auto name__ = builder->CreateString(qualified_name); - auto flds__ = builder->CreateVectorOfSortedTables(&field_offsets); - auto attr__ = SerializeAttributes(builder, parser); - auto docs__ = parser.opts.binary_schema_comments - ? builder->CreateVectorOfStrings(doc_comment) - : 0; - return reflection::CreateObject(*builder, name__, flds__, fixed, - static_cast<int>(minalign), - static_cast<int>(bytesize), attr__, docs__); + const auto qualified_name = defined_namespace->GetFullyQualifiedName(name); + const auto name__ = builder->CreateString(qualified_name); + const auto flds__ = builder->CreateVectorOfSortedTables(&field_offsets); + const auto attr__ = SerializeAttributes(builder, parser); + const auto docs__ = parser.opts.binary_schema_comments && !doc_comment.empty() + ? builder->CreateVectorOfStrings(doc_comment) + : 0; + std::string decl_file_in_project = declaration_file ? *declaration_file : ""; + const auto file__ = builder->CreateSharedString(decl_file_in_project); + return reflection::CreateObject( + *builder, name__, flds__, fixed, static_cast<int>(minalign), + static_cast<int>(bytesize), attr__, docs__, file__); } bool StructDef::Deserialize(Parser &parser, const reflection::Object *object) { @@ -3580,6 +3978,14 @@ bool StructDef::Deserialize(Parser &parser, const reflection::Object *object) { delete field_def; return false; } + if (field_def->key) { + if (has_key) { + // only one field may be set as key + delete field_def; + return false; + } + has_key = true; + } if (fixed) { // Recompute padding since that's currently not serialized. auto size = InlineSize(field_def->value.type); @@ -3602,7 +4008,7 @@ Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder, auto name__ = builder->CreateString(name); auto type__ = value.type.Serialize(builder); auto attr__ = SerializeAttributes(builder, parser); - auto docs__ = parser.opts.binary_schema_comments + auto docs__ = parser.opts.binary_schema_comments && !doc_comment.empty() ? builder->CreateVectorOfStrings(doc_comment) : 0; double d; @@ -3613,7 +4019,7 @@ Offset<reflection::Field> FieldDef::Serialize(FlatBufferBuilder *builder, IsInteger(value.type.base_type) ? StringToInt(value.constant.c_str()) : 0, // result may be platform-dependent if underlying is float (not double) IsFloat(value.type.base_type) ? d : 0.0, deprecated, IsRequired(), key, - attr__, docs__, IsOptional()); + attr__, docs__, IsOptional(), static_cast<uint16_t>(padding), offset64); // TODO: value.constant is almost always "0", we could save quite a bit of // space by sharing it. Same for common values of value.type. } @@ -3626,10 +4032,12 @@ bool FieldDef::Deserialize(Parser &parser, const reflection::Field *field) { if (IsInteger(value.type.base_type)) { value.constant = NumToString(field->default_integer()); } else if (IsFloat(value.type.base_type)) { - value.constant = FloatToString(field->default_real(), 16); + value.constant = FloatToString(field->default_real(), 17); } presence = FieldDef::MakeFieldPresence(field->optional(), field->required()); + padding = field->padding(); key = field->key(); + offset64 = field->offset64(); if (!DeserializeAttributes(parser, field->attributes())) return false; // TODO: this should probably be handled by a separate attribute if (attributes.Lookup("flexbuffer")) { @@ -3654,7 +4062,7 @@ Offset<reflection::RPCCall> RPCCall::Serialize(FlatBufferBuilder *builder, const Parser &parser) const { auto name__ = builder->CreateString(name); auto attr__ = SerializeAttributes(builder, parser); - auto docs__ = parser.opts.binary_schema_comments + auto docs__ = parser.opts.binary_schema_comments && !doc_comment.empty() ? builder->CreateVectorOfStrings(doc_comment) : 0; return reflection::CreateRPCCall( @@ -3678,14 +4086,17 @@ Offset<reflection::Service> ServiceDef::Serialize(FlatBufferBuilder *builder, for (auto it = calls.vec.begin(); it != calls.vec.end(); ++it) { servicecall_offsets.push_back((*it)->Serialize(builder, parser)); } - auto qualified_name = defined_namespace->GetFullyQualifiedName(name); - auto name__ = builder->CreateString(qualified_name); - auto call__ = builder->CreateVector(servicecall_offsets); - auto attr__ = SerializeAttributes(builder, parser); - auto docs__ = parser.opts.binary_schema_comments - ? builder->CreateVectorOfStrings(doc_comment) - : 0; - return reflection::CreateService(*builder, name__, call__, attr__, docs__); + const auto qualified_name = defined_namespace->GetFullyQualifiedName(name); + const auto name__ = builder->CreateString(qualified_name); + const auto call__ = builder->CreateVector(servicecall_offsets); + const auto attr__ = SerializeAttributes(builder, parser); + const auto docs__ = parser.opts.binary_schema_comments && !doc_comment.empty() + ? builder->CreateVectorOfStrings(doc_comment) + : 0; + std::string decl_file_in_project = declaration_file ? *declaration_file : ""; + const auto file__ = builder->CreateSharedString(decl_file_in_project); + return reflection::CreateService(*builder, name__, call__, attr__, docs__, + file__); } bool ServiceDef::Deserialize(Parser &parser, @@ -3712,16 +4123,18 @@ Offset<reflection::Enum> EnumDef::Serialize(FlatBufferBuilder *builder, for (auto it = vals.vec.begin(); it != vals.vec.end(); ++it) { enumval_offsets.push_back((*it)->Serialize(builder, parser)); } - auto qualified_name = defined_namespace->GetFullyQualifiedName(name); - auto name__ = builder->CreateString(qualified_name); - auto vals__ = builder->CreateVector(enumval_offsets); - auto type__ = underlying_type.Serialize(builder); - auto attr__ = SerializeAttributes(builder, parser); - auto docs__ = parser.opts.binary_schema_comments - ? builder->CreateVectorOfStrings(doc_comment) - : 0; + const auto qualified_name = defined_namespace->GetFullyQualifiedName(name); + const auto name__ = builder->CreateString(qualified_name); + const auto vals__ = builder->CreateVector(enumval_offsets); + const auto type__ = underlying_type.Serialize(builder); + const auto attr__ = SerializeAttributes(builder, parser); + const auto docs__ = parser.opts.binary_schema_comments && !doc_comment.empty() + ? builder->CreateVectorOfStrings(doc_comment) + : 0; + std::string decl_file_in_project = declaration_file ? *declaration_file : ""; + const auto file__ = builder->CreateSharedString(decl_file_in_project); return reflection::CreateEnum(*builder, name__, vals__, is_union, type__, - attr__, docs__); + attr__, docs__, file__); } bool EnumDef::Deserialize(Parser &parser, const reflection::Enum *_enum) { @@ -3743,34 +4156,52 @@ bool EnumDef::Deserialize(Parser &parser, const reflection::Enum *_enum) { return true; } +flatbuffers::Offset< + flatbuffers::Vector<flatbuffers::Offset<reflection::KeyValue>>> +EnumVal::SerializeAttributes(FlatBufferBuilder *builder, + const Parser &parser) const { + return SerializeAttributesCommon(attributes, builder, parser); +} + +bool EnumVal::DeserializeAttributes( + Parser &parser, const Vector<Offset<reflection::KeyValue>> *attrs) { + return DeserializeAttributesCommon(attributes, parser, attrs); +} + Offset<reflection::EnumVal> EnumVal::Serialize(FlatBufferBuilder *builder, const Parser &parser) const { - auto name__ = builder->CreateString(name); - auto type__ = union_type.Serialize(builder); - auto docs__ = parser.opts.binary_schema_comments - ? builder->CreateVectorOfStrings(doc_comment) - : 0; - return reflection::CreateEnumVal( - *builder, name__, value, - union_type.struct_def ? union_type.struct_def->serialized_location : 0, - type__, docs__); + const auto name__ = builder->CreateString(name); + const auto type__ = union_type.Serialize(builder); + const auto attr__ = SerializeAttributes(builder, parser); + const auto docs__ = parser.opts.binary_schema_comments && !doc_comment.empty() + ? builder->CreateVectorOfStrings(doc_comment) + : 0; + return reflection::CreateEnumVal(*builder, name__, value, type__, docs__, + attr__); } -bool EnumVal::Deserialize(const Parser &parser, - const reflection::EnumVal *val) { +bool EnumVal::Deserialize(Parser &parser, const reflection::EnumVal *val) { name = val->name()->str(); value = val->value(); if (!union_type.Deserialize(parser, val->union_type())) return false; + if (!DeserializeAttributes(parser, val->attributes())) return false; DeserializeDoc(doc_comment, val->documentation()); return true; } Offset<reflection::Type> Type::Serialize(FlatBufferBuilder *builder) const { + size_t element_size = SizeOf(element); + if (base_type == BASE_TYPE_VECTOR && element == BASE_TYPE_STRUCT && + struct_def->bytesize != 0) { + // struct_def->bytesize==0 means struct is table + element_size = struct_def->bytesize; + } return reflection::CreateType( *builder, static_cast<reflection::BaseType>(base_type), static_cast<reflection::BaseType>(element), struct_def ? struct_def->index : (enum_def ? enum_def->index : -1), - fixed_length); + fixed_length, static_cast<uint32_t>(SizeOf(base_type)), + static_cast<uint32_t>(element_size)); } bool Type::Deserialize(const Parser &parser, const reflection::Type *type) { @@ -3804,37 +4235,12 @@ flatbuffers::Offset< flatbuffers::Vector<flatbuffers::Offset<reflection::KeyValue>>> Definition::SerializeAttributes(FlatBufferBuilder *builder, const Parser &parser) const { - std::vector<flatbuffers::Offset<reflection::KeyValue>> attrs; - for (auto kv = attributes.dict.begin(); kv != attributes.dict.end(); ++kv) { - auto it = parser.known_attributes_.find(kv->first); - FLATBUFFERS_ASSERT(it != parser.known_attributes_.end()); - if (parser.opts.binary_schema_builtins || !it->second) { - auto key = builder->CreateString(kv->first); - auto val = builder->CreateString(kv->second->constant); - attrs.push_back(reflection::CreateKeyValue(*builder, key, val)); - } - } - if (attrs.size()) { - return builder->CreateVectorOfSortedTables(&attrs); - } else { - return 0; - } + return SerializeAttributesCommon(attributes, builder, parser); } bool Definition::DeserializeAttributes( Parser &parser, const Vector<Offset<reflection::KeyValue>> *attrs) { - if (attrs == nullptr) return true; - for (uoffset_t i = 0; i < attrs->size(); ++i) { - auto kv = attrs->Get(i); - auto value = new Value(); - if (kv->value()) { value->constant = kv->value()->str(); } - if (attributes.Add(kv->key()->str(), value)) { - delete value; - return false; - } - parser.known_attributes_[kv->key()->str()]; - } - return true; + return DeserializeAttributesCommon(attributes, parser, attrs); } /************************************************************************/ @@ -3927,6 +4333,18 @@ bool Parser::Deserialize(const reflection::Schema *schema) { } } advanced_features_ = schema->advanced_features(); + + if (schema->fbs_files()) + for (auto s = schema->fbs_files()->begin(); s != schema->fbs_files()->end(); + ++s) { + for (auto f = s->included_filenames()->begin(); + f != s->included_filenames()->end(); ++f) { + IncludedFile included_file; + included_file.filename = f->str(); + files_included_per_file_[s->filename()->str()].insert(included_file); + } + } + return true; } @@ -3937,17 +4355,25 @@ std::string Parser::ConformTo(const Parser &base) { struct_def.defined_namespace->GetFullyQualifiedName(struct_def.name); auto struct_def_base = base.LookupStruct(qualified_name); if (!struct_def_base) continue; + std::set<FieldDef *> renamed_fields; for (auto fit = struct_def.fields.vec.begin(); fit != struct_def.fields.vec.end(); ++fit) { auto &field = **fit; auto field_base = struct_def_base->fields.Lookup(field.name); + const auto qualified_field_name = qualified_name + "." + field.name; if (field_base) { - if (field.value.offset != field_base->value.offset) - return "offsets differ for field: " + field.name; - if (field.value.constant != field_base->value.constant) - return "defaults differ for field: " + field.name; - if (!EqualByName(field.value.type, field_base->value.type)) - return "types differ for field: " + field.name; + if (field.value.offset != field_base->value.offset) { + return "offsets differ for field: " + qualified_field_name; + } + if (field.value.constant != field_base->value.constant) { + return "defaults differ for field: " + qualified_field_name; + } + if (!EqualByName(field.value.type, field_base->value.type)) { + return "types differ for field: " + qualified_field_name; + } + if (field.offset64 != field_base->offset64) { + return "offset types differ for field: " + qualified_field_name; + } } else { // Doesn't have to exist, deleting fields is fine. // But we should check if there is a field that has the same offset @@ -3956,14 +4382,33 @@ std::string Parser::ConformTo(const Parser &base) { fbit != struct_def_base->fields.vec.end(); ++fbit) { field_base = *fbit; if (field.value.offset == field_base->value.offset) { - if (!EqualByName(field.value.type, field_base->value.type)) - return "field renamed to different type: " + field.name; + renamed_fields.insert(field_base); + if (!EqualByName(field.value.type, field_base->value.type)) { + const auto qualified_field_base = + qualified_name + "." + field_base->name; + return "field renamed to different type: " + + qualified_field_name + " (renamed from " + + qualified_field_base + ")"; + } break; } } } } + // deletion of trailing fields are not allowed + for (auto fit = struct_def_base->fields.vec.begin(); + fit != struct_def_base->fields.vec.end(); ++fit) { + auto &field_base = **fit; + // not a renamed field + if (renamed_fields.find(&field_base) == renamed_fields.end()) { + auto field = struct_def.fields.Lookup(field_base.name); + if (!field) { + return "field deleted: " + qualified_name + "." + field_base.name; + } + } + } } + for (auto eit = enums_.vec.begin(); eit != enums_.vec.end(); ++eit) { auto &enum_def = **eit; auto qualified_name = diff --git a/contrib/libs/flatbuffers/src/namer.h b/contrib/libs/flatbuffers/src/namer.h new file mode 100644 index 00000000000..6a7fadcd14b --- /dev/null +++ b/contrib/libs/flatbuffers/src/namer.h @@ -0,0 +1,266 @@ +#ifndef FLATBUFFERS_NAMER +#define FLATBUFFERS_NAMER + +#include "flatbuffers/util.h" + +namespace flatbuffers { + +// Options for Namer::File. +enum class SkipFile { + None = 0, + Suffix = 1, + Extension = 2, + SuffixAndExtension = 3, +}; +inline SkipFile operator&(SkipFile a, SkipFile b) { + return static_cast<SkipFile>(static_cast<int>(a) & static_cast<int>(b)); +} +// Options for Namer::Directories +enum class SkipDir { + None = 0, + // Skip prefixing the -o $output_path. + OutputPath = 1, + // Skip trailing path seperator. + TrailingPathSeperator = 2, + OutputPathAndTrailingPathSeparator = 3, +}; +inline SkipDir operator&(SkipDir a, SkipDir b) { + return static_cast<SkipDir>(static_cast<int>(a) & static_cast<int>(b)); +} + +// `Namer` applies style configuration to symbols in generated code. It manages +// casing, escapes keywords, and object API naming. +// TODO: Refactor all code generators to use this. +class Namer { + public: + struct Config { + // Symbols in code. + + // Case style for flatbuffers-defined types. + // e.g. `class TableA {}` + Case types; + // Case style for flatbuffers-defined constants. + // e.g. `uint64_t ENUM_A_MAX`; + Case constants; + // Case style for flatbuffers-defined methods. + // e.g. `class TableA { int field_a(); }` + Case methods; + // Case style for flatbuffers-defined functions. + // e.g. `TableA* get_table_a_root()`; + Case functions; + // Case style for flatbuffers-defined fields. + // e.g. `struct Struct { int my_field; }` + Case fields; + // Case style for flatbuffers-defined variables. + // e.g. `int my_variable = 2` + Case variables; + // Case style for flatbuffers-defined variants. + // e.g. `enum class Enum { MyVariant, }` + Case variants; + // Seperator for qualified enum names. + // e.g. `Enum::MyVariant` uses `::`. + std::string enum_variant_seperator; + + // Configures, when formatting code, whether symbols are checked against + // keywords and escaped before or after case conversion. It does not make + // sense to do so before, but its legacy behavior. :shrug: + // TODO(caspern): Deprecate. + enum class Escape { + BeforeConvertingCase, + AfterConvertingCase, + }; + Escape escape_keywords; + + // Namespaces + + // e.g. `namespace my_namespace {}` + Case namespaces; + // The seperator between namespaces in a namespace path. + std::string namespace_seperator; + + // Object API. + // Native versions flatbuffers types have this prefix. + // e.g. "" (it's usually empty string) + std::string object_prefix; + // Native versions flatbuffers types have this suffix. + // e.g. "T" + std::string object_suffix; + + // Keywords. + // Prefix used to escape keywords. It is usually empty string. + std::string keyword_prefix; + // Suffix used to escape keywords. It is usually "_". + std::string keyword_suffix; + + // Files. + + // Case style for filenames. e.g. `foo_bar_generated.rs` + Case filenames; + // Case style for directories, e.g. `output_files/foo_bar/baz/` + Case directories; + // The directory within which we will generate files. + std::string output_path; + // Suffix for generated file names, e.g. "_generated". + std::string filename_suffix; + // Extension for generated files, e.g. ".cpp" or ".rs". + std::string filename_extension; + }; + Namer(Config config, std::set<std::string> keywords) + : config_(config), keywords_(std::move(keywords)) {} + + virtual ~Namer() {} + + template<typename T> std::string Method(const T &s) const { + return Method(s.name); + } + + virtual std::string Method(const std::string &pre, + const std::string &mid, + const std::string &suf) const { + return Format(pre + "_" + mid + "_" + suf, config_.methods); + } + virtual std::string Method(const std::string &pre, + const std::string &suf) const { + return Format(pre + "_" + suf, config_.methods); + } + virtual std::string Method(const std::string &s) const { + return Format(s, config_.methods); + } + + virtual std::string Constant(const std::string &s) const { + return Format(s, config_.constants); + } + + virtual std::string Function(const std::string &s) const { + return Format(s, config_.functions); + } + + virtual std::string Variable(const std::string &s) const { + return Format(s, config_.variables); + } + + template<typename T> + std::string Variable(const std::string &p, const T &s) const { + return Format(p + "_" + s.name, config_.variables); + } + virtual std::string Variable(const std::string &p, + const std::string &s) const { + return Format(p + "_" + s, config_.variables); + } + + virtual std::string Namespace(const std::string &s) const { + return Format(s, config_.namespaces); + } + + virtual std::string Namespace(const std::vector<std::string> &ns) const { + std::string result; + for (auto it = ns.begin(); it != ns.end(); it++) { + if (it != ns.begin()) result += config_.namespace_seperator; + result += Namespace(*it); + } + return result; + } + + virtual std::string NamespacedType(const std::vector<std::string> &ns, + const std::string &s) const { + return (ns.empty() ? "" : (Namespace(ns) + config_.namespace_seperator)) + + Type(s); + } + + // Returns `filename` with the right casing, suffix, and extension. + virtual std::string File(const std::string &filename, + SkipFile skips = SkipFile::None) const { + const bool skip_suffix = (skips & SkipFile::Suffix) != SkipFile::None; + const bool skip_ext = (skips & SkipFile::Extension) != SkipFile::None; + return ConvertCase(filename, config_.filenames, Case::kUpperCamel) + + (skip_suffix ? "" : config_.filename_suffix) + + (skip_ext ? "" : config_.filename_extension); + } + template<typename T> + std::string File(const T &f, SkipFile skips = SkipFile::None) const { + return File(f.name, skips); + } + + // Formats `directories` prefixed with the output_path and joined with the + // right seperator. Output path prefixing and the trailing separator may be + // skiped using `skips`. + // Callers may want to use `EnsureDirExists` with the result. + virtual std::string Directories(const std::vector<std::string> &directories, + SkipDir skips = SkipDir::None) const { + const bool skip_output_path = + (skips & SkipDir::OutputPath) != SkipDir::None; + const bool skip_trailing_seperator = + (skips & SkipDir::TrailingPathSeperator) != SkipDir::None; + std::string result = skip_output_path ? "" : config_.output_path; + for (auto d = directories.begin(); d != directories.end(); d++) { + result += ConvertCase(*d, config_.directories, Case::kUpperCamel); + result.push_back(kPathSeparator); + } + if (skip_trailing_seperator && !result.empty()) result.pop_back(); + return result; + } + + virtual std::string EscapeKeyword(const std::string &name) const { + if (keywords_.find(name) == keywords_.end()) { + return name; + } else { + return config_.keyword_prefix + name + config_.keyword_suffix; + } + } + + virtual std::string Type(const std::string &s) const { + return Format(s, config_.types); + } + virtual std::string Type(const std::string &t, const std::string &s) const { + return Format(t + "_" + s, config_.types); + } + + virtual std::string ObjectType(const std::string &s) const { + return config_.object_prefix + Type(s) + config_.object_suffix; + } + + virtual std::string Field(const std::string &s) const { + return Format(s, config_.fields); + } + + virtual std::string Variant(const std::string &s) const { + return Format(s, config_.variants); + } + + virtual std::string Format(const std::string &s, Case casing) const { + if (config_.escape_keywords == Config::Escape::BeforeConvertingCase) { + return ConvertCase(EscapeKeyword(s), casing, Case::kLowerCamel); + } else { + return EscapeKeyword(ConvertCase(s, casing, Case::kLowerCamel)); + } + } + + // Denamespaces a string (e.g. The.Quick.Brown.Fox) by returning the last part + // after the `delimiter` (Fox) and placing the rest in `namespace_prefix` + // (The.Quick.Brown). + virtual std::string Denamespace(const std::string &s, + std::string &namespace_prefix, + const char delimiter = '.') const { + const size_t pos = s.find_last_of(delimiter); + if (pos == std::string::npos) { + namespace_prefix = ""; + return s; + } + namespace_prefix = s.substr(0, pos); + return s.substr(pos + 1); + } + + // Same as above, but disregards the prefix. + virtual std::string Denamespace(const std::string &s, + const char delimiter = '.') const { + std::string prefix; + return Denamespace(s, prefix, delimiter); + } + + const Config config_; + const std::set<std::string> keywords_; +}; + +} // namespace flatbuffers + +#endif // FLATBUFFERS_NAMER diff --git a/contrib/libs/flatbuffers/src/reflection.cpp b/contrib/libs/flatbuffers/src/reflection.cpp index 2dedcb4f18d..0d0814ef622 100644 --- a/contrib/libs/flatbuffers/src/reflection.cpp +++ b/contrib/libs/flatbuffers/src/reflection.cpp @@ -22,8 +22,240 @@ namespace flatbuffers { +namespace { + +static void CopyInline(FlatBufferBuilder &fbb, const reflection::Field &fielddef, + const Table &table, size_t align, size_t size) { + fbb.Align(align); + fbb.PushBytes(table.GetStruct<const uint8_t *>(fielddef.offset()), size); + fbb.TrackField(fielddef.offset(), fbb.GetSize()); +} + +static bool VerifyStruct(flatbuffers::Verifier &v, + const flatbuffers::Table &parent_table, + voffset_t field_offset, const reflection::Object &obj, + bool required) { + auto offset = parent_table.GetOptionalFieldOffset(field_offset); + if (required && !offset) { return false; } + + return !offset || + v.VerifyFieldStruct(reinterpret_cast<const uint8_t *>(&parent_table), + offset, obj.bytesize(), obj.minalign()); +} + +static bool VerifyVectorOfStructs(flatbuffers::Verifier &v, + const flatbuffers::Table &parent_table, + voffset_t field_offset, + const reflection::Object &obj, bool required) { + auto p = parent_table.GetPointer<const uint8_t *>(field_offset); + if (required && !p) { return false; } + + return !p || v.VerifyVectorOrString(p, obj.bytesize()); +} + +// forward declare to resolve cyclic deps between VerifyObject and VerifyVector +static bool VerifyObject(flatbuffers::Verifier &v, const reflection::Schema &schema, + const reflection::Object &obj, + const flatbuffers::Table *table, bool required); + +static bool VerifyUnion(flatbuffers::Verifier &v, const reflection::Schema &schema, + uint8_t utype, const uint8_t *elem, + const reflection::Field &union_field) { + if (!utype) return true; // Not present. + auto fb_enum = schema.enums()->Get(union_field.type()->index()); + if (utype >= fb_enum->values()->size()) return false; + auto elem_type = fb_enum->values()->Get(utype)->union_type(); + switch (elem_type->base_type()) { + case reflection::Obj: { + auto elem_obj = schema.objects()->Get(elem_type->index()); + if (elem_obj->is_struct()) { + return v.VerifyFromPointer(elem, elem_obj->bytesize()); + } else { + return VerifyObject(v, schema, *elem_obj, + reinterpret_cast<const flatbuffers::Table *>(elem), + true); + } + } + case reflection::String: + return v.VerifyString( + reinterpret_cast<const flatbuffers::String *>(elem)); + default: return false; + } +} + +static bool VerifyVector(flatbuffers::Verifier &v, const reflection::Schema &schema, + const flatbuffers::Table &table, + const reflection::Field &vec_field) { + FLATBUFFERS_ASSERT(vec_field.type()->base_type() == reflection::Vector); + if (!table.VerifyField<uoffset_t>(v, vec_field.offset(), sizeof(uoffset_t))) + return false; + + switch (vec_field.type()->element()) { + case reflection::UType: + return v.VerifyVector(flatbuffers::GetFieldV<uint8_t>(table, vec_field)); + case reflection::Bool: + case reflection::Byte: + case reflection::UByte: + return v.VerifyVector(flatbuffers::GetFieldV<int8_t>(table, vec_field)); + case reflection::Short: + case reflection::UShort: + return v.VerifyVector(flatbuffers::GetFieldV<int16_t>(table, vec_field)); + case reflection::Int: + case reflection::UInt: + return v.VerifyVector(flatbuffers::GetFieldV<int32_t>(table, vec_field)); + case reflection::Long: + case reflection::ULong: + return v.VerifyVector(flatbuffers::GetFieldV<int64_t>(table, vec_field)); + case reflection::Float: + return v.VerifyVector(flatbuffers::GetFieldV<float>(table, vec_field)); + case reflection::Double: + return v.VerifyVector(flatbuffers::GetFieldV<double>(table, vec_field)); + case reflection::String: { + auto vec_string = + flatbuffers::GetFieldV<flatbuffers::Offset<flatbuffers::String>>( + table, vec_field); + if (v.VerifyVector(vec_string) && v.VerifyVectorOfStrings(vec_string)) { + return true; + } else { + return false; + } + } + case reflection::Obj: { + auto obj = schema.objects()->Get(vec_field.type()->index()); + if (obj->is_struct()) { + return VerifyVectorOfStructs(v, table, vec_field.offset(), *obj, + vec_field.required()); + } else { + auto vec = + flatbuffers::GetFieldV<flatbuffers::Offset<flatbuffers::Table>>( + table, vec_field); + if (!v.VerifyVector(vec)) return false; + if (!vec) return true; + for (uoffset_t j = 0; j < vec->size(); j++) { + if (!VerifyObject(v, schema, *obj, vec->Get(j), true)) { + return false; + } + } + return true; + } + } + case reflection::Union: { + auto vec = flatbuffers::GetFieldV<flatbuffers::Offset<uint8_t>>( + table, vec_field); + if (!v.VerifyVector(vec)) return false; + if (!vec) return true; + auto type_vec = table.GetPointer<Vector<uint8_t> *>(vec_field.offset() - + sizeof(voffset_t)); + if (!v.VerifyVector(type_vec)) return false; + for (uoffset_t j = 0; j < vec->size(); j++) { + // get union type from the prev field + auto utype = type_vec->Get(j); + auto elem = vec->Get(j); + if (!VerifyUnion(v, schema, utype, elem, vec_field)) return false; + } + return true; + } + case reflection::Vector: + case reflection::None: + default: FLATBUFFERS_ASSERT(false); return false; + } +} + +static bool VerifyObject(flatbuffers::Verifier &v, const reflection::Schema &schema, + const reflection::Object &obj, + const flatbuffers::Table *table, bool required) { + if (!table) return !required; + if (!table->VerifyTableStart(v)) return false; + for (uoffset_t i = 0; i < obj.fields()->size(); i++) { + auto field_def = obj.fields()->Get(i); + switch (field_def->type()->base_type()) { + case reflection::None: FLATBUFFERS_ASSERT(false); break; + case reflection::UType: + if (!table->VerifyField<uint8_t>(v, field_def->offset(), + sizeof(uint8_t))) + return false; + break; + case reflection::Bool: + case reflection::Byte: + case reflection::UByte: + if (!table->VerifyField<int8_t>(v, field_def->offset(), sizeof(int8_t))) + return false; + break; + case reflection::Short: + case reflection::UShort: + if (!table->VerifyField<int16_t>(v, field_def->offset(), + sizeof(int16_t))) + return false; + break; + case reflection::Int: + case reflection::UInt: + if (!table->VerifyField<int32_t>(v, field_def->offset(), + sizeof(int32_t))) + return false; + break; + case reflection::Long: + case reflection::ULong: + if (!table->VerifyField<int64_t>(v, field_def->offset(), + sizeof(int64_t))) + return false; + break; + case reflection::Float: + if (!table->VerifyField<float>(v, field_def->offset(), sizeof(float))) + return false; + break; + case reflection::Double: + if (!table->VerifyField<double>(v, field_def->offset(), sizeof(double))) + return false; + break; + case reflection::String: + if (!table->VerifyField<uoffset_t>(v, field_def->offset(), + sizeof(uoffset_t)) || + !v.VerifyString(flatbuffers::GetFieldS(*table, *field_def))) { + return false; + } + break; + case reflection::Vector: + if (!VerifyVector(v, schema, *table, *field_def)) return false; + break; + case reflection::Obj: { + auto child_obj = schema.objects()->Get(field_def->type()->index()); + if (child_obj->is_struct()) { + if (!VerifyStruct(v, *table, field_def->offset(), *child_obj, + field_def->required())) { + return false; + } + } else { + if (!VerifyObject(v, schema, *child_obj, + flatbuffers::GetFieldT(*table, *field_def), + field_def->required())) { + return false; + } + } + break; + } + case reflection::Union: { + // get union type from the prev field + voffset_t utype_offset = field_def->offset() - sizeof(voffset_t); + auto utype = table->GetField<uint8_t>(utype_offset, 0); + auto uval = reinterpret_cast<const uint8_t *>( + flatbuffers::GetFieldT(*table, *field_def)); + if (!VerifyUnion(v, schema, utype, uval, *field_def)) { return false; } + break; + } + default: FLATBUFFERS_ASSERT(false); break; + } + } + + if (!v.EndTable()) return false; + + return true; +} + + +} // namespace + int64_t GetAnyValueI(reflection::BaseType type, const uint8_t *data) { -// clang-format off + // clang-format off #define FLATBUFFERS_GET(T) static_cast<int64_t>(ReadScalar<T>(data)) switch (type) { case reflection::UType: @@ -120,8 +352,25 @@ std::string GetAnyValueS(reflection::BaseType type, const uint8_t *data, } } +void ForAllFields(const reflection::Object *object, bool reverse, + std::function<void(const reflection::Field *)> func) { + std::vector<uint32_t> field_to_id_map; + field_to_id_map.resize(object->fields()->size()); + + // Create the mapping of field ID to the index into the vector. + for (uint32_t i = 0; i < object->fields()->size(); ++i) { + auto field = object->fields()->Get(i); + field_to_id_map[field->id()] = i; + } + + for (size_t i = 0; i < field_to_id_map.size(); ++i) { + func(object->fields()->Get( + field_to_id_map[reverse ? field_to_id_map.size() - i + 1 : i])); + } +} + void SetAnyValueI(reflection::BaseType type, uint8_t *data, int64_t val) { -// clang-format off + // clang-format off #define FLATBUFFERS_SET(T) WriteScalar(data, static_cast<T>(val)) switch (type) { case reflection::UType: @@ -180,7 +429,7 @@ class ResizeContext { std::vector<uint8_t> *flatbuf, const reflection::Object *root_table = nullptr) : schema_(schema), - startptr_(vector_data(*flatbuf) + start), + startptr_(flatbuf->data() + start), delta_(delta), buf_(*flatbuf), dag_check_(flatbuf->size() / sizeof(uoffset_t), false) { @@ -188,8 +437,8 @@ class ResizeContext { delta_ = (delta_ + mask) & ~mask; if (!delta_) return; // We can't shrink by less than largest_scalar_t. // Now change all the offsets by delta_. - auto root = GetAnyRoot(vector_data(buf_)); - Straddle<uoffset_t, 1>(vector_data(buf_), root, vector_data(buf_)); + auto root = GetAnyRoot(buf_.data()); + Straddle<uoffset_t, 1>(buf_.data(), root, buf_.data()); ResizeTable(root_table ? *root_table : *schema.root_table(), root); // We can now add or remove bytes at start. if (delta_ > 0) @@ -217,7 +466,7 @@ class ResizeContext { // will straddle and which won't. uint8_t &DagCheck(const void *offsetloc) { auto dag_idx = reinterpret_cast<const uoffset_t *>(offsetloc) - - reinterpret_cast<const uoffset_t *>(vector_data(buf_)); + reinterpret_cast<const uoffset_t *>(buf_.data()); return dag_check_[dag_idx]; } @@ -257,7 +506,9 @@ class ResizeContext { // Recurse. switch (base_type) { case reflection::Obj: { - ResizeTable(*subobjectdef, reinterpret_cast<Table *>(ref)); + if (subobjectdef) { + ResizeTable(*subobjectdef, reinterpret_cast<Table *>(ref)); + } break; } case reflection::Vector: { @@ -309,19 +560,19 @@ void SetString(const reflection::Schema &schema, const std::string &val, const reflection::Object *root_table) { auto delta = static_cast<int>(val.size()) - static_cast<int>(str->size()); auto str_start = static_cast<uoffset_t>( - reinterpret_cast<const uint8_t *>(str) - vector_data(*flatbuf)); + reinterpret_cast<const uint8_t *>(str) - flatbuf->data()); auto start = str_start + static_cast<uoffset_t>(sizeof(uoffset_t)); if (delta) { // Clear the old string, since we don't want parts of it remaining. - memset(vector_data(*flatbuf) + start, 0, str->size()); + memset(flatbuf->data() + start, 0, str->size()); // Different size, we must expand (or contract). - ResizeContext(schema, start, delta, flatbuf, root_table); + ResizeContext ctx(schema, start, delta, flatbuf, root_table); // Set the new length. - WriteScalar(vector_data(*flatbuf) + str_start, + WriteScalar(flatbuf->data() + str_start, static_cast<uoffset_t>(val.size())); } // Copy new data. Safe because we created the right amount of space. - memcpy(vector_data(*flatbuf) + start, val.c_str(), val.size() + 1); + memcpy(flatbuf->data() + start, val.c_str(), val.size() + 1); } uint8_t *ResizeAnyVector(const reflection::Schema &schema, uoffset_t newsize, @@ -330,25 +581,26 @@ uint8_t *ResizeAnyVector(const reflection::Schema &schema, uoffset_t newsize, const reflection::Object *root_table) { auto delta_elem = static_cast<int>(newsize) - static_cast<int>(num_elems); auto delta_bytes = delta_elem * static_cast<int>(elem_size); - auto vec_start = - reinterpret_cast<const uint8_t *>(vec) - vector_data(*flatbuf); - auto start = static_cast<uoffset_t>(vec_start + sizeof(uoffset_t) + - elem_size * num_elems); + auto vec_start = reinterpret_cast<const uint8_t *>(vec) - flatbuf->data(); + auto start = static_cast<uoffset_t>(vec_start) + + static_cast<uoffset_t>(sizeof(uoffset_t)) + + elem_size * num_elems; if (delta_bytes) { if (delta_elem < 0) { // Clear elements we're throwing away, since some might remain in the // buffer. auto size_clear = -delta_elem * elem_size; - memset(vector_data(*flatbuf) + start - size_clear, 0, size_clear); + memset(flatbuf->data() + start - size_clear, 0, size_clear); } - ResizeContext(schema, start, delta_bytes, flatbuf, root_table); - WriteScalar(vector_data(*flatbuf) + vec_start, newsize); // Length field. + ResizeContext ctx(schema, start, delta_bytes, flatbuf, root_table); + WriteScalar(flatbuf->data() + vec_start, newsize); // Length field. // Set new elements to 0.. this can be overwritten by the caller. if (delta_elem > 0) { - memset(vector_data(*flatbuf) + start, 0, delta_elem * elem_size); + memset(flatbuf->data() + start, 0, + static_cast<size_t>(delta_elem) * elem_size); } } - return vector_data(*flatbuf) + start; + return flatbuf->data() + start; } const uint8_t *AddFlatBuffer(std::vector<uint8_t> &flatbuf, @@ -363,15 +615,11 @@ const uint8_t *AddFlatBuffer(std::vector<uint8_t> &flatbuf, // Insert the entire FlatBuffer minus the root pointer. flatbuf.insert(flatbuf.end(), newbuf + sizeof(uoffset_t), newbuf + newlen); auto root_offset = ReadScalar<uoffset_t>(newbuf) - sizeof(uoffset_t); - return vector_data(flatbuf) + insertion_point + root_offset; + return flatbuf.data() + insertion_point + root_offset; } -void CopyInline(FlatBufferBuilder &fbb, const reflection::Field &fielddef, - const Table &table, size_t align, size_t size) { - fbb.Align(align); - fbb.PushBytes(table.GetStruct<const uint8_t *>(fielddef.offset()), size); - fbb.TrackField(fielddef.offset(), fbb.GetSize()); -} + + Offset<const Table *> CopyTable(FlatBufferBuilder &fbb, const reflection::Schema &schema, @@ -443,9 +691,10 @@ Offset<const Table *> CopyTable(FlatBufferBuilder &fbb, FLATBUFFERS_FALLTHROUGH(); // fall thru default: { // Scalars and structs. auto element_size = GetTypeSize(element_base_type); + auto element_alignment = element_size; // For primitive elements if (elemobjectdef && elemobjectdef->is_struct()) element_size = elemobjectdef->bytesize(); - fbb.StartVector(vec->size(), element_size); + fbb.StartVector(vec->size(), element_size, element_alignment); fbb.PushBytes(vec->Data(), element_size * vec->size()); offset = fbb.EndVector(vec->size()); break; @@ -497,217 +746,22 @@ Offset<const Table *> CopyTable(FlatBufferBuilder &fbb, } } -bool VerifyStruct(flatbuffers::Verifier &v, - const flatbuffers::Table &parent_table, - voffset_t field_offset, const reflection::Object &obj, - bool required) { - auto offset = parent_table.GetOptionalFieldOffset(field_offset); - if (required && !offset) { return false; } - - return !offset || v.Verify(reinterpret_cast<const uint8_t *>(&parent_table), - offset, obj.bytesize()); -} - -bool VerifyVectorOfStructs(flatbuffers::Verifier &v, - const flatbuffers::Table &parent_table, - voffset_t field_offset, - const reflection::Object &obj, bool required) { - auto p = parent_table.GetPointer<const uint8_t *>(field_offset); - if (required && !p) { return false; } - - return !p || v.VerifyVectorOrString(p, obj.bytesize()); -} - -// forward declare to resolve cyclic deps between VerifyObject and VerifyVector -bool VerifyObject(flatbuffers::Verifier &v, const reflection::Schema &schema, - const reflection::Object &obj, - const flatbuffers::Table *table, bool required); - -bool VerifyUnion(flatbuffers::Verifier &v, const reflection::Schema &schema, - uint8_t utype, const uint8_t *elem, - const reflection::Field &union_field) { - if (!utype) return true; // Not present. - auto fb_enum = schema.enums()->Get(union_field.type()->index()); - if (utype >= fb_enum->values()->size()) return false; - auto elem_type = fb_enum->values()->Get(utype)->union_type(); - switch (elem_type->base_type()) { - case reflection::Obj: { - auto elem_obj = schema.objects()->Get(elem_type->index()); - if (elem_obj->is_struct()) { - return v.VerifyFromPointer(elem, elem_obj->bytesize()); - } else { - return VerifyObject(v, schema, *elem_obj, - reinterpret_cast<const flatbuffers::Table *>(elem), - true); - } - } - case reflection::String: - return v.VerifyString( - reinterpret_cast<const flatbuffers::String *>(elem)); - default: return false; - } -} - -bool VerifyVector(flatbuffers::Verifier &v, const reflection::Schema &schema, - const flatbuffers::Table &table, - const reflection::Field &vec_field) { - FLATBUFFERS_ASSERT(vec_field.type()->base_type() == reflection::Vector); - if (!table.VerifyField<uoffset_t>(v, vec_field.offset())) return false; - - switch (vec_field.type()->element()) { - case reflection::UType: - return v.VerifyVector(flatbuffers::GetFieldV<uint8_t>(table, vec_field)); - case reflection::Bool: - case reflection::Byte: - case reflection::UByte: - return v.VerifyVector(flatbuffers::GetFieldV<int8_t>(table, vec_field)); - case reflection::Short: - case reflection::UShort: - return v.VerifyVector(flatbuffers::GetFieldV<int16_t>(table, vec_field)); - case reflection::Int: - case reflection::UInt: - return v.VerifyVector(flatbuffers::GetFieldV<int32_t>(table, vec_field)); - case reflection::Long: - case reflection::ULong: - return v.VerifyVector(flatbuffers::GetFieldV<int64_t>(table, vec_field)); - case reflection::Float: - return v.VerifyVector(flatbuffers::GetFieldV<float>(table, vec_field)); - case reflection::Double: - return v.VerifyVector(flatbuffers::GetFieldV<double>(table, vec_field)); - case reflection::String: { - auto vec_string = - flatbuffers::GetFieldV<flatbuffers::Offset<flatbuffers::String>>( - table, vec_field); - if (v.VerifyVector(vec_string) && v.VerifyVectorOfStrings(vec_string)) { - return true; - } else { - return false; - } - } - case reflection::Obj: { - auto obj = schema.objects()->Get(vec_field.type()->index()); - if (obj->is_struct()) { - return VerifyVectorOfStructs(v, table, vec_field.offset(), *obj, - vec_field.required()); - } else { - auto vec = - flatbuffers::GetFieldV<flatbuffers::Offset<flatbuffers::Table>>( - table, vec_field); - if (!v.VerifyVector(vec)) return false; - if (!vec) return true; - for (uoffset_t j = 0; j < vec->size(); j++) { - if (!VerifyObject(v, schema, *obj, vec->Get(j), true)) { - return false; - } - } - return true; - } - } - case reflection::Union: { - auto vec = flatbuffers::GetFieldV<flatbuffers::Offset<uint8_t>>( - table, vec_field); - if (!v.VerifyVector(vec)) return false; - if (!vec) return true; - auto type_vec = table.GetPointer<Vector<uint8_t> *>(vec_field.offset() - - sizeof(voffset_t)); - if (!v.VerifyVector(type_vec)) return false; - for (uoffset_t j = 0; j < vec->size(); j++) { - // get union type from the prev field - auto utype = type_vec->Get(j); - auto elem = vec->Get(j); - if (!VerifyUnion(v, schema, utype, elem, vec_field)) return false; - } - return true; - } - case reflection::Vector: - case reflection::None: - default: FLATBUFFERS_ASSERT(false); return false; - } -} - -bool VerifyObject(flatbuffers::Verifier &v, const reflection::Schema &schema, - const reflection::Object &obj, - const flatbuffers::Table *table, bool required) { - if (!table) return !required; - if (!table->VerifyTableStart(v)) return false; - for (uoffset_t i = 0; i < obj.fields()->size(); i++) { - auto field_def = obj.fields()->Get(i); - switch (field_def->type()->base_type()) { - case reflection::None: FLATBUFFERS_ASSERT(false); break; - case reflection::UType: - if (!table->VerifyField<uint8_t>(v, field_def->offset())) return false; - break; - case reflection::Bool: - case reflection::Byte: - case reflection::UByte: - if (!table->VerifyField<int8_t>(v, field_def->offset())) return false; - break; - case reflection::Short: - case reflection::UShort: - if (!table->VerifyField<int16_t>(v, field_def->offset())) return false; - break; - case reflection::Int: - case reflection::UInt: - if (!table->VerifyField<int32_t>(v, field_def->offset())) return false; - break; - case reflection::Long: - case reflection::ULong: - if (!table->VerifyField<int64_t>(v, field_def->offset())) return false; - break; - case reflection::Float: - if (!table->VerifyField<float>(v, field_def->offset())) return false; - break; - case reflection::Double: - if (!table->VerifyField<double>(v, field_def->offset())) return false; - break; - case reflection::String: - if (!table->VerifyField<uoffset_t>(v, field_def->offset()) || - !v.VerifyString(flatbuffers::GetFieldS(*table, *field_def))) { - return false; - } - break; - case reflection::Vector: - if (!VerifyVector(v, schema, *table, *field_def)) return false; - break; - case reflection::Obj: { - auto child_obj = schema.objects()->Get(field_def->type()->index()); - if (child_obj->is_struct()) { - if (!VerifyStruct(v, *table, field_def->offset(), *child_obj, - field_def->required())) { - return false; - } - } else { - if (!VerifyObject(v, schema, *child_obj, - flatbuffers::GetFieldT(*table, *field_def), - field_def->required())) { - return false; - } - } - break; - } - case reflection::Union: { - // get union type from the prev field - voffset_t utype_offset = field_def->offset() - sizeof(voffset_t); - auto utype = table->GetField<uint8_t>(utype_offset, 0); - auto uval = reinterpret_cast<const uint8_t *>( - flatbuffers::GetFieldT(*table, *field_def)); - if (!VerifyUnion(v, schema, utype, uval, *field_def)) { return false; } - break; - } - default: FLATBUFFERS_ASSERT(false); break; - } - } - - if (!v.EndTable()) return false; - return true; +bool Verify(const reflection::Schema &schema, const reflection::Object &root, + const uint8_t *const buf, const size_t length, + const uoffset_t max_depth, const uoffset_t max_tables) { + Verifier v(buf, length, max_depth, max_tables); + return VerifyObject(v, schema, root, flatbuffers::GetAnyRoot(buf), + /*required=*/true); } -bool Verify(const reflection::Schema &schema, const reflection::Object &root, - const uint8_t *buf, size_t length, uoffset_t max_depth /*= 64*/, - uoffset_t max_tables /*= 1000000*/) { +bool VerifySizePrefixed(const reflection::Schema &schema, + const reflection::Object &root, + const uint8_t *const buf, const size_t length, + const uoffset_t max_depth, const uoffset_t max_tables) { Verifier v(buf, length, max_depth, max_tables); - return VerifyObject(v, schema, root, flatbuffers::GetAnyRoot(buf), true); + return VerifyObject(v, schema, root, flatbuffers::GetAnySizePrefixedRoot(buf), + /*required=*/true); } } // namespace flatbuffers diff --git a/contrib/libs/flatbuffers/src/util.cpp b/contrib/libs/flatbuffers/src/util.cpp index 3670a019395..38d8536c090 100644 --- a/contrib/libs/flatbuffers/src/util.cpp +++ b/contrib/libs/flatbuffers/src/util.cpp @@ -17,13 +17,14 @@ // clang-format off // Dont't remove `format off`, it prevent reordering of win-includes. +#include <cstring> #if defined(__MINGW32__) || defined(__MINGW64__) || defined(__CYGWIN__) || \ defined(__QNXNTO__) # define _POSIX_C_SOURCE 200809L # define _XOPEN_SOURCE 700L #endif -#ifdef _WIN32 +#if defined(_WIN32) || defined(__MINGW32__) || defined(__MINGW64__) || defined(__CYGWIN__) # ifndef WIN32_LEAN_AND_MEAN # define WIN32_LEAN_AND_MEAN # endif @@ -34,28 +35,35 @@ # include <crtdbg.h> # endif # include <windows.h> // Must be included before <direct.h> -# include <direct.h> +# ifndef __CYGWIN__ +# include <direct.h> +# endif # include <winbase.h> # undef interface // This is also important because of reasons #endif // clang-format on -#include "flatbuffers/base.h" #include "flatbuffers/util.h" #include <sys/stat.h> + #include <clocale> #include <cstdlib> #include <fstream> +#include <functional> + +#include "flatbuffers/base.h" namespace flatbuffers { -bool FileExistsRaw(const char *name) { +namespace { + +static bool FileExistsRaw(const char *name) { std::ifstream ifs(name); return ifs.good(); } -bool LoadFileRaw(const char *name, bool binary, std::string *buf) { +static bool LoadFileRaw(const char *name, bool binary, std::string *buf) { if (DirExists(name)) return false; std::ifstream ifs(name, binary ? std::ifstream::binary : std::ifstream::in); if (!ifs.is_open()) return false; @@ -75,8 +83,131 @@ bool LoadFileRaw(const char *name, bool binary, std::string *buf) { return !ifs.bad(); } -static LoadFileFunction g_load_file_function = LoadFileRaw; -static FileExistsFunction g_file_exists_function = FileExistsRaw; +LoadFileFunction g_load_file_function = LoadFileRaw; +FileExistsFunction g_file_exists_function = FileExistsRaw; + +static std::string ToCamelCase(const std::string &input, bool is_upper) { + std::string s; + for (size_t i = 0; i < input.length(); i++) { + if (!i && input[i] == '_') { + s += input[i]; + // we ignore leading underscore but make following + // alphabet char upper. + if (i + 1 < input.length() && is_alpha(input[i + 1])) + s += CharToUpper(input[++i]); + } + else if (!i) + s += is_upper ? CharToUpper(input[i]) : CharToLower(input[i]); + else if (input[i] == '_' && i + 1 < input.length()) + s += CharToUpper(input[++i]); + else + s += input[i]; + } + return s; +} + +static std::string ToSnakeCase(const std::string &input, bool screaming) { + std::string s; + for (size_t i = 0; i < input.length(); i++) { + if (i == 0) { + s += screaming ? CharToUpper(input[i]) : CharToLower(input[i]); + } else if (input[i] == '_') { + s += '_'; + } else if (!islower(input[i])) { + // Prevent duplicate underscores for Upper_Snake_Case strings + // and UPPERCASE strings. + if (islower(input[i - 1]) || (isdigit(input[i-1]) && !isdigit(input[i]))) { s += '_'; } + s += screaming ? CharToUpper(input[i]) : CharToLower(input[i]); + } else { + s += screaming ? CharToUpper(input[i]) : input[i]; + } + } + return s; +} + +std::string ToAll(const std::string &input, + std::function<char(const char)> transform) { + std::string s; + for (size_t i = 0; i < input.length(); i++) { s += transform(input[i]); } + return s; +} + +std::string CamelToSnake(const std::string &input) { + std::string s; + for (size_t i = 0; i < input.length(); i++) { + if (i == 0) { + s += CharToLower(input[i]); + } else if (input[i] == '_') { + s += '_'; + } else if (!islower(input[i])) { + // Prevent duplicate underscores for Upper_Snake_Case strings + // and UPPERCASE strings. + if (islower(input[i - 1]) || (isdigit(input[i-1]) && !isdigit(input[i]))) { s += '_'; } + s += CharToLower(input[i]); + } else { + s += input[i]; + } + } + return s; +} + +std::string DasherToSnake(const std::string &input) { + std::string s; + for (size_t i = 0; i < input.length(); i++) { + if (input[i] == '-') { + s += "_"; + } else { + s += input[i]; + } + } + return s; +} + +std::string ToDasher(const std::string &input) { + std::string s; + char p = 0; + for (size_t i = 0; i < input.length(); i++) { + char const &c = input[i]; + if (c == '_') { + if (i > 0 && p != kPathSeparator && + // The following is a special case to ignore digits after a _. This is + // because ThisExample3 would be converted to this_example_3 in the + // CamelToSnake conversion, and then dasher would do this-example-3, + // but it expects this-example3. + !(i + 1 < input.length() && isdigit(input[i + 1]))) + s += "-"; + } else { + s += c; + } + p = c; + } + return s; +} + + +// Converts foo_bar_123baz_456 to foo_bar123_baz456 +std::string SnakeToSnake2(const std::string &s) { + if (s.length() <= 1) return s; + std::string result; + result.reserve(s.size()); + for (size_t i = 0; i < s.length() - 1; i++) { + if (s[i] == '_' && isdigit(s[i + 1])) { + continue; // Move the `_` until after the digits. + } + + result.push_back(s[i]); + + if (isdigit(s[i]) && isalpha(s[i + 1]) && islower(s[i + 1])) { + result.push_back('_'); + } + } + result.push_back(s.back()); + + return result; +} + +} // namespace + bool LoadFile(const char *name, bool binary, std::string *buf) { FLATBUFFERS_ASSERT(g_load_file_function); @@ -152,11 +283,20 @@ std::string StripFileName(const std::string &filepath) { return i != std::string::npos ? filepath.substr(0, i) : ""; } +std::string StripPrefix(const std::string &filepath, + const std::string &prefix_to_remove) { + if (!strncmp(filepath.c_str(), prefix_to_remove.c_str(), + prefix_to_remove.size())) { + return filepath.substr(prefix_to_remove.size()); + } + return filepath; +} + std::string ConCatPathFileName(const std::string &path, const std::string &filename) { std::string filepath = path; if (filepath.length()) { - char &filepath_last_character = string_back(filepath); + char &filepath_last_character = filepath.back(); if (filepath_last_character == kPathSeparatorWindows) { filepath_last_character = kPathSeparator; } else if (filepath_last_character != kPathSeparator) { @@ -176,6 +316,9 @@ std::string PosixPath(const char *path) { std::replace(p.begin(), p.end(), '\\', '/'); return p; } +std::string PosixPath(const std::string &path) { + return PosixPath(path.c_str()); +} void EnsureDirExists(const std::string &filepath) { auto parent = StripFileName(filepath); @@ -196,7 +339,7 @@ std::string AbsolutePath(const std::string &filepath) { #ifdef FLATBUFFERS_NO_ABSOLUTE_PATH_RESOLUTION return filepath; #else - #ifdef _WIN32 + #if defined(_WIN32) || defined(__MINGW32__) || defined(__MINGW64__) || defined(__CYGWIN__) char abs_path[MAX_PATH]; return GetFullPathNameA(filepath.c_str(), MAX_PATH, abs_path, nullptr) #else @@ -215,6 +358,36 @@ std::string AbsolutePath(const std::string &filepath) { // clang-format on } +std::string RelativeToRootPath(const std::string &project, + const std::string &filepath) { + std::string absolute_project = PosixPath(AbsolutePath(project)); + if (absolute_project.back() != '/') absolute_project += "/"; + std::string absolute_filepath = PosixPath(AbsolutePath(filepath)); + + // Find the first character where they disagree. + // The previous directory is the lowest common ancestor; + const char *a = absolute_project.c_str(); + const char *b = absolute_filepath.c_str(); + size_t common_prefix_len = 0; + while (*a != '\0' && *b != '\0' && *a == *b) { + if (*a == '/') common_prefix_len = a - absolute_project.c_str(); + a++; + b++; + } + // the number of ../ to prepend to b depends on the number of remaining + // directories in A. + const char *suffix = absolute_project.c_str() + common_prefix_len; + size_t num_up = 0; + while (*suffix != '\0') + if (*suffix++ == '/') num_up++; + num_up--; // last one is known to be '/'. + std::string result = "//"; + for (size_t i = 0; i < num_up; i++) result += "../"; + result += absolute_filepath.substr(common_prefix_len + 1); + + return result; +} + // Locale-independent code. #if defined(FLATBUFFERS_LOCALE_INDEPENDENT) && \ (FLATBUFFERS_LOCALE_INDEPENDENT > 0) @@ -238,8 +411,7 @@ ClassicLocale ClassicLocale::instance_; std::string RemoveStringQuotes(const std::string &s) { auto ch = *s.c_str(); - return ((s.size() >= 2) && (ch == '\"' || ch == '\'') && - (ch == string_back(s))) + return ((s.size() >= 2) && (ch == '\"' || ch == '\'') && (ch == s.back())) ? s.substr(1, s.length() - 2) : s; } @@ -261,27 +433,36 @@ bool ReadEnvironmentVariable(const char *var_name, std::string *_value) { return true; } -void SetupDefaultCRTReportMode() { - // clang-format off - - #ifdef _MSC_VER - // By default, send all reports to STDOUT to prevent CI hangs. - // Enable assert report box [Abort|Retry|Ignore] if a debugger is present. - const int dbg_mode = (_CRTDBG_MODE_FILE | _CRTDBG_MODE_DEBUG) | - (IsDebuggerPresent() ? _CRTDBG_MODE_WNDW : 0); - (void)dbg_mode; // release mode fix - // CrtDebug reports to _CRT_WARN channel. - _CrtSetReportMode(_CRT_WARN, dbg_mode); - _CrtSetReportFile(_CRT_WARN, _CRTDBG_FILE_STDOUT); - // The assert from <assert.h> reports to _CRT_ERROR channel - _CrtSetReportMode(_CRT_ERROR, dbg_mode); - _CrtSetReportFile(_CRT_ERROR, _CRTDBG_FILE_STDOUT); - // Internal CRT assert channel? - _CrtSetReportMode(_CRT_ASSERT, dbg_mode); - _CrtSetReportFile(_CRT_ASSERT, _CRTDBG_FILE_STDOUT); - #endif +std::string ConvertCase(const std::string &input, Case output_case, + Case input_case) { + if (output_case == Case::kKeep) return input; + // The output cases expect snake_case inputs, so if we don't have that input + // format, try to convert to snake_case. + switch (input_case) { + case Case::kLowerCamel: + case Case::kUpperCamel: + return ConvertCase(CamelToSnake(input), output_case); + case Case::kDasher: return ConvertCase(DasherToSnake(input), output_case); + case Case::kKeep: printf("WARNING: Converting from kKeep case.\n"); break; + default: + case Case::kSnake: + case Case::kScreamingSnake: + case Case::kAllLower: + case Case::kAllUpper: break; + } - // clang-format on + switch (output_case) { + case Case::kUpperCamel: return ToCamelCase(input, true); + case Case::kLowerCamel: return ToCamelCase(input, false); + case Case::kSnake: return input; + case Case::kScreamingSnake: return ToSnakeCase(input, true); + case Case::kAllUpper: return ToAll(input, CharToUpper); + case Case::kAllLower: return ToAll(input, CharToLower); + case Case::kDasher: return ToDasher(input); + case Case::kSnake2: return SnakeToSnake2(input); + default: + case Case::kUnknown: return input; + } } } // namespace flatbuffers |