diff options
author | thegeorg <thegeorg@yandex-team.ru> | 2022-02-10 16:45:12 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 16:45:12 +0300 |
commit | 49116032d905455a7b1c994e4a696afc885c1e71 (patch) | |
tree | be835aa92c6248212e705f25388ebafcf84bc7a1 /contrib/libs/apache/avro | |
parent | 4e839db24a3bbc9f1c610c43d6faaaa99824dcca (diff) | |
download | ydb-49116032d905455a7b1c994e4a696afc885c1e71.tar.gz |
Restoring authorship annotation for <thegeorg@yandex-team.ru>. Commit 2 of 2.
Diffstat (limited to 'contrib/libs/apache/avro')
103 files changed, 16912 insertions, 16912 deletions
diff --git a/contrib/libs/apache/avro/.yandex_meta/devtools.licenses.report b/contrib/libs/apache/avro/.yandex_meta/devtools.licenses.report index 2b71d33024..0c6b8995a0 100644 --- a/contrib/libs/apache/avro/.yandex_meta/devtools.licenses.report +++ b/contrib/libs/apache/avro/.yandex_meta/devtools.licenses.report @@ -44,7 +44,7 @@ BELONGS ya.make KEEP Apache-2.0 807648c73ed5fa476be45e9005a86248 BELONGS ya.make -FILE_INCLUDE NOTICE found in files: api/AvroParse.hh at line 3, api/AvroSerialize.hh at line 3, api/AvroTraits.hh at line 3, api/Compiler.hh at line 3, api/Config.hh at line 3, api/DataFile.hh at line 3, api/Decoder.hh at line 3, api/Encoder.hh at line 3, api/Exception.hh at line 3, api/Generic.hh at line 3, api/GenericDatum.hh at line 3, api/Layout.hh at line 3, api/LogicalType.hh at line 3, api/Node.hh at line 3, api/NodeConcepts.hh at line 3, api/NodeImpl.hh at line 3, api/Parser.hh at line 3, api/Reader.hh at line 3, api/Resolver.hh at line 3, api/ResolverSchema.hh at line 3, api/ResolvingReader.hh at line 3, api/Schema.hh at line 3, api/SchemaResolution.hh at line 3, api/Serializer.hh at line 3, api/Specific.hh at line 3, api/Stream.hh at line 3, api/Types.hh at line 3, api/ValidSchema.hh at line 3, api/Validator.hh at line 3, api/Writer.hh at line 3, api/Zigzag.hh at line 3, api/buffer/Buffer.hh at line 3, api/buffer/BufferReader.hh at line 3, api/buffer/detail/BufferDetail.hh at line 3, api/buffer/detail/BufferDetailIterator.hh at line 3, impl/BinaryDecoder.cc at line 3, impl/BinaryEncoder.cc at line 3, impl/Compiler.cc at line 3, impl/DataFile.cc at line 3, impl/FileStream.cc at line 3, impl/Generic.cc at line 3, impl/GenericDatum.cc at line 3, impl/LogicalType.cc at line 3, impl/Node.cc at line 3, impl/NodeImpl.cc at line 3, impl/Resolver.cc at line 4, impl/ResolverSchema.cc at line 4, impl/Schema.cc at line 3, impl/Stream.cc at line 3, impl/Types.cc at line 3, impl/ValidSchema.cc at line 3, impl/Validator.cc at line 3, impl/Zigzag.cc at line 3, impl/json/JsonDom.cc at line 3, impl/json/JsonDom.hh at line 3, impl/json/JsonIO.cc at line 3, impl/json/JsonIO.hh at line 3, impl/parsing/JsonCodec.cc at line 3, impl/parsing/ResolvingDecoder.cc at line 3, impl/parsing/Symbol.cc at line 3, impl/parsing/Symbol.hh at line 3, impl/parsing/ValidatingCodec.cc at line 3, impl/parsing/ValidatingCodec.hh at line 3 +FILE_INCLUDE NOTICE found in files: api/AvroParse.hh at line 3, api/AvroSerialize.hh at line 3, api/AvroTraits.hh at line 3, api/Compiler.hh at line 3, api/Config.hh at line 3, api/DataFile.hh at line 3, api/Decoder.hh at line 3, api/Encoder.hh at line 3, api/Exception.hh at line 3, api/Generic.hh at line 3, api/GenericDatum.hh at line 3, api/Layout.hh at line 3, api/LogicalType.hh at line 3, api/Node.hh at line 3, api/NodeConcepts.hh at line 3, api/NodeImpl.hh at line 3, api/Parser.hh at line 3, api/Reader.hh at line 3, api/Resolver.hh at line 3, api/ResolverSchema.hh at line 3, api/ResolvingReader.hh at line 3, api/Schema.hh at line 3, api/SchemaResolution.hh at line 3, api/Serializer.hh at line 3, api/Specific.hh at line 3, api/Stream.hh at line 3, api/Types.hh at line 3, api/ValidSchema.hh at line 3, api/Validator.hh at line 3, api/Writer.hh at line 3, api/Zigzag.hh at line 3, api/buffer/Buffer.hh at line 3, api/buffer/BufferReader.hh at line 3, api/buffer/detail/BufferDetail.hh at line 3, api/buffer/detail/BufferDetailIterator.hh at line 3, impl/BinaryDecoder.cc at line 3, impl/BinaryEncoder.cc at line 3, impl/Compiler.cc at line 3, impl/DataFile.cc at line 3, impl/FileStream.cc at line 3, impl/Generic.cc at line 3, impl/GenericDatum.cc at line 3, impl/LogicalType.cc at line 3, impl/Node.cc at line 3, impl/NodeImpl.cc at line 3, impl/Resolver.cc at line 4, impl/ResolverSchema.cc at line 4, impl/Schema.cc at line 3, impl/Stream.cc at line 3, impl/Types.cc at line 3, impl/ValidSchema.cc at line 3, impl/Validator.cc at line 3, impl/Zigzag.cc at line 3, impl/json/JsonDom.cc at line 3, impl/json/JsonDom.hh at line 3, impl/json/JsonIO.cc at line 3, impl/json/JsonIO.hh at line 3, impl/parsing/JsonCodec.cc at line 3, impl/parsing/ResolvingDecoder.cc at line 3, impl/parsing/Symbol.cc at line 3, impl/parsing/Symbol.hh at line 3, impl/parsing/ValidatingCodec.cc at line 3, impl/parsing/ValidatingCodec.hh at line 3 Note: matched license text is too long. Read it in the source files. Scancode info: Original SPDX id: Apache-2.0 @@ -52,8 +52,8 @@ FILE_INCLUDE NOTICE found in files: api/AvroParse.hh at line 3, api/AvroSerializ Match type : NOTICE Links : http://www.apache.org/licenses/, http://www.apache.org/licenses/LICENSE-2.0, https://spdx.org/licenses/Apache-2.0 Files with this license: - api/AvroParse.hh [2:16] - api/AvroSerialize.hh [2:16] + api/AvroParse.hh [2:16] + api/AvroSerialize.hh [2:16] api/AvroTraits.hh [2:16] api/Compiler.hh [2:16] api/Config.hh [2:16] @@ -68,20 +68,20 @@ FILE_INCLUDE NOTICE found in files: api/AvroParse.hh at line 3, api/AvroSerializ api/Node.hh [2:16] api/NodeConcepts.hh [2:16] api/NodeImpl.hh [2:16] - api/Parser.hh [2:16] + api/Parser.hh [2:16] api/Reader.hh [2:16] api/Resolver.hh [2:16] api/ResolverSchema.hh [2:16] - api/ResolvingReader.hh [2:16] + api/ResolvingReader.hh [2:16] api/Schema.hh [2:16] api/SchemaResolution.hh [2:16] - api/Serializer.hh [2:16] + api/Serializer.hh [2:16] api/Specific.hh [2:16] api/Stream.hh [2:16] api/Types.hh [2:16] api/ValidSchema.hh [2:16] api/Validator.hh [2:16] - api/Writer.hh [2:16] + api/Writer.hh [2:16] api/Zigzag.hh [2:16] api/buffer/Buffer.hh [2:16] api/buffer/BufferReader.hh [2:16] diff --git a/contrib/libs/apache/avro/AUTHORS b/contrib/libs/apache/avro/AUTHORS index ce1e0107b4..3ebe7c3f82 100644 --- a/contrib/libs/apache/avro/AUTHORS +++ b/contrib/libs/apache/avro/AUTHORS @@ -1,4 +1,4 @@ - -See https://avro.apache.org/ for a list of authors - - + +See https://avro.apache.org/ for a list of authors + + diff --git a/contrib/libs/apache/avro/ChangeLog b/contrib/libs/apache/avro/ChangeLog index 596d5ebc24..317b99f4d6 100644 --- a/contrib/libs/apache/avro/ChangeLog +++ b/contrib/libs/apache/avro/ChangeLog @@ -1 +1 @@ -Refer to CHANGES.txt in the root of avro repository for change log +Refer to CHANGES.txt in the root of avro repository for change log diff --git a/contrib/libs/apache/avro/LICENSE b/contrib/libs/apache/avro/LICENSE index af50186a8b..d641439cde 100644 --- a/contrib/libs/apache/avro/LICENSE +++ b/contrib/libs/apache/avro/LICENSE @@ -1,261 +1,261 @@ - - Apache License - Version 2.0, January 2004 - https://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - ----------------------------------------------------------------------- -License for the m4 macros used by the C++ implementation: - -Files: -* lang/c++/m4/m4_ax_boost_system.m4 - Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de> - Copyright (c) 2008 Michael Tindal - Copyright (c) 2008 Daniel Casimiro <dan.casimiro@gmail.com> -* lang/c++/m4/m4_ax_boost_asio.m4 - Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de> - Copyright (c) 2008 Pete Greenwell <pete@mu.org> -* lang/c++/m4/m4_ax_boost_filesystem.m4 - Copyright (c) 2009 Thomas Porschberg <thomas@randspringer.de> - Copyright (c) 2009 Michael Tindal - Copyright (c) 2009 Roman Rybalko <libtorrent@romanr.info> -* lang/c++/m4/m4_ax_boost_thread.m4 - Copyright (c) 2009 Thomas Porschberg <thomas@randspringer.de> - Copyright (c) 2009 Michael Tindal -* lang/c++/m4/m4_ax_boost_regex.m4 - Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de> - Copyright (c) 2008 Michael Tindal -* lang/c++/m4/m4_ax_boost_base.m4 - Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de> - -License text: -| Copying and distribution of this file, with or without modification, are -| permitted in any medium without royalty provided the copyright notice -| and this notice are preserved. This file is offered as-is, without any -| warranty. - ----------------------------------------------------------------------- -License for the AVRO_BOOT_NO_TRAIT code in the C++ implementation: -File: lang/c++/api/Boost.hh - -| Boost Software License - Version 1.0 - August 17th, 2003 -| -| Permission is hereby granted, free of charge, to any person or organization -| obtaining a copy of the software and accompanying documentation covered by -| this license (the "Software") to use, reproduce, display, distribute, -| execute, and transmit the Software, and to prepare derivative works of the -| Software, and to permit third-parties to whom the Software is furnished to -| do so, all subject to the following: -| -| The copyright notices in the Software and this entire statement, including -| the above license grant, this restriction and the following disclaimer, -| must be included in all copies of the Software, in whole or in part, and -| all derivative works of the Software, unless such copies or derivative -| works are solely in the form of machine-executable object code generated by -| a source language processor. -| -| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -| FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -| SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -| FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -| ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -| DEALINGS IN THE SOFTWARE. - + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License for the m4 macros used by the C++ implementation: + +Files: +* lang/c++/m4/m4_ax_boost_system.m4 + Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de> + Copyright (c) 2008 Michael Tindal + Copyright (c) 2008 Daniel Casimiro <dan.casimiro@gmail.com> +* lang/c++/m4/m4_ax_boost_asio.m4 + Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de> + Copyright (c) 2008 Pete Greenwell <pete@mu.org> +* lang/c++/m4/m4_ax_boost_filesystem.m4 + Copyright (c) 2009 Thomas Porschberg <thomas@randspringer.de> + Copyright (c) 2009 Michael Tindal + Copyright (c) 2009 Roman Rybalko <libtorrent@romanr.info> +* lang/c++/m4/m4_ax_boost_thread.m4 + Copyright (c) 2009 Thomas Porschberg <thomas@randspringer.de> + Copyright (c) 2009 Michael Tindal +* lang/c++/m4/m4_ax_boost_regex.m4 + Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de> + Copyright (c) 2008 Michael Tindal +* lang/c++/m4/m4_ax_boost_base.m4 + Copyright (c) 2008 Thomas Porschberg <thomas@randspringer.de> + +License text: +| Copying and distribution of this file, with or without modification, are +| permitted in any medium without royalty provided the copyright notice +| and this notice are preserved. This file is offered as-is, without any +| warranty. + +---------------------------------------------------------------------- +License for the AVRO_BOOT_NO_TRAIT code in the C++ implementation: +File: lang/c++/api/Boost.hh + +| Boost Software License - Version 1.0 - August 17th, 2003 +| +| Permission is hereby granted, free of charge, to any person or organization +| obtaining a copy of the software and accompanying documentation covered by +| this license (the "Software") to use, reproduce, display, distribute, +| execute, and transmit the Software, and to prepare derivative works of the +| Software, and to permit third-parties to whom the Software is furnished to +| do so, all subject to the following: +| +| The copyright notices in the Software and this entire statement, including +| the above license grant, this restriction and the following disclaimer, +| must be included in all copies of the Software, in whole or in part, and +| all derivative works of the Software, unless such copies or derivative +| works are solely in the form of machine-executable object code generated by +| a source language processor. +| +| THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +| IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +| FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +| SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +| FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +| ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +| DEALINGS IN THE SOFTWARE. + diff --git a/contrib/libs/apache/avro/MSBUILD.md b/contrib/libs/apache/avro/MSBUILD.md index dfd339bc03..11ffd9851e 100644 --- a/contrib/libs/apache/avro/MSBUILD.md +++ b/contrib/libs/apache/avro/MSBUILD.md @@ -1,33 +1,33 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - -https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. ---> -# Visual Studio 2019 Build Instructions - -## Prerequisites - - * Microsoft Visual Studio 2019. - * CMake >= 3.12 (should be supplied as part of VS2019 installation). - * Clone [https://github.com/spektom/snappy-visual-cpp](https://github.com/spektom/snappy-visual-cpp), and follow build instructions in `README.md`. - * Install Boost from [https://netcologne.dl.sourceforge.net/project/boost/boost-binaries/1.68.0/boost_1_68_0-msvc-14.1-64.exe](https://netcologne.dl.sourceforge.net/project/boost/boost-binaries/1.68.0/boost_1_68_0-msvc-14.1-64.exe). - * Add `C:\<path to>\boost_1_68_0\lib64-msvc-14.1` to PATH environment variable. - -## Building - - cd lang\c++ - cmake -G "Visual Studio 16 2019" -DBOOST_ROOT=C:\<path to>\boost_1_68_0 -DBOOST_INCLUDEDIR=c:\<path to>\boost_1_68_0\boost -DBOOST_LIBRARYDIR=c:\<path to>\boost_1_68_0\lib64-msvc-14.1 -DSNAPPY_INCLUDE_DIR=C:\<path to>\snappy-visual-cpp -DSNAPPY_LIBRARIES=C:\<path to>\snappy-visual-cpp\x64\Release\snappy.lib .. - msbuild Avro-cpp.sln /p:Configuration=Release /p:Platform=x64 - +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> +# Visual Studio 2019 Build Instructions + +## Prerequisites + + * Microsoft Visual Studio 2019. + * CMake >= 3.12 (should be supplied as part of VS2019 installation). + * Clone [https://github.com/spektom/snappy-visual-cpp](https://github.com/spektom/snappy-visual-cpp), and follow build instructions in `README.md`. + * Install Boost from [https://netcologne.dl.sourceforge.net/project/boost/boost-binaries/1.68.0/boost_1_68_0-msvc-14.1-64.exe](https://netcologne.dl.sourceforge.net/project/boost/boost-binaries/1.68.0/boost_1_68_0-msvc-14.1-64.exe). + * Add `C:\<path to>\boost_1_68_0\lib64-msvc-14.1` to PATH environment variable. + +## Building + + cd lang\c++ + cmake -G "Visual Studio 16 2019" -DBOOST_ROOT=C:\<path to>\boost_1_68_0 -DBOOST_INCLUDEDIR=c:\<path to>\boost_1_68_0\boost -DBOOST_LIBRARYDIR=c:\<path to>\boost_1_68_0\lib64-msvc-14.1 -DSNAPPY_INCLUDE_DIR=C:\<path to>\snappy-visual-cpp -DSNAPPY_LIBRARIES=C:\<path to>\snappy-visual-cpp\x64\Release\snappy.lib .. + msbuild Avro-cpp.sln /p:Configuration=Release /p:Platform=x64 + diff --git a/contrib/libs/apache/avro/NEWS b/contrib/libs/apache/avro/NEWS index 8b7b12d8d2..415280b0a2 100644 --- a/contrib/libs/apache/avro/NEWS +++ b/contrib/libs/apache/avro/NEWS @@ -1,5 +1,5 @@ - -For news, visit the Avro web site at -https://avro.apache.org/ - - + +For news, visit the Avro web site at +https://avro.apache.org/ + + diff --git a/contrib/libs/apache/avro/NOTICE b/contrib/libs/apache/avro/NOTICE index 35e863231e..859ea81d74 100644 --- a/contrib/libs/apache/avro/NOTICE +++ b/contrib/libs/apache/avro/NOTICE @@ -1,6 +1,6 @@ -Apache Avro -Copyright 2010-2015 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (https://www.apache.org/). - +Apache Avro +Copyright 2010-2015 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + diff --git a/contrib/libs/apache/avro/README b/contrib/libs/apache/avro/README index e03d42023e..6b081f13a8 100644 --- a/contrib/libs/apache/avro/README +++ b/contrib/libs/apache/avro/README @@ -1,69 +1,69 @@ -Avro C++ README.txt - -The C++ port is thus far incomplete. Currently, it contains: - - - Serializer/Parser- objects for writing/reading raw binary. - - - xxxSchema- objects for composing schemas. - - - ValidSchema- a schema object that has been converted to a parse tree - (with some sanity checks). - - - ValidSchema.toJson() writes the schema as a json object. - - - ValidatingSerializer/ValidatingParser- check that reads/writes - match the expected schema type (more expensive than the raw - serializer/parser but they detect errors, and allow dynamic - discovery of parsed data/attributes). - - - Compiler (compileJsonSchema())- converts a Json string schema to a - ValidSchema. - - - Code Generation (experimental) - given a schema it generates C++ - objects of the same data types, and the code to serialize and parse - it. - -What's missing: Rpc containers are not yet implemented. Documentation is sparse. - -INSTRUCTIONS - -Pre-requisites: - -To compile requires boost headers, and the boost regex library. Optionally, it requires Snappy compression library. If Snappy is available, it builds support for Snappy compression and skips it otherwise. (Please see your OS-specific instructions on how to install Boost and Snappy for your OS). - -To build one requires cmake 2.6 or later. - -To generate a Makefile under Unix, MacOS (using GNU) or Cygwin use: - -mkdir build -cd build -cmake -G "Unix Makefiles" .. - -If it doesn't work, either you are missing some packages (boost, flex or bison), -or you need to help configure locate them. - -If the Makefile is configured correctly, then you can make and run tests: - - make - ctest - -To install - - make package - -and then untar the generated .tar.gz file. - -To build and test on MacOS (using Xcode) - -mkdir build.mac -cd build.mac -cmake -G Xcode - -xcodebuild -configuration Release -ctest -C Release - -If debug version is required, replace 'Release' above with 'Debug'. - -Note: The LICENSE and NOTICE files in the lang/c++ source directory are used to -build the binary distribution. The LICENSE and NOTICE information for the Avro -C++ source distribution is in the root directory. +Avro C++ README.txt + +The C++ port is thus far incomplete. Currently, it contains: + + - Serializer/Parser- objects for writing/reading raw binary. + + - xxxSchema- objects for composing schemas. + + - ValidSchema- a schema object that has been converted to a parse tree + (with some sanity checks). + + - ValidSchema.toJson() writes the schema as a json object. + + - ValidatingSerializer/ValidatingParser- check that reads/writes + match the expected schema type (more expensive than the raw + serializer/parser but they detect errors, and allow dynamic + discovery of parsed data/attributes). + + - Compiler (compileJsonSchema())- converts a Json string schema to a + ValidSchema. + + - Code Generation (experimental) - given a schema it generates C++ + objects of the same data types, and the code to serialize and parse + it. + +What's missing: Rpc containers are not yet implemented. Documentation is sparse. + +INSTRUCTIONS + +Pre-requisites: + +To compile requires boost headers, and the boost regex library. Optionally, it requires Snappy compression library. If Snappy is available, it builds support for Snappy compression and skips it otherwise. (Please see your OS-specific instructions on how to install Boost and Snappy for your OS). + +To build one requires cmake 2.6 or later. + +To generate a Makefile under Unix, MacOS (using GNU) or Cygwin use: + +mkdir build +cd build +cmake -G "Unix Makefiles" .. + +If it doesn't work, either you are missing some packages (boost, flex or bison), +or you need to help configure locate them. + +If the Makefile is configured correctly, then you can make and run tests: + + make + ctest + +To install + + make package + +and then untar the generated .tar.gz file. + +To build and test on MacOS (using Xcode) + +mkdir build.mac +cd build.mac +cmake -G Xcode + +xcodebuild -configuration Release +ctest -C Release + +If debug version is required, replace 'Release' above with 'Debug'. + +Note: The LICENSE and NOTICE files in the lang/c++ source directory are used to +build the binary distribution. The LICENSE and NOTICE information for the Avro +C++ source distribution is in the root directory. diff --git a/contrib/libs/apache/avro/api/AvroParse.hh b/contrib/libs/apache/avro/api/AvroParse.hh index 999ff86122..dd2b98c419 100644 --- a/contrib/libs/apache/avro/api/AvroParse.hh +++ b/contrib/libs/apache/avro/api/AvroParse.hh @@ -1,85 +1,85 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_AvroParse_hh__ -#define avro_AvroParse_hh__ - -#include "Config.hh" -#include "AvroTraits.hh" -#include "ResolvingReader.hh" - -/// \file -/// -/// Standalone parse functions for Avro types. - -namespace avro { - -/// The main parse entry point function. Takes a parser (either validating or -/// plain) and the object that should receive the parsed data. - -template <typename Reader, typename T> -void parse(Reader &p, T& val) -{ - parse(p, val, is_serializable<T>()); -} - -template <typename T> -void parse(ResolvingReader &p, T& val) -{ - translatingParse(p, val, is_serializable<T>()); -} - -/// Type trait should be set to is_serializable in otherwise force the compiler to complain. - -template <typename Reader, typename T> -void parse(Reader &p, T& val, const std::false_type &) -{ - static_assert(sizeof(T) == 0, "Not a valid type to parse"); -} - -template <typename Reader, typename T> -void translatingParse(Reader &p, T& val, const std::false_type &) -{ - static_assert(sizeof(T) == 0, "Not a valid type to parse"); -} - -// @{ - -/// The remainder of the file includes default implementations for serializable types. - - -template <typename Reader, typename T> -void parse(Reader &p, T &val, const std::true_type &) { - p.readValue(val); -} - -template <typename Reader> -void parse(Reader &p, std::vector<uint8_t> &val, const std::true_type &) { - p.readBytes(val); -} - -template<typename T> -void translatingParse(ResolvingReader &p, T& val, const std::true_type &) { - p.parse(val); -} - -// @} - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_AvroParse_hh__ +#define avro_AvroParse_hh__ + +#include "Config.hh" +#include "AvroTraits.hh" +#include "ResolvingReader.hh" + +/// \file +/// +/// Standalone parse functions for Avro types. + +namespace avro { + +/// The main parse entry point function. Takes a parser (either validating or +/// plain) and the object that should receive the parsed data. + +template <typename Reader, typename T> +void parse(Reader &p, T& val) +{ + parse(p, val, is_serializable<T>()); +} + +template <typename T> +void parse(ResolvingReader &p, T& val) +{ + translatingParse(p, val, is_serializable<T>()); +} + +/// Type trait should be set to is_serializable in otherwise force the compiler to complain. + +template <typename Reader, typename T> +void parse(Reader &p, T& val, const std::false_type &) +{ + static_assert(sizeof(T) == 0, "Not a valid type to parse"); +} + +template <typename Reader, typename T> +void translatingParse(Reader &p, T& val, const std::false_type &) +{ + static_assert(sizeof(T) == 0, "Not a valid type to parse"); +} + +// @{ + +/// The remainder of the file includes default implementations for serializable types. + + +template <typename Reader, typename T> +void parse(Reader &p, T &val, const std::true_type &) { + p.readValue(val); +} + +template <typename Reader> +void parse(Reader &p, std::vector<uint8_t> &val, const std::true_type &) { + p.readBytes(val); +} + +template<typename T> +void translatingParse(ResolvingReader &p, T& val, const std::true_type &) { + p.parse(val); +} + +// @} + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/AvroSerialize.hh b/contrib/libs/apache/avro/api/AvroSerialize.hh index 365e8ff648..9495940c9c 100644 --- a/contrib/libs/apache/avro/api/AvroSerialize.hh +++ b/contrib/libs/apache/avro/api/AvroSerialize.hh @@ -1,66 +1,66 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_AvroSerialize_hh__ -#define avro_AvroSerialize_hh__ - -#include "Config.hh" -#include "AvroTraits.hh" - -/// \file -/// -/// Standalone serialize functions for Avro types. - -namespace avro { - -/// The main serializer entry point function. Takes a serializer (either validating or -/// plain) and the object that should be serialized. - -template <typename Writer, typename T> -void serialize(Writer &s, const T& val) -{ - serialize(s, val, is_serializable<T>()); -} - -/// Type trait should be set to is_serializable in otherwise force the compiler to complain. - -template <typename Writer, typename T> -void serialize(Writer &s, const T& val, const std::false_type &) -{ - static_assert(sizeof(T) == 0, "Not a valid type to serialize"); -} - -/// The remainder of the file includes default implementations for serializable types. - -// @{ - -template <typename Writer, typename T> -void serialize(Writer &s, T val, const std::true_type &) { - s.writeValue(val); -} - -template <typename Writer> -void serialize(Writer &s, const std::vector<uint8_t> &val, const std::true_type &) { - s.writeBytes(val.data(), val.size()); -} - -// @} - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_AvroSerialize_hh__ +#define avro_AvroSerialize_hh__ + +#include "Config.hh" +#include "AvroTraits.hh" + +/// \file +/// +/// Standalone serialize functions for Avro types. + +namespace avro { + +/// The main serializer entry point function. Takes a serializer (either validating or +/// plain) and the object that should be serialized. + +template <typename Writer, typename T> +void serialize(Writer &s, const T& val) +{ + serialize(s, val, is_serializable<T>()); +} + +/// Type trait should be set to is_serializable in otherwise force the compiler to complain. + +template <typename Writer, typename T> +void serialize(Writer &s, const T& val, const std::false_type &) +{ + static_assert(sizeof(T) == 0, "Not a valid type to serialize"); +} + +/// The remainder of the file includes default implementations for serializable types. + +// @{ + +template <typename Writer, typename T> +void serialize(Writer &s, T val, const std::true_type &) { + s.writeValue(val); +} + +template <typename Writer> +void serialize(Writer &s, const std::vector<uint8_t> &val, const std::true_type &) { + s.writeBytes(val.data(), val.size()); +} + +// @} + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/AvroTraits.hh b/contrib/libs/apache/avro/api/AvroTraits.hh index d4d76efb2e..91e2130c76 100644 --- a/contrib/libs/apache/avro/api/AvroTraits.hh +++ b/contrib/libs/apache/avro/api/AvroTraits.hh @@ -1,116 +1,116 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_AvroTraits_hh__ -#define avro_AvroTraits_hh__ - -#include "Config.hh" -#include "Types.hh" -#include <stdint.h> -#include <type_traits> - -/** @file - * - * This header contains type traits and similar utilities used by the library. - */ -namespace avro { - -/** - * Define an is_serializable trait for types we can serialize natively. - * New types will need to define the trait as well. - */ -template <typename T> -struct is_serializable : public std::false_type{}; - -template <typename T> -struct is_promotable : public std::false_type{}; - -template <typename T> -struct type_to_avro { - static const Type type = AVRO_NUM_TYPES; -}; - -/** - * Check if a \p T is a complete type i.e. it is defined as opposed to just - * declared. - * - * is_defined<T>::value will be true or false depending on whether T is a - * complete type or not respectively. - */ -template <class T> -struct is_defined { - - typedef char yes[1]; - - typedef char no[2]; - - template <class U> static yes& test(char(*)[sizeof(U)]) { throw 0; }; - - template <class U> static no& test(...) { throw 0; }; - - static const bool value = sizeof(test<T>(0)) == sizeof(yes); -}; - -/** - * Similar to is_defined, but used to check if T is not defined. - * - * is_not_defined<T>::value will be true or false depending on whether T is an - * incomplete type or not respectively. - */ -template <class T> -struct is_not_defined { - - typedef char yes[1]; - - typedef char no[2]; - - template <class U> static yes& test(char(*)[sizeof(U)]) { throw 0; }; - - template <class U> static no& test(...) { throw 0; }; - - static const bool value = sizeof(test<T>(0)) == sizeof(no); -}; - -#define DEFINE_PRIMITIVE(CTYPE, AVROTYPE) \ -template <> \ -struct is_serializable<CTYPE> : public std::true_type{}; \ -\ -template <> \ -struct type_to_avro<CTYPE> { \ - static const Type type = AVROTYPE; \ -}; - -#define DEFINE_PROMOTABLE_PRIMITIVE(CTYPE, AVROTYPE) \ -template <> \ -struct is_promotable<CTYPE> : public std::true_type{}; \ -\ -DEFINE_PRIMITIVE(CTYPE, AVROTYPE) - -DEFINE_PROMOTABLE_PRIMITIVE(int32_t, AVRO_INT) -DEFINE_PROMOTABLE_PRIMITIVE(int64_t, AVRO_LONG) -DEFINE_PROMOTABLE_PRIMITIVE(float, AVRO_FLOAT) -DEFINE_PRIMITIVE(double, AVRO_DOUBLE) -DEFINE_PRIMITIVE(bool, AVRO_BOOL) -DEFINE_PRIMITIVE(Null, AVRO_NULL) -DEFINE_PRIMITIVE(std::string, AVRO_STRING) -DEFINE_PRIMITIVE(std::vector<uint8_t>, AVRO_BYTES) - - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_AvroTraits_hh__ +#define avro_AvroTraits_hh__ + +#include "Config.hh" +#include "Types.hh" +#include <stdint.h> +#include <type_traits> + +/** @file + * + * This header contains type traits and similar utilities used by the library. + */ +namespace avro { + +/** + * Define an is_serializable trait for types we can serialize natively. + * New types will need to define the trait as well. + */ +template <typename T> +struct is_serializable : public std::false_type{}; + +template <typename T> +struct is_promotable : public std::false_type{}; + +template <typename T> +struct type_to_avro { + static const Type type = AVRO_NUM_TYPES; +}; + +/** + * Check if a \p T is a complete type i.e. it is defined as opposed to just + * declared. + * + * is_defined<T>::value will be true or false depending on whether T is a + * complete type or not respectively. + */ +template <class T> +struct is_defined { + + typedef char yes[1]; + + typedef char no[2]; + + template <class U> static yes& test(char(*)[sizeof(U)]) { throw 0; }; + + template <class U> static no& test(...) { throw 0; }; + + static const bool value = sizeof(test<T>(0)) == sizeof(yes); +}; + +/** + * Similar to is_defined, but used to check if T is not defined. + * + * is_not_defined<T>::value will be true or false depending on whether T is an + * incomplete type or not respectively. + */ +template <class T> +struct is_not_defined { + + typedef char yes[1]; + + typedef char no[2]; + + template <class U> static yes& test(char(*)[sizeof(U)]) { throw 0; }; + + template <class U> static no& test(...) { throw 0; }; + + static const bool value = sizeof(test<T>(0)) == sizeof(no); +}; + +#define DEFINE_PRIMITIVE(CTYPE, AVROTYPE) \ +template <> \ +struct is_serializable<CTYPE> : public std::true_type{}; \ +\ +template <> \ +struct type_to_avro<CTYPE> { \ + static const Type type = AVROTYPE; \ +}; + +#define DEFINE_PROMOTABLE_PRIMITIVE(CTYPE, AVROTYPE) \ +template <> \ +struct is_promotable<CTYPE> : public std::true_type{}; \ +\ +DEFINE_PRIMITIVE(CTYPE, AVROTYPE) + +DEFINE_PROMOTABLE_PRIMITIVE(int32_t, AVRO_INT) +DEFINE_PROMOTABLE_PRIMITIVE(int64_t, AVRO_LONG) +DEFINE_PROMOTABLE_PRIMITIVE(float, AVRO_FLOAT) +DEFINE_PRIMITIVE(double, AVRO_DOUBLE) +DEFINE_PRIMITIVE(bool, AVRO_BOOL) +DEFINE_PRIMITIVE(Null, AVRO_NULL) +DEFINE_PRIMITIVE(std::string, AVRO_STRING) +DEFINE_PRIMITIVE(std::vector<uint8_t>, AVRO_BYTES) + + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Compiler.hh b/contrib/libs/apache/avro/api/Compiler.hh index 3df875cd16..892d60580d 100644 --- a/contrib/libs/apache/avro/api/Compiler.hh +++ b/contrib/libs/apache/avro/api/Compiler.hh @@ -1,63 +1,63 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Compiler_hh__ -#define avro_Compiler_hh__ - -#include "Config.hh" -#include <stdint.h> -#include <istream> - -namespace avro { - -class AVRO_DECL InputStream; - -/// This class is used to implement an avro spec parser using a flex/bison -/// compiler. In order for the lexer to be reentrant, this class provides a -/// lexer object for each parse. The bison parser also uses this class to -/// build up an avro parse tree as the avro spec is parsed. - -class AVRO_DECL ValidSchema; - -/// Given a stream comtaining a JSON schema, compiles the schema to a -/// ValidSchema object. Throws if the schema cannot be compiled to a valid -/// schema - -AVRO_DECL void compileJsonSchema(std::istream &is, ValidSchema &schema); - -/// Non-throwing version of compileJsonSchema. -/// -/// \return True if no error, false if error (with the error string set) -/// - -AVRO_DECL bool compileJsonSchema(std::istream &is, ValidSchema &schema, - std::string &error); - -AVRO_DECL ValidSchema compileJsonSchemaFromStream(InputStream& is); - -AVRO_DECL ValidSchema compileJsonSchemaFromMemory(const uint8_t* input, size_t len); - -AVRO_DECL ValidSchema compileJsonSchemaFromString(const char* input); - -AVRO_DECL ValidSchema compileJsonSchemaFromString(const std::string& input); - -AVRO_DECL ValidSchema compileJsonSchemaFromFile(const char* filename); - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Compiler_hh__ +#define avro_Compiler_hh__ + +#include "Config.hh" +#include <stdint.h> +#include <istream> + +namespace avro { + +class AVRO_DECL InputStream; + +/// This class is used to implement an avro spec parser using a flex/bison +/// compiler. In order for the lexer to be reentrant, this class provides a +/// lexer object for each parse. The bison parser also uses this class to +/// build up an avro parse tree as the avro spec is parsed. + +class AVRO_DECL ValidSchema; + +/// Given a stream comtaining a JSON schema, compiles the schema to a +/// ValidSchema object. Throws if the schema cannot be compiled to a valid +/// schema + +AVRO_DECL void compileJsonSchema(std::istream &is, ValidSchema &schema); + +/// Non-throwing version of compileJsonSchema. +/// +/// \return True if no error, false if error (with the error string set) +/// + +AVRO_DECL bool compileJsonSchema(std::istream &is, ValidSchema &schema, + std::string &error); + +AVRO_DECL ValidSchema compileJsonSchemaFromStream(InputStream& is); + +AVRO_DECL ValidSchema compileJsonSchemaFromMemory(const uint8_t* input, size_t len); + +AVRO_DECL ValidSchema compileJsonSchemaFromString(const char* input); + +AVRO_DECL ValidSchema compileJsonSchemaFromString(const std::string& input); + +AVRO_DECL ValidSchema compileJsonSchemaFromFile(const char* filename); + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Config.hh b/contrib/libs/apache/avro/api/Config.hh index 6daa843e9c..69d36f2abe 100644 --- a/contrib/libs/apache/avro/api/Config.hh +++ b/contrib/libs/apache/avro/api/Config.hh @@ -1,45 +1,45 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Config_hh -#define avro_Config_hh - -// Windows DLL suport - -#ifdef _WIN32 -#pragma warning (disable: 4275 4251) - -#if defined(AVRO_DYN_LINK) -#ifdef AVRO_SOURCE -# define AVRO_DECL __declspec(dllexport) -#else -# define AVRO_DECL __declspec(dllimport) -#endif // AVRO_SOURCE -#endif // AVRO_DYN_LINK - -#include <intsafe.h> -typedef SSIZE_T ssize_t; - -#endif // _WIN32 - -#ifndef AVRO_DECL -#define AVRO_DECL -#endif - -#endif - +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Config_hh +#define avro_Config_hh + +// Windows DLL suport + +#ifdef _WIN32 +#pragma warning (disable: 4275 4251) + +#if defined(AVRO_DYN_LINK) +#ifdef AVRO_SOURCE +# define AVRO_DECL __declspec(dllexport) +#else +# define AVRO_DECL __declspec(dllimport) +#endif // AVRO_SOURCE +#endif // AVRO_DYN_LINK + +#include <intsafe.h> +typedef SSIZE_T ssize_t; + +#endif // _WIN32 + +#ifndef AVRO_DECL +#define AVRO_DECL +#endif + +#endif + diff --git a/contrib/libs/apache/avro/api/DataFile.hh b/contrib/libs/apache/avro/api/DataFile.hh index cc333d70ba..50169106b1 100644 --- a/contrib/libs/apache/avro/api/DataFile.hh +++ b/contrib/libs/apache/avro/api/DataFile.hh @@ -1,419 +1,419 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_DataFile_hh__ -#define avro_DataFile_hh__ - -#include "Config.hh" -#include "Encoder.hh" -#include "buffer/Buffer.hh" -#include "ValidSchema.hh" -#include "Specific.hh" -#include "Stream.hh" - -#include <map> -#include <string> -#include <vector> - -#include "array" -#include "boost/utility.hpp" -#include <boost/iostreams/filtering_stream.hpp> - -namespace avro { - -/** Specify type of compression to use when writing data files. */ -enum Codec { - NULL_CODEC, - DEFLATE_CODEC, - -#ifdef SNAPPY_CODEC_AVAILABLE - SNAPPY_CODEC -#endif - -}; - -const int SyncSize = 16; -/** - * The sync value. - */ -typedef std::array<uint8_t, SyncSize> DataFileSync; - -/** - * Type-independent portion of DataFileWriter. - * At any given point in time, at most one file can be written using - * this object. - */ -class AVRO_DECL DataFileWriterBase : boost::noncopyable { - const std::string filename_; - const ValidSchema schema_; - const EncoderPtr encoderPtr_; - const size_t syncInterval_; - Codec codec_; - - std::unique_ptr<OutputStream> stream_; - std::unique_ptr<OutputStream> buffer_; - const DataFileSync sync_; - int64_t objectCount_; - - typedef std::map<std::string, std::vector<uint8_t> > Metadata; - - Metadata metadata_; - int64_t lastSync_; - - static std::unique_ptr<OutputStream> makeStream(const char* filename); - static DataFileSync makeSync(); - - void writeHeader(); - void setMetadata(const std::string& key, const std::string& value); - - /** - * Generates a sync marker in the file. - */ - void sync(); - - /** - * Shared constructor portion since we aren't using C++11 - */ - void init(const ValidSchema &schema, size_t syncInterval, const Codec &codec); - -public: - /** - * Returns the current encoder for this writer. - */ - Encoder& encoder() const { return *encoderPtr_; } - - /** - * Returns true if the buffer has sufficient data for a sync to be - * inserted. - */ - void syncIfNeeded(); - - /** - * Returns the byte offset (within the current file) of the start of the current block being written. - */ - uint64_t getCurrentBlockStart(); - - /** - * Increments the object count. - */ - void incr() { - ++objectCount_; - } - /** - * Constructs a data file writer with the given sync interval and name. - */ - DataFileWriterBase(const char* filename, const ValidSchema& schema, - size_t syncInterval, Codec codec = NULL_CODEC); - DataFileWriterBase(std::unique_ptr<OutputStream> outputStream, - const ValidSchema& schema, size_t syncInterval, Codec codec); - - ~DataFileWriterBase(); - /** - * Closes the current file. Once closed this datafile object cannot be - * used for writing any more. - */ - void close(); - - /** - * Returns the schema for this data file. - */ - const ValidSchema& schema() const { return schema_; } - - /** - * Flushes any unwritten data into the file. - */ - void flush(); -}; - -/** - * An Avro datafile that can store objects of type T. - */ -template <typename T> -class DataFileWriter : boost::noncopyable { - std::unique_ptr<DataFileWriterBase> base_; -public: - /** - * Constructs a new data file. - */ - DataFileWriter(const char* filename, const ValidSchema& schema, - size_t syncInterval = 16 * 1024, Codec codec = NULL_CODEC) : - base_(new DataFileWriterBase(filename, schema, syncInterval, codec)) { } - - DataFileWriter(std::unique_ptr<OutputStream> outputStream, const ValidSchema& schema, - size_t syncInterval = 16 * 1024, Codec codec = NULL_CODEC) : - base_(new DataFileWriterBase(std::move(outputStream), schema, syncInterval, codec)) { } - - /** - * Writes the given piece of data into the file. - */ - void write(const T& datum) { - base_->syncIfNeeded(); - avro::encode(base_->encoder(), datum); - base_->incr(); - } - - /** - * Returns the byte offset (within the current file) of the start of the current block being written. - */ - uint64_t getCurrentBlockStart() { return base_->getCurrentBlockStart(); } - - - /** - * Closes the current file. Once closed this datafile object cannot be - * used for writing any more. - */ - void close() { base_->close(); } - - /** - * Returns the schema for this data file. - */ - const ValidSchema& schema() const { return base_->schema(); } - - /** - * Flushes any unwritten data into the file. - */ - void flush() { base_->flush(); } -}; - -/** - * The type independent portion of reader. - */ -class AVRO_DECL DataFileReaderBase : boost::noncopyable { - const std::string filename_; - const std::unique_ptr<InputStream> stream_; - const DecoderPtr decoder_; - int64_t objectCount_; - bool eof_; - Codec codec_; - int64_t blockStart_; - int64_t blockEnd_; - - ValidSchema readerSchema_; - ValidSchema dataSchema_; - DecoderPtr dataDecoder_; - std::unique_ptr<InputStream> dataStream_; - typedef std::map<std::string, std::vector<uint8_t> > Metadata; - - Metadata metadata_; - DataFileSync sync_; - - // for compressed buffer - std::unique_ptr<boost::iostreams::filtering_istream> os_; - std::vector<char> compressed_; - std::string uncompressed; - void readHeader(); - - void readDataBlock(); - void doSeek(int64_t position); -public: - /** - * Returns the current decoder for this reader. - */ - Decoder& decoder() { return *dataDecoder_; } - - /** - * Returns true if and only if there is more to read. - */ - bool hasMore(); - - /** - * Decrements the number of objects yet to read. - */ - void decr() { --objectCount_; } - - /** - * Constructs the reader for the given file and the reader is - * expected to use the schema that is used with data. - * This function should be called exactly once after constructing - * the DataFileReaderBase object. - */ - DataFileReaderBase(const char* filename); - - DataFileReaderBase(std::unique_ptr<InputStream> inputStream); - - /** - * Initializes the reader so that the reader and writer schemas - * are the same. - */ - void init(); - - /** - * Initializes the reader to read objects according to the given - * schema. This gives an opportunity for the reader to see the schema - * in the data file before deciding the right schema to use for reading. - * This must be called exactly once after constructing the - * DataFileReaderBase object. - */ - void init(const ValidSchema& readerSchema); - - /** - * Returns the schema for this object. - */ - const ValidSchema& readerSchema() { return readerSchema_; } - - /** - * Returns the schema stored with the data file. - */ - const ValidSchema& dataSchema() { return dataSchema_; } - - /** - * Closes the reader. No further operation is possible on this reader. - */ - void close(); - - /** - * Move to a specific, known synchronization point, for example one returned - * from tell() after sync(). - */ - void seek(int64_t position); - - /** - * Move to the next synchronization point after a position. To process a - * range of file entries, call this with the starting position, then check - * pastSync() with the end point before each use of decoder(). - */ - void sync(int64_t position); - - /** - * Return true if past the next synchronization point after a position. - */ - bool pastSync(int64_t position); - - /** - * Return the last synchronization point before our current position. - */ - int64_t previousSync(); -}; - -/** - * Reads the contents of data file one after another. - */ -template <typename T> -class DataFileReader : boost::noncopyable { - std::unique_ptr<DataFileReaderBase> base_; -public: - /** - * Constructs the reader for the given file and the reader is - * expected to use the given schema. - */ - DataFileReader(const char* filename, const ValidSchema& readerSchema) : - base_(new DataFileReaderBase(filename)) { - base_->init(readerSchema); - } - - DataFileReader(std::unique_ptr<InputStream> inputStream, const ValidSchema& readerSchema) : - base_(new DataFileReaderBase(std::move(inputStream))) { - base_->init(readerSchema); - } - - /** - * Constructs the reader for the given file and the reader is - * expected to use the schema that is used with data. - */ - DataFileReader(const char* filename) : - base_(new DataFileReaderBase(filename)) { - base_->init(); - } - - DataFileReader(std::unique_ptr<InputStream> inputStream) : - base_(new DataFileReaderBase(std::move(inputStream))) { - base_->init(); - } - - /** - * Constructs a reader using the reader base. This form of constructor - * allows the user to examine the schema of a given file and then - * decide to use the right type of data to be deserialize. Without this - * the user must know the type of data for the template _before_ - * he knows the schema within the file. - * The schema present in the data file will be used for reading - * from this reader. - */ - DataFileReader(std::unique_ptr<DataFileReaderBase> base) : base_(std::move(base)) { - base_->init(); - } - - /** - * Constructs a reader using the reader base. This form of constructor - * allows the user to examine the schema of a given file and then - * decide to use the right type of data to be deserialize. Without this - * the user must know the type of data for the template _before_ - * he knows the schema within the file. - * The argument readerSchema will be used for reading - * from this reader. - */ - DataFileReader(std::unique_ptr<DataFileReaderBase> base, - const ValidSchema& readerSchema) : base_(std::move(base)) { - base_->init(readerSchema); - } - - /** - * Reads the next entry from the data file. - * \return true if an object has been successfully read into \p datum and - * false if there are no more entries in the file. - */ - bool read(T& datum) { - if (base_->hasMore()) { - base_->decr(); - avro::decode(base_->decoder(), datum); - return true; - } - return false; - } - - /** - * Returns the schema for this object. - */ - const ValidSchema& readerSchema() { return base_->readerSchema(); } - - /** - * Returns the schema stored with the data file. - */ - const ValidSchema& dataSchema() { return base_->dataSchema(); } - - /** - * Closes the reader. No further operation is possible on this reader. - */ - void close() { return base_->close(); } - - /** - * Move to a specific, known synchronization point, for example one returned - * from previousSync(). - */ - void seek(int64_t position) { base_->seek(position); } - - /** - * Move to the next synchronization point after a position. To process a - * range of file entries, call this with the starting position, then check - * pastSync() with the end point before each call to read(). - */ - void sync(int64_t position) { base_->sync(position); } - - /** - * Return true if past the next synchronization point after a position. - */ - bool pastSync(int64_t position) { return base_->pastSync(position); } - - /** - * Return the last synchronization point before our current position. - */ - int64_t previousSync() { return base_->previousSync(); } -}; - -} // namespace avro -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_DataFile_hh__ +#define avro_DataFile_hh__ + +#include "Config.hh" +#include "Encoder.hh" +#include "buffer/Buffer.hh" +#include "ValidSchema.hh" +#include "Specific.hh" +#include "Stream.hh" + +#include <map> +#include <string> +#include <vector> + +#include "array" +#include "boost/utility.hpp" +#include <boost/iostreams/filtering_stream.hpp> + +namespace avro { + +/** Specify type of compression to use when writing data files. */ +enum Codec { + NULL_CODEC, + DEFLATE_CODEC, + +#ifdef SNAPPY_CODEC_AVAILABLE + SNAPPY_CODEC +#endif + +}; + +const int SyncSize = 16; +/** + * The sync value. + */ +typedef std::array<uint8_t, SyncSize> DataFileSync; + +/** + * Type-independent portion of DataFileWriter. + * At any given point in time, at most one file can be written using + * this object. + */ +class AVRO_DECL DataFileWriterBase : boost::noncopyable { + const std::string filename_; + const ValidSchema schema_; + const EncoderPtr encoderPtr_; + const size_t syncInterval_; + Codec codec_; + + std::unique_ptr<OutputStream> stream_; + std::unique_ptr<OutputStream> buffer_; + const DataFileSync sync_; + int64_t objectCount_; + + typedef std::map<std::string, std::vector<uint8_t> > Metadata; + + Metadata metadata_; + int64_t lastSync_; + + static std::unique_ptr<OutputStream> makeStream(const char* filename); + static DataFileSync makeSync(); + + void writeHeader(); + void setMetadata(const std::string& key, const std::string& value); + + /** + * Generates a sync marker in the file. + */ + void sync(); + + /** + * Shared constructor portion since we aren't using C++11 + */ + void init(const ValidSchema &schema, size_t syncInterval, const Codec &codec); + +public: + /** + * Returns the current encoder for this writer. + */ + Encoder& encoder() const { return *encoderPtr_; } + + /** + * Returns true if the buffer has sufficient data for a sync to be + * inserted. + */ + void syncIfNeeded(); + + /** + * Returns the byte offset (within the current file) of the start of the current block being written. + */ + uint64_t getCurrentBlockStart(); + + /** + * Increments the object count. + */ + void incr() { + ++objectCount_; + } + /** + * Constructs a data file writer with the given sync interval and name. + */ + DataFileWriterBase(const char* filename, const ValidSchema& schema, + size_t syncInterval, Codec codec = NULL_CODEC); + DataFileWriterBase(std::unique_ptr<OutputStream> outputStream, + const ValidSchema& schema, size_t syncInterval, Codec codec); + + ~DataFileWriterBase(); + /** + * Closes the current file. Once closed this datafile object cannot be + * used for writing any more. + */ + void close(); + + /** + * Returns the schema for this data file. + */ + const ValidSchema& schema() const { return schema_; } + + /** + * Flushes any unwritten data into the file. + */ + void flush(); +}; + +/** + * An Avro datafile that can store objects of type T. + */ +template <typename T> +class DataFileWriter : boost::noncopyable { + std::unique_ptr<DataFileWriterBase> base_; +public: + /** + * Constructs a new data file. + */ + DataFileWriter(const char* filename, const ValidSchema& schema, + size_t syncInterval = 16 * 1024, Codec codec = NULL_CODEC) : + base_(new DataFileWriterBase(filename, schema, syncInterval, codec)) { } + + DataFileWriter(std::unique_ptr<OutputStream> outputStream, const ValidSchema& schema, + size_t syncInterval = 16 * 1024, Codec codec = NULL_CODEC) : + base_(new DataFileWriterBase(std::move(outputStream), schema, syncInterval, codec)) { } + + /** + * Writes the given piece of data into the file. + */ + void write(const T& datum) { + base_->syncIfNeeded(); + avro::encode(base_->encoder(), datum); + base_->incr(); + } + + /** + * Returns the byte offset (within the current file) of the start of the current block being written. + */ + uint64_t getCurrentBlockStart() { return base_->getCurrentBlockStart(); } + + + /** + * Closes the current file. Once closed this datafile object cannot be + * used for writing any more. + */ + void close() { base_->close(); } + + /** + * Returns the schema for this data file. + */ + const ValidSchema& schema() const { return base_->schema(); } + + /** + * Flushes any unwritten data into the file. + */ + void flush() { base_->flush(); } +}; + +/** + * The type independent portion of reader. + */ +class AVRO_DECL DataFileReaderBase : boost::noncopyable { + const std::string filename_; + const std::unique_ptr<InputStream> stream_; + const DecoderPtr decoder_; + int64_t objectCount_; + bool eof_; + Codec codec_; + int64_t blockStart_; + int64_t blockEnd_; + + ValidSchema readerSchema_; + ValidSchema dataSchema_; + DecoderPtr dataDecoder_; + std::unique_ptr<InputStream> dataStream_; + typedef std::map<std::string, std::vector<uint8_t> > Metadata; + + Metadata metadata_; + DataFileSync sync_; + + // for compressed buffer + std::unique_ptr<boost::iostreams::filtering_istream> os_; + std::vector<char> compressed_; + std::string uncompressed; + void readHeader(); + + void readDataBlock(); + void doSeek(int64_t position); +public: + /** + * Returns the current decoder for this reader. + */ + Decoder& decoder() { return *dataDecoder_; } + + /** + * Returns true if and only if there is more to read. + */ + bool hasMore(); + + /** + * Decrements the number of objects yet to read. + */ + void decr() { --objectCount_; } + + /** + * Constructs the reader for the given file and the reader is + * expected to use the schema that is used with data. + * This function should be called exactly once after constructing + * the DataFileReaderBase object. + */ + DataFileReaderBase(const char* filename); + + DataFileReaderBase(std::unique_ptr<InputStream> inputStream); + + /** + * Initializes the reader so that the reader and writer schemas + * are the same. + */ + void init(); + + /** + * Initializes the reader to read objects according to the given + * schema. This gives an opportunity for the reader to see the schema + * in the data file before deciding the right schema to use for reading. + * This must be called exactly once after constructing the + * DataFileReaderBase object. + */ + void init(const ValidSchema& readerSchema); + + /** + * Returns the schema for this object. + */ + const ValidSchema& readerSchema() { return readerSchema_; } + + /** + * Returns the schema stored with the data file. + */ + const ValidSchema& dataSchema() { return dataSchema_; } + + /** + * Closes the reader. No further operation is possible on this reader. + */ + void close(); + + /** + * Move to a specific, known synchronization point, for example one returned + * from tell() after sync(). + */ + void seek(int64_t position); + + /** + * Move to the next synchronization point after a position. To process a + * range of file entries, call this with the starting position, then check + * pastSync() with the end point before each use of decoder(). + */ + void sync(int64_t position); + + /** + * Return true if past the next synchronization point after a position. + */ + bool pastSync(int64_t position); + + /** + * Return the last synchronization point before our current position. + */ + int64_t previousSync(); +}; + +/** + * Reads the contents of data file one after another. + */ +template <typename T> +class DataFileReader : boost::noncopyable { + std::unique_ptr<DataFileReaderBase> base_; +public: + /** + * Constructs the reader for the given file and the reader is + * expected to use the given schema. + */ + DataFileReader(const char* filename, const ValidSchema& readerSchema) : + base_(new DataFileReaderBase(filename)) { + base_->init(readerSchema); + } + + DataFileReader(std::unique_ptr<InputStream> inputStream, const ValidSchema& readerSchema) : + base_(new DataFileReaderBase(std::move(inputStream))) { + base_->init(readerSchema); + } + + /** + * Constructs the reader for the given file and the reader is + * expected to use the schema that is used with data. + */ + DataFileReader(const char* filename) : + base_(new DataFileReaderBase(filename)) { + base_->init(); + } + + DataFileReader(std::unique_ptr<InputStream> inputStream) : + base_(new DataFileReaderBase(std::move(inputStream))) { + base_->init(); + } + + /** + * Constructs a reader using the reader base. This form of constructor + * allows the user to examine the schema of a given file and then + * decide to use the right type of data to be deserialize. Without this + * the user must know the type of data for the template _before_ + * he knows the schema within the file. + * The schema present in the data file will be used for reading + * from this reader. + */ + DataFileReader(std::unique_ptr<DataFileReaderBase> base) : base_(std::move(base)) { + base_->init(); + } + + /** + * Constructs a reader using the reader base. This form of constructor + * allows the user to examine the schema of a given file and then + * decide to use the right type of data to be deserialize. Without this + * the user must know the type of data for the template _before_ + * he knows the schema within the file. + * The argument readerSchema will be used for reading + * from this reader. + */ + DataFileReader(std::unique_ptr<DataFileReaderBase> base, + const ValidSchema& readerSchema) : base_(std::move(base)) { + base_->init(readerSchema); + } + + /** + * Reads the next entry from the data file. + * \return true if an object has been successfully read into \p datum and + * false if there are no more entries in the file. + */ + bool read(T& datum) { + if (base_->hasMore()) { + base_->decr(); + avro::decode(base_->decoder(), datum); + return true; + } + return false; + } + + /** + * Returns the schema for this object. + */ + const ValidSchema& readerSchema() { return base_->readerSchema(); } + + /** + * Returns the schema stored with the data file. + */ + const ValidSchema& dataSchema() { return base_->dataSchema(); } + + /** + * Closes the reader. No further operation is possible on this reader. + */ + void close() { return base_->close(); } + + /** + * Move to a specific, known synchronization point, for example one returned + * from previousSync(). + */ + void seek(int64_t position) { base_->seek(position); } + + /** + * Move to the next synchronization point after a position. To process a + * range of file entries, call this with the starting position, then check + * pastSync() with the end point before each call to read(). + */ + void sync(int64_t position) { base_->sync(position); } + + /** + * Return true if past the next synchronization point after a position. + */ + bool pastSync(int64_t position) { return base_->pastSync(position); } + + /** + * Return the last synchronization point before our current position. + */ + int64_t previousSync() { return base_->previousSync(); } +}; + +} // namespace avro +#endif diff --git a/contrib/libs/apache/avro/api/Decoder.hh b/contrib/libs/apache/avro/api/Decoder.hh index c57fc05505..5356d79f32 100644 --- a/contrib/libs/apache/avro/api/Decoder.hh +++ b/contrib/libs/apache/avro/api/Decoder.hh @@ -1,226 +1,226 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Decoder_hh__ -#define avro_Decoder_hh__ - -#include "Config.hh" -#include <stdint.h> -#include <string> -#include <vector> -#include <memory> - -#include "ValidSchema.hh" -#include "Stream.hh" - -/// \file -/// -/// Low level support for decoding avro values. -/// This class has two types of functions. One type of functions support -/// decoding of leaf values (for example, decodeLong and -/// decodeString). These functions have analogs in Encoder. -/// -/// The other type of functions support decoding of maps and arrays. -/// These functions are arrayStart, startItem, and arrayEnd -/// (and similar functions for maps). - -namespace avro { - -/** - * Decoder is an interface implemented by every decoder capable - * of decoding Avro data. - */ -class AVRO_DECL Decoder { -public: - virtual ~Decoder() { }; - /// All future decoding will come from is, which should be valid - /// until replaced by another call to init() or this Decoder is - /// destructed. - virtual void init(InputStream& is) = 0; - - /// Decodes a null from the current stream. - virtual void decodeNull() = 0; - - /// Decodes a bool from the current stream - virtual bool decodeBool() = 0; - - /// Decodes a 32-bit int from the current stream. - virtual int32_t decodeInt() = 0; - - /// Decodes a 64-bit signed int from the current stream. - virtual int64_t decodeLong() = 0; - - /// Decodes a single-precision floating point number from current stream. - virtual float decodeFloat() = 0; - - /// Decodes a double-precision floating point number from current stream. - virtual double decodeDouble() = 0; - - /// Decodes a UTF-8 string from the current stream. - std::string decodeString() { - std::string result; - decodeString(result); - return result; - } - - /** - * Decodes a UTF-8 string from the stream and assigns it to value. - */ - virtual void decodeString(std::string& value) = 0; - - /// Skips a string on the current stream. - virtual void skipString() = 0; - - /// Decodes arbitray binary data from the current stream. - std::vector<uint8_t> decodeBytes() { - std::vector<uint8_t> result; - decodeBytes(result); - return result; - } - - /// Decodes arbitrary binary data from the current stream and puts it - /// in value. - virtual void decodeBytes(std::vector<uint8_t>& value) = 0; - - /// Skips bytes on the current stream. - virtual void skipBytes() = 0; - - /** - * Decodes fixed length binary from the current stream. - * \param[in] n The size (byte count) of the fixed being read. - * \return The fixed data that has been read. The size of the returned - * vector is guaranteed to be equal to \p n. - */ - std::vector<uint8_t> decodeFixed(size_t n) { - std::vector<uint8_t> result; - decodeFixed(n, result); - return result; - } - - /** - * Decodes a fixed from the current stream. - * \param[in] n The size (byte count) of the fixed being read. - * \param[out] value The value that receives the fixed. The vector will - * be size-adjusted based on the fixed's size. - */ - virtual void decodeFixed(size_t n, std::vector<uint8_t>& value) = 0; - - /// Skips fixed length binary on the current stream. - virtual void skipFixed(size_t n) = 0; - - /// Decodes enum from the current stream. - virtual size_t decodeEnum() = 0; - - /// Start decoding an array. Returns the number of entries in first chunk. - virtual size_t arrayStart() = 0; - - /// Returns the number of entries in next chunk. 0 if last. - virtual size_t arrayNext() = 0; - - /// Tries to skip an array. If it can, it returns 0. Otherwise - /// it returns the number of elements to be skipped. The client - /// should skip the individual items. In such cases, skipArray - /// is identical to arrayStart. - virtual size_t skipArray() = 0; - - /// Start decoding a map. Returns the number of entries in first chunk. - virtual size_t mapStart() = 0; - - /// Returns the number of entries in next chunk. 0 if last. - virtual size_t mapNext() = 0; - - /// Tries to skip a map. If it can, it returns 0. Otherwise - /// it returns the number of elements to be skipped. The client - /// should skip the individual items. In such cases, skipMap - /// is identical to mapStart. - virtual size_t skipMap() = 0; - - /// Decodes a branch of a union. The actual value is to follow. - virtual size_t decodeUnionIndex() = 0; - - /// Drains any additional data at the end of the current entry in a stream. - /// It also returns any unused bytes back to any underlying input stream. - /// One situation this happens is when the reader's schema and - /// the writer's schema are records but are different and the writer's - /// record has more fields at the end of the record. - /// Leaving such data unread is usually not a problem. If multiple - /// records are stored consecutively in a stream (e.g. Avro data file) - /// any attempt to read the next record will automatically skip - /// those extra fields of the current record. It would still leave - /// the extra fields at the end of the last record in the stream. - /// This would mean that the stream is not in a good state. For example, - /// if some non-avro information is stored at the end of the stream, - /// the consumers of such data would see the bytes left behind - /// by the avro decoder. Similar set of problems occur if the Decoder - /// consumes more than what it should. - virtual void drain() = 0; -}; - -/** - * Shared pointer to Decoder. - */ -typedef std::shared_ptr<Decoder> DecoderPtr; - -/** - * ResolvingDecoder is derived from \ref Decoder, with an additional - * function to obtain the field ordering of fields within a record. - */ -class AVRO_DECL ResolvingDecoder : public Decoder { -public: - /// Returns the order of fields for records. - /// The order of fields could be different from the order of their - /// order in the schema because the writer's field order could - /// be different. In order to avoid buffering and later use, - /// we return the values in the writer's field order. - virtual const std::vector<size_t>& fieldOrder() = 0; -}; - -/** - * Shared pointer to ResolvingDecoder. - */ -typedef std::shared_ptr<ResolvingDecoder> ResolvingDecoderPtr; -/** - * Returns an decoder that can decode binary Avro standard. - */ -AVRO_DECL DecoderPtr binaryDecoder(); - -/** - * Returns an decoder that validates sequence of calls to an underlying - * Decoder against the given schema. - */ -AVRO_DECL DecoderPtr validatingDecoder(const ValidSchema& schema, - const DecoderPtr& base); - -/** - * Returns an decoder that can decode Avro standard for JSON. - */ -AVRO_DECL DecoderPtr jsonDecoder(const ValidSchema& schema); - -/** - * Returns a decoder that decodes avro data from base written according to - * writerSchema and resolves against readerSchema. - * The client uses the decoder as if the data were written using readerSchema. - * // FIXME: Handle out of order fields. - */ -AVRO_DECL ResolvingDecoderPtr resolvingDecoder(const ValidSchema& writer, - const ValidSchema& reader, const DecoderPtr& base); - - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Decoder_hh__ +#define avro_Decoder_hh__ + +#include "Config.hh" +#include <stdint.h> +#include <string> +#include <vector> +#include <memory> + +#include "ValidSchema.hh" +#include "Stream.hh" + +/// \file +/// +/// Low level support for decoding avro values. +/// This class has two types of functions. One type of functions support +/// decoding of leaf values (for example, decodeLong and +/// decodeString). These functions have analogs in Encoder. +/// +/// The other type of functions support decoding of maps and arrays. +/// These functions are arrayStart, startItem, and arrayEnd +/// (and similar functions for maps). + +namespace avro { + +/** + * Decoder is an interface implemented by every decoder capable + * of decoding Avro data. + */ +class AVRO_DECL Decoder { +public: + virtual ~Decoder() { }; + /// All future decoding will come from is, which should be valid + /// until replaced by another call to init() or this Decoder is + /// destructed. + virtual void init(InputStream& is) = 0; + + /// Decodes a null from the current stream. + virtual void decodeNull() = 0; + + /// Decodes a bool from the current stream + virtual bool decodeBool() = 0; + + /// Decodes a 32-bit int from the current stream. + virtual int32_t decodeInt() = 0; + + /// Decodes a 64-bit signed int from the current stream. + virtual int64_t decodeLong() = 0; + + /// Decodes a single-precision floating point number from current stream. + virtual float decodeFloat() = 0; + + /// Decodes a double-precision floating point number from current stream. + virtual double decodeDouble() = 0; + + /// Decodes a UTF-8 string from the current stream. + std::string decodeString() { + std::string result; + decodeString(result); + return result; + } + + /** + * Decodes a UTF-8 string from the stream and assigns it to value. + */ + virtual void decodeString(std::string& value) = 0; + + /// Skips a string on the current stream. + virtual void skipString() = 0; + + /// Decodes arbitray binary data from the current stream. + std::vector<uint8_t> decodeBytes() { + std::vector<uint8_t> result; + decodeBytes(result); + return result; + } + + /// Decodes arbitrary binary data from the current stream and puts it + /// in value. + virtual void decodeBytes(std::vector<uint8_t>& value) = 0; + + /// Skips bytes on the current stream. + virtual void skipBytes() = 0; + + /** + * Decodes fixed length binary from the current stream. + * \param[in] n The size (byte count) of the fixed being read. + * \return The fixed data that has been read. The size of the returned + * vector is guaranteed to be equal to \p n. + */ + std::vector<uint8_t> decodeFixed(size_t n) { + std::vector<uint8_t> result; + decodeFixed(n, result); + return result; + } + + /** + * Decodes a fixed from the current stream. + * \param[in] n The size (byte count) of the fixed being read. + * \param[out] value The value that receives the fixed. The vector will + * be size-adjusted based on the fixed's size. + */ + virtual void decodeFixed(size_t n, std::vector<uint8_t>& value) = 0; + + /// Skips fixed length binary on the current stream. + virtual void skipFixed(size_t n) = 0; + + /// Decodes enum from the current stream. + virtual size_t decodeEnum() = 0; + + /// Start decoding an array. Returns the number of entries in first chunk. + virtual size_t arrayStart() = 0; + + /// Returns the number of entries in next chunk. 0 if last. + virtual size_t arrayNext() = 0; + + /// Tries to skip an array. If it can, it returns 0. Otherwise + /// it returns the number of elements to be skipped. The client + /// should skip the individual items. In such cases, skipArray + /// is identical to arrayStart. + virtual size_t skipArray() = 0; + + /// Start decoding a map. Returns the number of entries in first chunk. + virtual size_t mapStart() = 0; + + /// Returns the number of entries in next chunk. 0 if last. + virtual size_t mapNext() = 0; + + /// Tries to skip a map. If it can, it returns 0. Otherwise + /// it returns the number of elements to be skipped. The client + /// should skip the individual items. In such cases, skipMap + /// is identical to mapStart. + virtual size_t skipMap() = 0; + + /// Decodes a branch of a union. The actual value is to follow. + virtual size_t decodeUnionIndex() = 0; + + /// Drains any additional data at the end of the current entry in a stream. + /// It also returns any unused bytes back to any underlying input stream. + /// One situation this happens is when the reader's schema and + /// the writer's schema are records but are different and the writer's + /// record has more fields at the end of the record. + /// Leaving such data unread is usually not a problem. If multiple + /// records are stored consecutively in a stream (e.g. Avro data file) + /// any attempt to read the next record will automatically skip + /// those extra fields of the current record. It would still leave + /// the extra fields at the end of the last record in the stream. + /// This would mean that the stream is not in a good state. For example, + /// if some non-avro information is stored at the end of the stream, + /// the consumers of such data would see the bytes left behind + /// by the avro decoder. Similar set of problems occur if the Decoder + /// consumes more than what it should. + virtual void drain() = 0; +}; + +/** + * Shared pointer to Decoder. + */ +typedef std::shared_ptr<Decoder> DecoderPtr; + +/** + * ResolvingDecoder is derived from \ref Decoder, with an additional + * function to obtain the field ordering of fields within a record. + */ +class AVRO_DECL ResolvingDecoder : public Decoder { +public: + /// Returns the order of fields for records. + /// The order of fields could be different from the order of their + /// order in the schema because the writer's field order could + /// be different. In order to avoid buffering and later use, + /// we return the values in the writer's field order. + virtual const std::vector<size_t>& fieldOrder() = 0; +}; + +/** + * Shared pointer to ResolvingDecoder. + */ +typedef std::shared_ptr<ResolvingDecoder> ResolvingDecoderPtr; +/** + * Returns an decoder that can decode binary Avro standard. + */ +AVRO_DECL DecoderPtr binaryDecoder(); + +/** + * Returns an decoder that validates sequence of calls to an underlying + * Decoder against the given schema. + */ +AVRO_DECL DecoderPtr validatingDecoder(const ValidSchema& schema, + const DecoderPtr& base); + +/** + * Returns an decoder that can decode Avro standard for JSON. + */ +AVRO_DECL DecoderPtr jsonDecoder(const ValidSchema& schema); + +/** + * Returns a decoder that decodes avro data from base written according to + * writerSchema and resolves against readerSchema. + * The client uses the decoder as if the data were written using readerSchema. + * // FIXME: Handle out of order fields. + */ +AVRO_DECL ResolvingDecoderPtr resolvingDecoder(const ValidSchema& writer, + const ValidSchema& reader, const DecoderPtr& base); + + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Encoder.hh b/contrib/libs/apache/avro/api/Encoder.hh index 0d1f198e76..7849e93458 100644 --- a/contrib/libs/apache/avro/api/Encoder.hh +++ b/contrib/libs/apache/avro/api/Encoder.hh @@ -1,173 +1,173 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Encoder_hh__ -#define avro_Encoder_hh__ - -#include "Config.hh" -#include <stdint.h> -#include <string> -#include <vector> -#include <memory> - -#include "ValidSchema.hh" -#include "Stream.hh" - -/// \file -/// -/// Low level support for encoding avro values. -/// This class has two types of funtions. One type of functions support -/// the writing of leaf values (for example, encodeLong and -/// encodeString). These functions have analogs in Decoder. -/// -/// The other type of functions support the writing of maps and arrays. -/// These functions are arrayStart, startItem, and arrayEnd -/// (and similar functions for maps). -/// Some implementations of Encoder handle the -/// buffering required to break large maps and arrays into blocks, -/// which is necessary for applications that want to do streaming. - -namespace avro { - -/** - * The abstract base class for all Avro encoders. The implementations - * differ in the method of encoding (binary vresus JSON) or in capabilities - * such as ability to verify the order of invocation of different functions. - */ -class AVRO_DECL Encoder { -public: - virtual ~Encoder() { }; - /// All future encodings will go to os, which should be valid until - /// it is reset with another call to init() or the encoder is - /// destructed. - virtual void init(OutputStream& os) = 0; - - /// Flushes any data in internal buffers. - virtual void flush() = 0; - - /// Returns the number of bytes produced so far. - /// For a meaningful value, do a flush() before invoking this function. - virtual int64_t byteCount() const = 0; - - /// Encodes a null to the current stream. - virtual void encodeNull() = 0; - - /// Encodes a bool to the current stream - virtual void encodeBool(bool b) = 0; - - /// Encodes a 32-bit int to the current stream. - virtual void encodeInt(int32_t i) = 0; - - /// Encodes a 64-bit signed int to the current stream. - virtual void encodeLong(int64_t l) = 0; - - /// Encodes a single-precision floating point number to the current stream. - virtual void encodeFloat(float f) = 0; - - /// Encodes a double-precision floating point number to the current stream. - virtual void encodeDouble(double d) = 0; - - /// Encodes a UTF-8 string to the current stream. - virtual void encodeString(const std::string& s) = 0; - - /** - * Encodes aribtray binary data into tthe current stream as Avro "bytes" - * data type. - * \param bytes Where the data is - * \param len Number of bytes at \p bytes. - */ - virtual void encodeBytes(const uint8_t *bytes, size_t len) = 0; - - /** - * Encodes aribtray binary data into tthe current stream as Avro "bytes" - * data type. - * \param bytes The data. - */ - void encodeBytes(const std::vector<uint8_t>& bytes) { - uint8_t b = 0; - encodeBytes(bytes.empty() ? &b : bytes.data(), bytes.size()); - } - - /// Encodes fixed length binary to the current stream. - virtual void encodeFixed(const uint8_t *bytes, size_t len) = 0; - - /** - * Encodes an Avro data type Fixed. - * \param bytes The fixed, the length of which is taken as the size - * of fixed. - */ - void encodeFixed(const std::vector<uint8_t>& bytes) { - encodeFixed(bytes.data(), bytes.size()); - } - - /// Encodes enum to the current stream. - virtual void encodeEnum(size_t e) = 0; - - /// Indicates that an array of items is being encoded. - virtual void arrayStart() = 0; - - /// Indicates that the current array of items have ended. - virtual void arrayEnd() = 0; - - /// Indicates that a map of items is being encoded. - virtual void mapStart() = 0; - - /// Indicates that the current map of items have ended. - virtual void mapEnd() = 0; - - /// Indicates that count number of items are to follow in the current array - /// or map. - virtual void setItemCount(size_t count) = 0; - - /// Marks a beginning of an item in the current array or map. - virtual void startItem() = 0; - - /// Encodes a branch of a union. The actual value is to follow. - virtual void encodeUnionIndex(size_t e) = 0; -}; - -/** - * Shared pointer to Encoder. - */ -typedef std::shared_ptr<Encoder> EncoderPtr; - -/** - * Returns an encoder that can encode binary Avro standard. - */ -AVRO_DECL EncoderPtr binaryEncoder(); - -/** - * Returns an encoder that validates sequence of calls to an underlying - * Encoder against the given schema. - */ -AVRO_DECL EncoderPtr validatingEncoder(const ValidSchema& schema, - const EncoderPtr& base); - -/** - * Returns an encoder that encodes Avro standard for JSON. - */ -AVRO_DECL EncoderPtr jsonEncoder(const ValidSchema& schema); - -/** - * Returns an encoder that encodes Avro standard for pretty printed JSON. - */ -AVRO_DECL EncoderPtr jsonPrettyEncoder(const ValidSchema& schema); - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Encoder_hh__ +#define avro_Encoder_hh__ + +#include "Config.hh" +#include <stdint.h> +#include <string> +#include <vector> +#include <memory> + +#include "ValidSchema.hh" +#include "Stream.hh" + +/// \file +/// +/// Low level support for encoding avro values. +/// This class has two types of funtions. One type of functions support +/// the writing of leaf values (for example, encodeLong and +/// encodeString). These functions have analogs in Decoder. +/// +/// The other type of functions support the writing of maps and arrays. +/// These functions are arrayStart, startItem, and arrayEnd +/// (and similar functions for maps). +/// Some implementations of Encoder handle the +/// buffering required to break large maps and arrays into blocks, +/// which is necessary for applications that want to do streaming. + +namespace avro { + +/** + * The abstract base class for all Avro encoders. The implementations + * differ in the method of encoding (binary vresus JSON) or in capabilities + * such as ability to verify the order of invocation of different functions. + */ +class AVRO_DECL Encoder { +public: + virtual ~Encoder() { }; + /// All future encodings will go to os, which should be valid until + /// it is reset with another call to init() or the encoder is + /// destructed. + virtual void init(OutputStream& os) = 0; + + /// Flushes any data in internal buffers. + virtual void flush() = 0; + + /// Returns the number of bytes produced so far. + /// For a meaningful value, do a flush() before invoking this function. + virtual int64_t byteCount() const = 0; + + /// Encodes a null to the current stream. + virtual void encodeNull() = 0; + + /// Encodes a bool to the current stream + virtual void encodeBool(bool b) = 0; + + /// Encodes a 32-bit int to the current stream. + virtual void encodeInt(int32_t i) = 0; + + /// Encodes a 64-bit signed int to the current stream. + virtual void encodeLong(int64_t l) = 0; + + /// Encodes a single-precision floating point number to the current stream. + virtual void encodeFloat(float f) = 0; + + /// Encodes a double-precision floating point number to the current stream. + virtual void encodeDouble(double d) = 0; + + /// Encodes a UTF-8 string to the current stream. + virtual void encodeString(const std::string& s) = 0; + + /** + * Encodes aribtray binary data into tthe current stream as Avro "bytes" + * data type. + * \param bytes Where the data is + * \param len Number of bytes at \p bytes. + */ + virtual void encodeBytes(const uint8_t *bytes, size_t len) = 0; + + /** + * Encodes aribtray binary data into tthe current stream as Avro "bytes" + * data type. + * \param bytes The data. + */ + void encodeBytes(const std::vector<uint8_t>& bytes) { + uint8_t b = 0; + encodeBytes(bytes.empty() ? &b : bytes.data(), bytes.size()); + } + + /// Encodes fixed length binary to the current stream. + virtual void encodeFixed(const uint8_t *bytes, size_t len) = 0; + + /** + * Encodes an Avro data type Fixed. + * \param bytes The fixed, the length of which is taken as the size + * of fixed. + */ + void encodeFixed(const std::vector<uint8_t>& bytes) { + encodeFixed(bytes.data(), bytes.size()); + } + + /// Encodes enum to the current stream. + virtual void encodeEnum(size_t e) = 0; + + /// Indicates that an array of items is being encoded. + virtual void arrayStart() = 0; + + /// Indicates that the current array of items have ended. + virtual void arrayEnd() = 0; + + /// Indicates that a map of items is being encoded. + virtual void mapStart() = 0; + + /// Indicates that the current map of items have ended. + virtual void mapEnd() = 0; + + /// Indicates that count number of items are to follow in the current array + /// or map. + virtual void setItemCount(size_t count) = 0; + + /// Marks a beginning of an item in the current array or map. + virtual void startItem() = 0; + + /// Encodes a branch of a union. The actual value is to follow. + virtual void encodeUnionIndex(size_t e) = 0; +}; + +/** + * Shared pointer to Encoder. + */ +typedef std::shared_ptr<Encoder> EncoderPtr; + +/** + * Returns an encoder that can encode binary Avro standard. + */ +AVRO_DECL EncoderPtr binaryEncoder(); + +/** + * Returns an encoder that validates sequence of calls to an underlying + * Encoder against the given schema. + */ +AVRO_DECL EncoderPtr validatingEncoder(const ValidSchema& schema, + const EncoderPtr& base); + +/** + * Returns an encoder that encodes Avro standard for JSON. + */ +AVRO_DECL EncoderPtr jsonEncoder(const ValidSchema& schema); + +/** + * Returns an encoder that encodes Avro standard for pretty printed JSON. + */ +AVRO_DECL EncoderPtr jsonPrettyEncoder(const ValidSchema& schema); + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Exception.hh b/contrib/libs/apache/avro/api/Exception.hh index 4bcf63daa5..7c5410f96b 100644 --- a/contrib/libs/apache/avro/api/Exception.hh +++ b/contrib/libs/apache/avro/api/Exception.hh @@ -1,46 +1,46 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Exception_hh__ -#define avro_Exception_hh__ - -#include "Config.hh" -#include <stdexcept> -#include <boost/format.hpp> - -namespace avro { - -/// Wrapper for std::runtime_error that provides convenience constructor -/// for boost::format objects - -class AVRO_DECL Exception : public virtual std::runtime_error -{ - public: - - Exception(const std::string &msg) : - std::runtime_error(msg) - { } - - Exception(const boost::format &msg) : - std::runtime_error( boost::str(msg)) - { } -}; - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Exception_hh__ +#define avro_Exception_hh__ + +#include "Config.hh" +#include <stdexcept> +#include <boost/format.hpp> + +namespace avro { + +/// Wrapper for std::runtime_error that provides convenience constructor +/// for boost::format objects + +class AVRO_DECL Exception : public virtual std::runtime_error +{ + public: + + Exception(const std::string &msg) : + std::runtime_error(msg) + { } + + Exception(const boost::format &msg) : + std::runtime_error( boost::str(msg)) + { } +}; + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Generic.hh b/contrib/libs/apache/avro/api/Generic.hh index 12f8610f8d..e1b3a8290c 100644 --- a/contrib/libs/apache/avro/api/Generic.hh +++ b/contrib/libs/apache/avro/api/Generic.hh @@ -1,149 +1,149 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Generic_hh__ -#define avro_Generic_hh__ - -#include <boost/utility.hpp> - -#include "Config.hh" -#include "Types.hh" -#include "Encoder.hh" -#include "Decoder.hh" -#include "GenericDatum.hh" - -namespace avro { -/** - * A utility class to read generic datum from decoders. - */ -class AVRO_DECL GenericReader : boost::noncopyable { - const ValidSchema schema_; - const bool isResolving_; - const DecoderPtr decoder_; - - static void read(GenericDatum& datum, Decoder& d, bool isResolving); -public: - /** - * Constructs a reader for the given schema using the given decoder. - */ - GenericReader(const ValidSchema& s, const DecoderPtr& decoder); - - /** - * Constructs a reader for the given reader's schema \c readerSchema - * using the given - * decoder which holds data matching writer's schema \c writerSchema. - */ - GenericReader(const ValidSchema& writerSchema, - const ValidSchema& readerSchema, const DecoderPtr& decoder); - - /** - * Reads a value off the decoder. - */ - void read(GenericDatum& datum) const; - - /** - * Drains any residual bytes in the input stream (e.g. because - * reader's schema has no use of them) and return unused bytes - * back to the underlying input stream. - */ - void drain() { - decoder_->drain(); - } - /** - * Reads a generic datum from the stream, using the given schema. - */ - static void read(Decoder& d, GenericDatum& g); - - /** - * Reads a generic datum from the stream, using the given schema. - */ - static void read(Decoder& d, GenericDatum& g, const ValidSchema& s); -}; - - -/** - * A utility class to write generic datum to encoders. - */ -class AVRO_DECL GenericWriter : boost::noncopyable { - const ValidSchema schema_; - const EncoderPtr encoder_; - - static void write(const GenericDatum& datum, Encoder& e); -public: - /** - * Constructs a writer for the given schema using the given encoder. - */ - GenericWriter(const ValidSchema& s, const EncoderPtr& encoder); - - /** - * Writes a value onto the encoder. - */ - void write(const GenericDatum& datum) const; - - /** - * Writes a generic datum on to the stream. - */ - static void write(Encoder& e, const GenericDatum& g); - - /** - * Writes a generic datum on to the stream, using the given schema. - * Retained for backward compatibility. - */ - static void write(Encoder& e, const GenericDatum& g, const ValidSchema&) { - write(e, g); - } -}; - -template <typename T> struct codec_traits; - -/** - * Specialization of codec_traits for Generic datum along with its schema. - * This is maintained for compatibility with old code. Please use the - * cleaner codec_traits<GenericDatum> instead. - */ -template <> struct codec_traits<std::pair<ValidSchema, GenericDatum> > { - /** Encodes */ - static void encode(Encoder& e, - const std::pair<ValidSchema, GenericDatum>& p) { - GenericWriter::write(e, p.second, p.first); - } - - /** Decodes */ - static void decode(Decoder& d, std::pair<ValidSchema, GenericDatum>& p) { - GenericReader::read(d, p.second, p.first); - } -}; - -/** - * Specialization of codec_traits for GenericDatum. - */ -template <> struct codec_traits<GenericDatum> { - /** Encodes */ - static void encode(Encoder& e, const GenericDatum& g) { - GenericWriter::write(e, g); - } - - /** Decodes */ - static void decode(Decoder& d, GenericDatum& g) { - GenericReader::read(d, g); - } -}; - -} // namespace avro -#endif - +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Generic_hh__ +#define avro_Generic_hh__ + +#include <boost/utility.hpp> + +#include "Config.hh" +#include "Types.hh" +#include "Encoder.hh" +#include "Decoder.hh" +#include "GenericDatum.hh" + +namespace avro { +/** + * A utility class to read generic datum from decoders. + */ +class AVRO_DECL GenericReader : boost::noncopyable { + const ValidSchema schema_; + const bool isResolving_; + const DecoderPtr decoder_; + + static void read(GenericDatum& datum, Decoder& d, bool isResolving); +public: + /** + * Constructs a reader for the given schema using the given decoder. + */ + GenericReader(const ValidSchema& s, const DecoderPtr& decoder); + + /** + * Constructs a reader for the given reader's schema \c readerSchema + * using the given + * decoder which holds data matching writer's schema \c writerSchema. + */ + GenericReader(const ValidSchema& writerSchema, + const ValidSchema& readerSchema, const DecoderPtr& decoder); + + /** + * Reads a value off the decoder. + */ + void read(GenericDatum& datum) const; + + /** + * Drains any residual bytes in the input stream (e.g. because + * reader's schema has no use of them) and return unused bytes + * back to the underlying input stream. + */ + void drain() { + decoder_->drain(); + } + /** + * Reads a generic datum from the stream, using the given schema. + */ + static void read(Decoder& d, GenericDatum& g); + + /** + * Reads a generic datum from the stream, using the given schema. + */ + static void read(Decoder& d, GenericDatum& g, const ValidSchema& s); +}; + + +/** + * A utility class to write generic datum to encoders. + */ +class AVRO_DECL GenericWriter : boost::noncopyable { + const ValidSchema schema_; + const EncoderPtr encoder_; + + static void write(const GenericDatum& datum, Encoder& e); +public: + /** + * Constructs a writer for the given schema using the given encoder. + */ + GenericWriter(const ValidSchema& s, const EncoderPtr& encoder); + + /** + * Writes a value onto the encoder. + */ + void write(const GenericDatum& datum) const; + + /** + * Writes a generic datum on to the stream. + */ + static void write(Encoder& e, const GenericDatum& g); + + /** + * Writes a generic datum on to the stream, using the given schema. + * Retained for backward compatibility. + */ + static void write(Encoder& e, const GenericDatum& g, const ValidSchema&) { + write(e, g); + } +}; + +template <typename T> struct codec_traits; + +/** + * Specialization of codec_traits for Generic datum along with its schema. + * This is maintained for compatibility with old code. Please use the + * cleaner codec_traits<GenericDatum> instead. + */ +template <> struct codec_traits<std::pair<ValidSchema, GenericDatum> > { + /** Encodes */ + static void encode(Encoder& e, + const std::pair<ValidSchema, GenericDatum>& p) { + GenericWriter::write(e, p.second, p.first); + } + + /** Decodes */ + static void decode(Decoder& d, std::pair<ValidSchema, GenericDatum>& p) { + GenericReader::read(d, p.second, p.first); + } +}; + +/** + * Specialization of codec_traits for GenericDatum. + */ +template <> struct codec_traits<GenericDatum> { + /** Encodes */ + static void encode(Encoder& e, const GenericDatum& g) { + GenericWriter::write(e, g); + } + + /** Decodes */ + static void decode(Decoder& d, GenericDatum& g) { + GenericReader::read(d, g); + } +}; + +} // namespace avro +#endif + diff --git a/contrib/libs/apache/avro/api/GenericDatum.hh b/contrib/libs/apache/avro/api/GenericDatum.hh index 15cd53becb..ac0e5e5e20 100644 --- a/contrib/libs/apache/avro/api/GenericDatum.hh +++ b/contrib/libs/apache/avro/api/GenericDatum.hh @@ -1,576 +1,576 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_GenericDatum_hh__ -#define avro_GenericDatum_hh__ - -#include <stdint.h> -#include <vector> -#include <map> -#include <string> - -#if __cplusplus >= 201703L -#include <any> -#else -#include "boost/any.hpp" -#endif - -#include "LogicalType.hh" -#include "Node.hh" -#include "ValidSchema.hh" - -namespace avro { - -/** - * Generic datum which can hold any Avro type. The datum has a type - * and a value. The type is one of the Avro data types. The C++ type for - * value corresponds to the Avro type. - * \li An Avro <tt>null</tt> corresponds to no C++ type. It is illegal to - * to try to access values for <tt>null</tt>. - * \li Avro <tt>boolean</tt> maps to C++ <tt>bool</tt> - * \li Avro <tt>int</tt> maps to C++ <tt>int32_t</tt>. - * \li Avro <tt>long</tt> maps to C++ <tt>int64_t</tt>. - * \li Avro <tt>float</tt> maps to C++ <tt>float</tt>. - * \li Avro <tt>double</tt> maps to C++ <tt>double</tt>. - * \li Avro <tt>string</tt> maps to C++ <tt>std::string</tt>. - * \li Avro <tt>bytes</tt> maps to C++ <tt>std::vector<uint_t></tt>. - * \li Avro <tt>fixed</tt> maps to C++ class <tt>GenericFixed</tt>. - * \li Avro <tt>enum</tt> maps to C++ class <tt>GenericEnum</tt>. - * \li Avro <tt>array</tt> maps to C++ class <tt>GenericArray</tt>. - * \li Avro <tt>map</tt> maps to C++ class <tt>GenericMap</tt>. - * \li There is no C++ type corresponding to Avro <tt>union</tt>. The - * object should have the C++ type corresponing to one of the constituent - * types of the union. - * - */ -class AVRO_DECL GenericDatum { -protected: - Type type_; - LogicalType logicalType_; -#if __cplusplus >= 201703L - std::any value_; -#else - boost::any value_; -#endif - - GenericDatum(Type t) - : type_(t), logicalType_(LogicalType::NONE) { } - - GenericDatum(Type t, LogicalType logicalType) - : type_(t), logicalType_(logicalType) { } - - template <typename T> - GenericDatum(Type t, LogicalType logicalType, const T& v) - : type_(t), logicalType_(logicalType), value_(v) { } - - void init(const NodePtr& schema); -public: - /** - * The avro data type this datum holds. - */ - Type type() const; - - /** - * The avro logical type that augments the main data type this datum holds. - */ - LogicalType logicalType() const; - - /** - * Returns the value held by this datum. - * T The type for the value. This must correspond to the - * avro type returned by type(). - */ - template<typename T> const T& value() const; - - /** - * Returns the reference to the value held by this datum, which - * can be used to change the contents. Please note that only - * value can be changed, the data type of the value held cannot - * be changed. - * - * T The type for the value. This must correspond to the - * avro type returned by type(). - */ - template<typename T> T& value(); - - /** - * Returns true if and only if this datum is a union. - */ - bool isUnion() const { return type_ == AVRO_UNION; } - - /** - * Returns the index of the current branch, if this is a union. - * \sa isUnion(). - */ - size_t unionBranch() const; - - /** - * Selects a new branch in the union if this is a union. - * \sa isUnion(). - */ - void selectBranch(size_t branch); - - /// Makes a new AVRO_NULL datum. - GenericDatum() : type_(AVRO_NULL), logicalType_(LogicalType::NONE) { } - - /// Makes a new AVRO_BOOL datum whose value is of type bool. - GenericDatum(bool v) - : type_(AVRO_BOOL), logicalType_(LogicalType::NONE), value_(v) { } - - /// Makes a new AVRO_INT datum whose value is of type int32_t. - GenericDatum(int32_t v) - : type_(AVRO_INT), logicalType_(LogicalType::NONE), value_(v) { } - - /// Makes a new AVRO_LONG datum whose value is of type int64_t. - GenericDatum(int64_t v) - : type_(AVRO_LONG), logicalType_(LogicalType::NONE), value_(v) { } - - /// Makes a new AVRO_FLOAT datum whose value is of type float. - GenericDatum(float v) - : type_(AVRO_FLOAT), logicalType_(LogicalType::NONE), value_(v) { } - - /// Makes a new AVRO_DOUBLE datum whose value is of type double. - GenericDatum(double v) - : type_(AVRO_DOUBLE), logicalType_(LogicalType::NONE), value_(v) { } - - /// Makes a new AVRO_STRING datum whose value is of type std::string. - GenericDatum(const std::string& v) - : type_(AVRO_STRING), logicalType_(LogicalType::NONE), value_(v) { } - - /// Makes a new AVRO_BYTES datum whose value is of type - /// std::vector<uint8_t>. - GenericDatum(const std::vector<uint8_t>& v) : - type_(AVRO_BYTES), logicalType_(LogicalType::NONE), value_(v) { } - - /** - * Constructs a datum corresponding to the given avro type. - * The value will the appropriate default corresponding to the - * data type. - * \param schema The schema that defines the avro type. - */ - GenericDatum(const NodePtr& schema); - - /** - * Constructs a datum corresponding to the given avro type and set - * the value. - * \param schema The schema that defines the avro type. - * \param v The value for this type. - */ - template<typename T> - GenericDatum(const NodePtr& schema, const T& v) : - type_(schema->type()), logicalType_(schema->logicalType()) { - init(schema); -#if __cplusplus >= 201703L - *std::any_cast<T>(&value_) = v; -#else - *boost::any_cast<T>(&value_) = v; -#endif - } - - /** - * Constructs a datum corresponding to the given avro type. - * The value will the appropriate default corresponding to the - * data type. - * \param schema The schema that defines the avro type. - */ - GenericDatum(const ValidSchema& schema); -}; - -/** - * The base class for all generic type for containers. - */ -class AVRO_DECL GenericContainer { - NodePtr schema_; - static void assertType(const NodePtr& schema, Type type); -protected: - /** - * Constructs a container corresponding to the given schema. - */ - GenericContainer(Type type, const NodePtr& s) : schema_(s) { - assertType(s, type); - } - -public: - /// Returns the schema for this object - const NodePtr& schema() const { - return schema_; - } -}; - -/** - * Generic container for unions. - */ -class AVRO_DECL GenericUnion : public GenericContainer { - size_t curBranch_; - GenericDatum datum_; - -public: - /** - * Constructs a generic union corresponding to the given schema \p schema, - * and the given value. The schema should be of Avro type union - * and the value should correspond to one of the branches of the union. - */ - GenericUnion(const NodePtr& schema) : - GenericContainer(AVRO_UNION, schema), curBranch_(schema->leaves()) { - selectBranch(0); - } - - /** - * Returns the index of the current branch. - */ - size_t currentBranch() const { return curBranch_; } - - /** - * Selects a new branch. The type for the value is changed accordingly. - * \param branch The index for the selected branch. - */ - void selectBranch(size_t branch) { - if (curBranch_ != branch) { - datum_ = GenericDatum(schema()->leafAt(branch)); - curBranch_ = branch; - } - } - - /** - * Returns the datum corresponding to the currently selected branch - * in this union. - */ - GenericDatum& datum() { - return datum_; - } - - /** - * Returns the datum corresponding to the currently selected branch - * in this union. - */ - const GenericDatum& datum() const { - return datum_; - } -}; - -/** - * The generic container for Avro records. - */ -class AVRO_DECL GenericRecord : public GenericContainer { - std::vector<GenericDatum> fields_; -public: - /** - * Constructs a generic record corresponding to the given schema \p schema, - * which should be of Avro type record. - */ - GenericRecord(const NodePtr& schema); - - /** - * Returns the number of fields in the current record. - */ - size_t fieldCount() const { - return fields_.size(); - } - - /** - * Returns index of the field with the given name \p name - */ - size_t fieldIndex(const std::string& name) const { - size_t index = 0; - if (!schema()->nameIndex(name, index)) { - throw Exception("Invalid field name: " + name); - } - return index; - } - - /** - * Returns true if a field with the given name \p name is located in this r - * false otherwise - */ - bool hasField(const std::string& name) const { - size_t index = 0; - return schema()->nameIndex(name, index); - } - - /** - * Returns the field with the given name \p name. - */ - const GenericDatum& field(const std::string& name) const { - return fieldAt(fieldIndex(name)); - } - - /** - * Returns the reference to the field with the given name \p name, - * which can be used to change the contents. - */ - GenericDatum& field(const std::string& name) { - return fieldAt(fieldIndex(name)); - } - - /** - * Returns the field at the given position \p pos. - */ - const GenericDatum& fieldAt(size_t pos) const { - return fields_[pos]; - } - - /** - * Returns the reference to the field at the given position \p pos, - * which can be used to change the contents. - */ - GenericDatum& fieldAt(size_t pos) { - return fields_[pos]; - } - - /** - * Replaces the field at the given position \p pos with \p v. - */ - void setFieldAt(size_t pos, const GenericDatum& v) { - // assertSameType(v, schema()->leafAt(pos)); - fields_[pos] = v; - } -}; - -/** - * The generic container for Avro arrays. - */ -class AVRO_DECL GenericArray : public GenericContainer { -public: - /** - * The contents type for the array. - */ - typedef std::vector<GenericDatum> Value; - - /** - * Constructs a generic array corresponding to the given schema \p schema, - * which should be of Avro type array. - */ - GenericArray(const NodePtr& schema) : GenericContainer(AVRO_ARRAY, schema) { - } - - /** - * Returns the contents of this array. - */ - const Value& value() const { - return value_; - } - - /** - * Returns the reference to the contents of this array. - */ - Value& value() { - return value_; - } -private: - Value value_; -}; - -/** - * The generic container for Avro maps. - */ -class AVRO_DECL GenericMap : public GenericContainer { -public: - /** - * The contents type for the map. - */ - typedef std::vector<std::pair<std::string, GenericDatum> > Value; - - /** - * Constructs a generic map corresponding to the given schema \p schema, - * which should be of Avro type map. - */ - GenericMap(const NodePtr& schema) : GenericContainer(AVRO_MAP, schema) { - } - - /** - * Returns the contents of this map. - */ - const Value& value() const { - return value_; - } - - /** - * Returns the reference to the contents of this map. - */ - Value& value() { - return value_; - } -private: - Value value_; -}; - -/** - * Generic container for Avro enum. - */ -class AVRO_DECL GenericEnum : public GenericContainer { - size_t value_; - - static size_t index(const NodePtr& schema, const std::string& symbol) { - size_t result; - if (schema->nameIndex(symbol, result)) { - return result; - } - throw Exception("No such symbol"); - } - -public: - /** - * Constructs a generic enum corresponding to the given schema \p schema, - * which should be of Avro type enum. - */ - GenericEnum(const NodePtr& schema) : - GenericContainer(AVRO_ENUM, schema), value_(0) { - } - - GenericEnum(const NodePtr& schema, const std::string& symbol) : - GenericContainer(AVRO_ENUM, schema), value_(index(schema, symbol)) { - } - - /** - * Returns the symbol corresponding to the cardinal \p n. If the - * value for \p n is not within the limits an exception is thrown. - */ - const std::string& symbol(size_t n) { - if (n < schema()->names()) { - return schema()->nameAt(n); - } - throw Exception("Not as many symbols"); - } - - /** - * Returns the cardinal for the given symbol \c symbol. If the symbol - * is not defined for this enum and exception is thrown. - */ - size_t index(const std::string& symbol) const { - return index(schema(), symbol); - } - - /** - * Set the value for this enum corresponding to the given symbol \c symbol. - */ - size_t set(const std::string& symbol) { - return value_ = index(symbol); - } - - /** - * Set the value for this enum corresponding to the given cardinal \c n. - */ - void set(size_t n) { - if (n < schema()->names()) { - value_ = n; - return; - } - throw Exception("Not as many symbols"); - } - - /** - * Returns the cardinal for the current value of this enum. - */ - size_t value() const { - return value_; - } - - /** - * Returns the symbol for the current value of this enum. - */ - const std::string& symbol() const { - return schema()->nameAt(value_); - } -}; - -/** - * Generic container for Avro fixed. - */ -class AVRO_DECL GenericFixed : public GenericContainer { - std::vector<uint8_t> value_; -public: - /** - * Constructs a generic enum corresponding to the given schema \p schema, - * which should be of Avro type fixed. - */ - GenericFixed(const NodePtr& schema) : GenericContainer(AVRO_FIXED, schema) { - value_.resize(schema->fixedSize()); - } - - GenericFixed(const NodePtr& schema, const std::vector<uint8_t>& v) : - GenericContainer(AVRO_FIXED, schema), value_(v) { } - - /** - * Returns the contents of this fixed. - */ - const std::vector<uint8_t>& value() const { - return value_; - } - - /** - * Returns the reference to the contents of this fixed. - */ - std::vector<uint8_t>& value() { - return value_; - } -}; - -inline Type GenericDatum::type() const { - return (type_ == AVRO_UNION) ? -#if __cplusplus >= 201703L - std::any_cast<GenericUnion>(&value_)->datum().type() : -#else - boost::any_cast<GenericUnion>(&value_)->datum().type() : -#endif - type_; -} - -inline LogicalType GenericDatum::logicalType() const { - return logicalType_; -} - -template<typename T> T& GenericDatum::value() { - return (type_ == AVRO_UNION) ? -#if __cplusplus >= 201703L - std::any_cast<GenericUnion>(&value_)->datum().value<T>() : - *std::any_cast<T>(&value_); -#else - boost::any_cast<GenericUnion>(&value_)->datum().value<T>() : - *boost::any_cast<T>(&value_); -#endif -} - -template<typename T> const T& GenericDatum::value() const { - return (type_ == AVRO_UNION) ? -#if __cplusplus >= 201703L - std::any_cast<GenericUnion>(&value_)->datum().value<T>() : - *std::any_cast<T>(&value_); -#else - boost::any_cast<GenericUnion>(&value_)->datum().value<T>() : - *boost::any_cast<T>(&value_); -#endif -} - -inline size_t GenericDatum::unionBranch() const { -#if __cplusplus >= 201703L - return std::any_cast<GenericUnion>(&value_)->currentBranch(); -#else - return boost::any_cast<GenericUnion>(&value_)->currentBranch(); -#endif -} - -inline void GenericDatum::selectBranch(size_t branch) { -#if __cplusplus >= 201703L - std::any_cast<GenericUnion>(&value_)->selectBranch(branch); -#else - boost::any_cast<GenericUnion>(&value_)->selectBranch(branch); -#endif -} - -} // namespace avro -#endif // avro_GenericDatum_hh__ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_GenericDatum_hh__ +#define avro_GenericDatum_hh__ + +#include <stdint.h> +#include <vector> +#include <map> +#include <string> + +#if __cplusplus >= 201703L +#include <any> +#else +#include "boost/any.hpp" +#endif + +#include "LogicalType.hh" +#include "Node.hh" +#include "ValidSchema.hh" + +namespace avro { + +/** + * Generic datum which can hold any Avro type. The datum has a type + * and a value. The type is one of the Avro data types. The C++ type for + * value corresponds to the Avro type. + * \li An Avro <tt>null</tt> corresponds to no C++ type. It is illegal to + * to try to access values for <tt>null</tt>. + * \li Avro <tt>boolean</tt> maps to C++ <tt>bool</tt> + * \li Avro <tt>int</tt> maps to C++ <tt>int32_t</tt>. + * \li Avro <tt>long</tt> maps to C++ <tt>int64_t</tt>. + * \li Avro <tt>float</tt> maps to C++ <tt>float</tt>. + * \li Avro <tt>double</tt> maps to C++ <tt>double</tt>. + * \li Avro <tt>string</tt> maps to C++ <tt>std::string</tt>. + * \li Avro <tt>bytes</tt> maps to C++ <tt>std::vector<uint_t></tt>. + * \li Avro <tt>fixed</tt> maps to C++ class <tt>GenericFixed</tt>. + * \li Avro <tt>enum</tt> maps to C++ class <tt>GenericEnum</tt>. + * \li Avro <tt>array</tt> maps to C++ class <tt>GenericArray</tt>. + * \li Avro <tt>map</tt> maps to C++ class <tt>GenericMap</tt>. + * \li There is no C++ type corresponding to Avro <tt>union</tt>. The + * object should have the C++ type corresponing to one of the constituent + * types of the union. + * + */ +class AVRO_DECL GenericDatum { +protected: + Type type_; + LogicalType logicalType_; +#if __cplusplus >= 201703L + std::any value_; +#else + boost::any value_; +#endif + + GenericDatum(Type t) + : type_(t), logicalType_(LogicalType::NONE) { } + + GenericDatum(Type t, LogicalType logicalType) + : type_(t), logicalType_(logicalType) { } + + template <typename T> + GenericDatum(Type t, LogicalType logicalType, const T& v) + : type_(t), logicalType_(logicalType), value_(v) { } + + void init(const NodePtr& schema); +public: + /** + * The avro data type this datum holds. + */ + Type type() const; + + /** + * The avro logical type that augments the main data type this datum holds. + */ + LogicalType logicalType() const; + + /** + * Returns the value held by this datum. + * T The type for the value. This must correspond to the + * avro type returned by type(). + */ + template<typename T> const T& value() const; + + /** + * Returns the reference to the value held by this datum, which + * can be used to change the contents. Please note that only + * value can be changed, the data type of the value held cannot + * be changed. + * + * T The type for the value. This must correspond to the + * avro type returned by type(). + */ + template<typename T> T& value(); + + /** + * Returns true if and only if this datum is a union. + */ + bool isUnion() const { return type_ == AVRO_UNION; } + + /** + * Returns the index of the current branch, if this is a union. + * \sa isUnion(). + */ + size_t unionBranch() const; + + /** + * Selects a new branch in the union if this is a union. + * \sa isUnion(). + */ + void selectBranch(size_t branch); + + /// Makes a new AVRO_NULL datum. + GenericDatum() : type_(AVRO_NULL), logicalType_(LogicalType::NONE) { } + + /// Makes a new AVRO_BOOL datum whose value is of type bool. + GenericDatum(bool v) + : type_(AVRO_BOOL), logicalType_(LogicalType::NONE), value_(v) { } + + /// Makes a new AVRO_INT datum whose value is of type int32_t. + GenericDatum(int32_t v) + : type_(AVRO_INT), logicalType_(LogicalType::NONE), value_(v) { } + + /// Makes a new AVRO_LONG datum whose value is of type int64_t. + GenericDatum(int64_t v) + : type_(AVRO_LONG), logicalType_(LogicalType::NONE), value_(v) { } + + /// Makes a new AVRO_FLOAT datum whose value is of type float. + GenericDatum(float v) + : type_(AVRO_FLOAT), logicalType_(LogicalType::NONE), value_(v) { } + + /// Makes a new AVRO_DOUBLE datum whose value is of type double. + GenericDatum(double v) + : type_(AVRO_DOUBLE), logicalType_(LogicalType::NONE), value_(v) { } + + /// Makes a new AVRO_STRING datum whose value is of type std::string. + GenericDatum(const std::string& v) + : type_(AVRO_STRING), logicalType_(LogicalType::NONE), value_(v) { } + + /// Makes a new AVRO_BYTES datum whose value is of type + /// std::vector<uint8_t>. + GenericDatum(const std::vector<uint8_t>& v) : + type_(AVRO_BYTES), logicalType_(LogicalType::NONE), value_(v) { } + + /** + * Constructs a datum corresponding to the given avro type. + * The value will the appropriate default corresponding to the + * data type. + * \param schema The schema that defines the avro type. + */ + GenericDatum(const NodePtr& schema); + + /** + * Constructs a datum corresponding to the given avro type and set + * the value. + * \param schema The schema that defines the avro type. + * \param v The value for this type. + */ + template<typename T> + GenericDatum(const NodePtr& schema, const T& v) : + type_(schema->type()), logicalType_(schema->logicalType()) { + init(schema); +#if __cplusplus >= 201703L + *std::any_cast<T>(&value_) = v; +#else + *boost::any_cast<T>(&value_) = v; +#endif + } + + /** + * Constructs a datum corresponding to the given avro type. + * The value will the appropriate default corresponding to the + * data type. + * \param schema The schema that defines the avro type. + */ + GenericDatum(const ValidSchema& schema); +}; + +/** + * The base class for all generic type for containers. + */ +class AVRO_DECL GenericContainer { + NodePtr schema_; + static void assertType(const NodePtr& schema, Type type); +protected: + /** + * Constructs a container corresponding to the given schema. + */ + GenericContainer(Type type, const NodePtr& s) : schema_(s) { + assertType(s, type); + } + +public: + /// Returns the schema for this object + const NodePtr& schema() const { + return schema_; + } +}; + +/** + * Generic container for unions. + */ +class AVRO_DECL GenericUnion : public GenericContainer { + size_t curBranch_; + GenericDatum datum_; + +public: + /** + * Constructs a generic union corresponding to the given schema \p schema, + * and the given value. The schema should be of Avro type union + * and the value should correspond to one of the branches of the union. + */ + GenericUnion(const NodePtr& schema) : + GenericContainer(AVRO_UNION, schema), curBranch_(schema->leaves()) { + selectBranch(0); + } + + /** + * Returns the index of the current branch. + */ + size_t currentBranch() const { return curBranch_; } + + /** + * Selects a new branch. The type for the value is changed accordingly. + * \param branch The index for the selected branch. + */ + void selectBranch(size_t branch) { + if (curBranch_ != branch) { + datum_ = GenericDatum(schema()->leafAt(branch)); + curBranch_ = branch; + } + } + + /** + * Returns the datum corresponding to the currently selected branch + * in this union. + */ + GenericDatum& datum() { + return datum_; + } + + /** + * Returns the datum corresponding to the currently selected branch + * in this union. + */ + const GenericDatum& datum() const { + return datum_; + } +}; + +/** + * The generic container for Avro records. + */ +class AVRO_DECL GenericRecord : public GenericContainer { + std::vector<GenericDatum> fields_; +public: + /** + * Constructs a generic record corresponding to the given schema \p schema, + * which should be of Avro type record. + */ + GenericRecord(const NodePtr& schema); + + /** + * Returns the number of fields in the current record. + */ + size_t fieldCount() const { + return fields_.size(); + } + + /** + * Returns index of the field with the given name \p name + */ + size_t fieldIndex(const std::string& name) const { + size_t index = 0; + if (!schema()->nameIndex(name, index)) { + throw Exception("Invalid field name: " + name); + } + return index; + } + + /** + * Returns true if a field with the given name \p name is located in this r + * false otherwise + */ + bool hasField(const std::string& name) const { + size_t index = 0; + return schema()->nameIndex(name, index); + } + + /** + * Returns the field with the given name \p name. + */ + const GenericDatum& field(const std::string& name) const { + return fieldAt(fieldIndex(name)); + } + + /** + * Returns the reference to the field with the given name \p name, + * which can be used to change the contents. + */ + GenericDatum& field(const std::string& name) { + return fieldAt(fieldIndex(name)); + } + + /** + * Returns the field at the given position \p pos. + */ + const GenericDatum& fieldAt(size_t pos) const { + return fields_[pos]; + } + + /** + * Returns the reference to the field at the given position \p pos, + * which can be used to change the contents. + */ + GenericDatum& fieldAt(size_t pos) { + return fields_[pos]; + } + + /** + * Replaces the field at the given position \p pos with \p v. + */ + void setFieldAt(size_t pos, const GenericDatum& v) { + // assertSameType(v, schema()->leafAt(pos)); + fields_[pos] = v; + } +}; + +/** + * The generic container for Avro arrays. + */ +class AVRO_DECL GenericArray : public GenericContainer { +public: + /** + * The contents type for the array. + */ + typedef std::vector<GenericDatum> Value; + + /** + * Constructs a generic array corresponding to the given schema \p schema, + * which should be of Avro type array. + */ + GenericArray(const NodePtr& schema) : GenericContainer(AVRO_ARRAY, schema) { + } + + /** + * Returns the contents of this array. + */ + const Value& value() const { + return value_; + } + + /** + * Returns the reference to the contents of this array. + */ + Value& value() { + return value_; + } +private: + Value value_; +}; + +/** + * The generic container for Avro maps. + */ +class AVRO_DECL GenericMap : public GenericContainer { +public: + /** + * The contents type for the map. + */ + typedef std::vector<std::pair<std::string, GenericDatum> > Value; + + /** + * Constructs a generic map corresponding to the given schema \p schema, + * which should be of Avro type map. + */ + GenericMap(const NodePtr& schema) : GenericContainer(AVRO_MAP, schema) { + } + + /** + * Returns the contents of this map. + */ + const Value& value() const { + return value_; + } + + /** + * Returns the reference to the contents of this map. + */ + Value& value() { + return value_; + } +private: + Value value_; +}; + +/** + * Generic container for Avro enum. + */ +class AVRO_DECL GenericEnum : public GenericContainer { + size_t value_; + + static size_t index(const NodePtr& schema, const std::string& symbol) { + size_t result; + if (schema->nameIndex(symbol, result)) { + return result; + } + throw Exception("No such symbol"); + } + +public: + /** + * Constructs a generic enum corresponding to the given schema \p schema, + * which should be of Avro type enum. + */ + GenericEnum(const NodePtr& schema) : + GenericContainer(AVRO_ENUM, schema), value_(0) { + } + + GenericEnum(const NodePtr& schema, const std::string& symbol) : + GenericContainer(AVRO_ENUM, schema), value_(index(schema, symbol)) { + } + + /** + * Returns the symbol corresponding to the cardinal \p n. If the + * value for \p n is not within the limits an exception is thrown. + */ + const std::string& symbol(size_t n) { + if (n < schema()->names()) { + return schema()->nameAt(n); + } + throw Exception("Not as many symbols"); + } + + /** + * Returns the cardinal for the given symbol \c symbol. If the symbol + * is not defined for this enum and exception is thrown. + */ + size_t index(const std::string& symbol) const { + return index(schema(), symbol); + } + + /** + * Set the value for this enum corresponding to the given symbol \c symbol. + */ + size_t set(const std::string& symbol) { + return value_ = index(symbol); + } + + /** + * Set the value for this enum corresponding to the given cardinal \c n. + */ + void set(size_t n) { + if (n < schema()->names()) { + value_ = n; + return; + } + throw Exception("Not as many symbols"); + } + + /** + * Returns the cardinal for the current value of this enum. + */ + size_t value() const { + return value_; + } + + /** + * Returns the symbol for the current value of this enum. + */ + const std::string& symbol() const { + return schema()->nameAt(value_); + } +}; + +/** + * Generic container for Avro fixed. + */ +class AVRO_DECL GenericFixed : public GenericContainer { + std::vector<uint8_t> value_; +public: + /** + * Constructs a generic enum corresponding to the given schema \p schema, + * which should be of Avro type fixed. + */ + GenericFixed(const NodePtr& schema) : GenericContainer(AVRO_FIXED, schema) { + value_.resize(schema->fixedSize()); + } + + GenericFixed(const NodePtr& schema, const std::vector<uint8_t>& v) : + GenericContainer(AVRO_FIXED, schema), value_(v) { } + + /** + * Returns the contents of this fixed. + */ + const std::vector<uint8_t>& value() const { + return value_; + } + + /** + * Returns the reference to the contents of this fixed. + */ + std::vector<uint8_t>& value() { + return value_; + } +}; + +inline Type GenericDatum::type() const { + return (type_ == AVRO_UNION) ? +#if __cplusplus >= 201703L + std::any_cast<GenericUnion>(&value_)->datum().type() : +#else + boost::any_cast<GenericUnion>(&value_)->datum().type() : +#endif + type_; +} + +inline LogicalType GenericDatum::logicalType() const { + return logicalType_; +} + +template<typename T> T& GenericDatum::value() { + return (type_ == AVRO_UNION) ? +#if __cplusplus >= 201703L + std::any_cast<GenericUnion>(&value_)->datum().value<T>() : + *std::any_cast<T>(&value_); +#else + boost::any_cast<GenericUnion>(&value_)->datum().value<T>() : + *boost::any_cast<T>(&value_); +#endif +} + +template<typename T> const T& GenericDatum::value() const { + return (type_ == AVRO_UNION) ? +#if __cplusplus >= 201703L + std::any_cast<GenericUnion>(&value_)->datum().value<T>() : + *std::any_cast<T>(&value_); +#else + boost::any_cast<GenericUnion>(&value_)->datum().value<T>() : + *boost::any_cast<T>(&value_); +#endif +} + +inline size_t GenericDatum::unionBranch() const { +#if __cplusplus >= 201703L + return std::any_cast<GenericUnion>(&value_)->currentBranch(); +#else + return boost::any_cast<GenericUnion>(&value_)->currentBranch(); +#endif +} + +inline void GenericDatum::selectBranch(size_t branch) { +#if __cplusplus >= 201703L + std::any_cast<GenericUnion>(&value_)->selectBranch(branch); +#else + boost::any_cast<GenericUnion>(&value_)->selectBranch(branch); +#endif +} + +} // namespace avro +#endif // avro_GenericDatum_hh__ diff --git a/contrib/libs/apache/avro/api/Layout.hh b/contrib/libs/apache/avro/api/Layout.hh index 693bed1b42..ffd810c8b0 100644 --- a/contrib/libs/apache/avro/api/Layout.hh +++ b/contrib/libs/apache/avro/api/Layout.hh @@ -1,83 +1,83 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Layout_hh__ -#define avro_Layout_hh__ - -#include <boost/noncopyable.hpp> -#include "Config.hh" - -/// \file Layout.hh -/// - -namespace avro { - -class AVRO_DECL Layout : private boost::noncopyable { - - protected: - - Layout(size_t offset = 0) : - offset_(offset) - {} - - public: - - size_t offset() const { - return offset_; - } - - virtual ~Layout() {} - - private: - - const size_t offset_; -}; - -class AVRO_DECL PrimitiveLayout : public Layout { - - public: - - PrimitiveLayout(size_t offset = 0) : - Layout(offset) - {} -}; - -class AVRO_DECL CompoundLayout : public Layout { - - public: - - CompoundLayout(size_t offset = 0) : - Layout(offset) - {} - - void add(std::unique_ptr<Layout> &layout) { - layouts_.push_back(std::move(layout)); - } - - const Layout &at (size_t idx) const { - return *layouts_.at(idx); - } - - private: - - std::vector<std::unique_ptr<Layout> > layouts_; -}; - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Layout_hh__ +#define avro_Layout_hh__ + +#include <boost/noncopyable.hpp> +#include "Config.hh" + +/// \file Layout.hh +/// + +namespace avro { + +class AVRO_DECL Layout : private boost::noncopyable { + + protected: + + Layout(size_t offset = 0) : + offset_(offset) + {} + + public: + + size_t offset() const { + return offset_; + } + + virtual ~Layout() {} + + private: + + const size_t offset_; +}; + +class AVRO_DECL PrimitiveLayout : public Layout { + + public: + + PrimitiveLayout(size_t offset = 0) : + Layout(offset) + {} +}; + +class AVRO_DECL CompoundLayout : public Layout { + + public: + + CompoundLayout(size_t offset = 0) : + Layout(offset) + {} + + void add(std::unique_ptr<Layout> &layout) { + layouts_.push_back(std::move(layout)); + } + + const Layout &at (size_t idx) const { + return *layouts_.at(idx); + } + + private: + + std::vector<std::unique_ptr<Layout> > layouts_; +}; + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/LogicalType.hh b/contrib/libs/apache/avro/api/LogicalType.hh index ba9d86db98..33972788fb 100644 --- a/contrib/libs/apache/avro/api/LogicalType.hh +++ b/contrib/libs/apache/avro/api/LogicalType.hh @@ -1,65 +1,65 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_LogicalType_hh__ -#define avro_LogicalType_hh__ - -#include <iostream> - -#include "Config.hh" - -namespace avro { - -class AVRO_DECL LogicalType { - public: - enum Type { - NONE, - DECIMAL, - DATE, - TIME_MILLIS, - TIME_MICROS, - TIMESTAMP_MILLIS, - TIMESTAMP_MICROS, - DURATION, - UUID - }; - - explicit LogicalType(Type type); - - Type type() const; - - // Precision and scale can only be set for the DECIMAL logical type. - // Precision must be positive and scale must be either positive or zero. The - // setters will throw an exception if they are called on any type other - // than DECIMAL. - void setPrecision(int precision); - int precision() const { return precision_; } - void setScale(int scale); - int scale() const { return scale_; } - - void printJson(std::ostream& os) const; - - private: - Type type_; - int precision_; - int scale_; -}; - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_LogicalType_hh__ +#define avro_LogicalType_hh__ + +#include <iostream> + +#include "Config.hh" + +namespace avro { + +class AVRO_DECL LogicalType { + public: + enum Type { + NONE, + DECIMAL, + DATE, + TIME_MILLIS, + TIME_MICROS, + TIMESTAMP_MILLIS, + TIMESTAMP_MICROS, + DURATION, + UUID + }; + + explicit LogicalType(Type type); + + Type type() const; + + // Precision and scale can only be set for the DECIMAL logical type. + // Precision must be positive and scale must be either positive or zero. The + // setters will throw an exception if they are called on any type other + // than DECIMAL. + void setPrecision(int precision); + int precision() const { return precision_; } + void setScale(int scale); + int scale() const { return scale_; } + + void printJson(std::ostream& os) const; + + private: + Type type_; + int precision_; + int scale_; +}; + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Node.hh b/contrib/libs/apache/avro/api/Node.hh index b8bbf1d573..205b06f00e 100644 --- a/contrib/libs/apache/avro/api/Node.hh +++ b/contrib/libs/apache/avro/api/Node.hh @@ -1,211 +1,211 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Node_hh__ -#define avro_Node_hh__ - -#include "Config.hh" - -#include <cassert> -#include <memory> -#include <boost/noncopyable.hpp> - -#include "Exception.hh" -#include "LogicalType.hh" -#include "Types.hh" -#include "SchemaResolution.hh" - -namespace avro { - -class Node; -class GenericDatum; - -typedef std::shared_ptr<Node> NodePtr; - -class AVRO_DECL Name { - std::string ns_; - std::string simpleName_; -public: - Name() { } - Name(const std::string& fullname); - Name(const std::string& simpleName, const std::string& ns) : ns_(ns), simpleName_(simpleName) { check(); } - - const std::string fullname() const; - const std::string& ns() const { return ns_; } - const std::string& simpleName() const { return simpleName_; } - - void ns(const std::string& n) { ns_ = n; } - void simpleName(const std::string& n) { simpleName_ = n; } - void fullname(const std::string& n); - - bool operator < (const Name& n) const; - void check() const; - bool operator == (const Name& n) const; - bool operator != (const Name& n) const { return !((*this) == n); } - void clear() { - ns_.clear(); - simpleName_.clear(); - } - operator std::string() const { - return fullname(); - } -}; - -inline -std::ostream& operator << (std::ostream& os, const Name& n) { - return os << n.fullname(); -} - -/// Node is the building block for parse trees. Each node represents an avro -/// type. Compound types have leaf nodes that represent the types they are -/// composed of. -/// -/// The user does not use the Node object directly, they interface with Schema -/// objects. -/// -/// The Node object uses reference-counted pointers. This is so that schemas -/// may be reused in other schemas, without needing to worry about memory -/// deallocation for nodes that are added to multiple schema parse trees. -/// -/// Node has minimal implementation, serving as an abstract base class for -/// different node types. -/// - -class AVRO_DECL Node : private boost::noncopyable -{ - public: - - Node(Type type) : - type_(type), - logicalType_(LogicalType::NONE), - locked_(false) - {} - - virtual ~Node(); - - Type type() const { - return type_; - } - - LogicalType logicalType() const { - return logicalType_; - } - - void setLogicalType(LogicalType logicalType); - - void lock() { - locked_ = true; - } - - bool locked() const { - return locked_; - } - - virtual bool hasName() const = 0; - - void setName(const Name &name) { - checkLock(); - checkName(name); - doSetName(name); - } - virtual const Name &name() const = 0; - - virtual const std::string &getDoc() const = 0; - void setDoc(const std::string &doc) { - checkLock(); - doSetDoc(doc); - } - - void addLeaf(const NodePtr &newLeaf) { - checkLock(); - doAddLeaf(newLeaf); - } - virtual size_t leaves() const = 0; - virtual const NodePtr& leafAt(int index) const = 0; - virtual const GenericDatum& defaultValueAt(int index) { - throw Exception(boost::format("No default value at: %1%") % index); - } - - void addName(const std::string &name) { - checkLock(); - checkName(name); - doAddName(name); - } - virtual size_t names() const = 0; - virtual const std::string &nameAt(int index) const = 0; - virtual bool nameIndex(const std::string &name, size_t &index) const = 0; - - void setFixedSize(int size) { - checkLock(); - doSetFixedSize(size); - } - virtual int fixedSize() const = 0; - - virtual bool isValid() const = 0; - - virtual SchemaResolution resolve(const Node &reader) const = 0; - - virtual void printJson(std::ostream &os, int depth) const = 0; - - virtual void printBasicInfo(std::ostream &os) const = 0; - - virtual void setLeafToSymbolic(int index, const NodePtr &node) = 0; - - // Serialize the default value GenericDatum g for the node contained - // in a record node. - virtual void printDefaultToJson(const GenericDatum& g, std::ostream &os, - int depth) const = 0; - - protected: - - void checkLock() const { - if(locked()) { - throw Exception("Cannot modify locked schema"); - } - } - - virtual void checkName(const Name &name) const { - name.check(); - } - - virtual void doSetName(const Name &name) = 0; - virtual void doSetDoc(const std::string &name) = 0; - - virtual void doAddLeaf(const NodePtr &newLeaf) = 0; - virtual void doAddName(const std::string &name) = 0; - virtual void doSetFixedSize(int size) = 0; - - private: - - const Type type_; - LogicalType logicalType_; - bool locked_; -}; - -} // namespace avro - -namespace std { -inline std::ostream& operator<<(std::ostream& os, const avro::Node& n) -{ - n.printJson(os, 0); - return os; -} -} - - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Node_hh__ +#define avro_Node_hh__ + +#include "Config.hh" + +#include <cassert> +#include <memory> +#include <boost/noncopyable.hpp> + +#include "Exception.hh" +#include "LogicalType.hh" +#include "Types.hh" +#include "SchemaResolution.hh" + +namespace avro { + +class Node; +class GenericDatum; + +typedef std::shared_ptr<Node> NodePtr; + +class AVRO_DECL Name { + std::string ns_; + std::string simpleName_; +public: + Name() { } + Name(const std::string& fullname); + Name(const std::string& simpleName, const std::string& ns) : ns_(ns), simpleName_(simpleName) { check(); } + + const std::string fullname() const; + const std::string& ns() const { return ns_; } + const std::string& simpleName() const { return simpleName_; } + + void ns(const std::string& n) { ns_ = n; } + void simpleName(const std::string& n) { simpleName_ = n; } + void fullname(const std::string& n); + + bool operator < (const Name& n) const; + void check() const; + bool operator == (const Name& n) const; + bool operator != (const Name& n) const { return !((*this) == n); } + void clear() { + ns_.clear(); + simpleName_.clear(); + } + operator std::string() const { + return fullname(); + } +}; + +inline +std::ostream& operator << (std::ostream& os, const Name& n) { + return os << n.fullname(); +} + +/// Node is the building block for parse trees. Each node represents an avro +/// type. Compound types have leaf nodes that represent the types they are +/// composed of. +/// +/// The user does not use the Node object directly, they interface with Schema +/// objects. +/// +/// The Node object uses reference-counted pointers. This is so that schemas +/// may be reused in other schemas, without needing to worry about memory +/// deallocation for nodes that are added to multiple schema parse trees. +/// +/// Node has minimal implementation, serving as an abstract base class for +/// different node types. +/// + +class AVRO_DECL Node : private boost::noncopyable +{ + public: + + Node(Type type) : + type_(type), + logicalType_(LogicalType::NONE), + locked_(false) + {} + + virtual ~Node(); + + Type type() const { + return type_; + } + + LogicalType logicalType() const { + return logicalType_; + } + + void setLogicalType(LogicalType logicalType); + + void lock() { + locked_ = true; + } + + bool locked() const { + return locked_; + } + + virtual bool hasName() const = 0; + + void setName(const Name &name) { + checkLock(); + checkName(name); + doSetName(name); + } + virtual const Name &name() const = 0; + + virtual const std::string &getDoc() const = 0; + void setDoc(const std::string &doc) { + checkLock(); + doSetDoc(doc); + } + + void addLeaf(const NodePtr &newLeaf) { + checkLock(); + doAddLeaf(newLeaf); + } + virtual size_t leaves() const = 0; + virtual const NodePtr& leafAt(int index) const = 0; + virtual const GenericDatum& defaultValueAt(int index) { + throw Exception(boost::format("No default value at: %1%") % index); + } + + void addName(const std::string &name) { + checkLock(); + checkName(name); + doAddName(name); + } + virtual size_t names() const = 0; + virtual const std::string &nameAt(int index) const = 0; + virtual bool nameIndex(const std::string &name, size_t &index) const = 0; + + void setFixedSize(int size) { + checkLock(); + doSetFixedSize(size); + } + virtual int fixedSize() const = 0; + + virtual bool isValid() const = 0; + + virtual SchemaResolution resolve(const Node &reader) const = 0; + + virtual void printJson(std::ostream &os, int depth) const = 0; + + virtual void printBasicInfo(std::ostream &os) const = 0; + + virtual void setLeafToSymbolic(int index, const NodePtr &node) = 0; + + // Serialize the default value GenericDatum g for the node contained + // in a record node. + virtual void printDefaultToJson(const GenericDatum& g, std::ostream &os, + int depth) const = 0; + + protected: + + void checkLock() const { + if(locked()) { + throw Exception("Cannot modify locked schema"); + } + } + + virtual void checkName(const Name &name) const { + name.check(); + } + + virtual void doSetName(const Name &name) = 0; + virtual void doSetDoc(const std::string &name) = 0; + + virtual void doAddLeaf(const NodePtr &newLeaf) = 0; + virtual void doAddName(const std::string &name) = 0; + virtual void doSetFixedSize(int size) = 0; + + private: + + const Type type_; + LogicalType logicalType_; + bool locked_; +}; + +} // namespace avro + +namespace std { +inline std::ostream& operator<<(std::ostream& os, const avro::Node& n) +{ + n.printJson(os, 0); + return os; +} +} + + +#endif diff --git a/contrib/libs/apache/avro/api/NodeConcepts.hh b/contrib/libs/apache/avro/api/NodeConcepts.hh index 36d4f413f1..e914d925b6 100644 --- a/contrib/libs/apache/avro/api/NodeConcepts.hh +++ b/contrib/libs/apache/avro/api/NodeConcepts.hh @@ -1,224 +1,224 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_NodeConcepts_hh__ -#define avro_NodeConcepts_hh__ - -#include "Config.hh" - -#include <vector> -#include <map> -#include "Exception.hh" - -namespace avro { - - -/// -/// The concept classes are used to simplify NodeImpl. Since different types -/// of avro types carry different attributes, such as names, or field names for -/// record members. Using the concept class of NoAttribute vs Attribute, the -/// NodeImpl object can enable/disable the attribute, but the code is the same -/// in either case. -/// -/// Furthermore, attributes may have different types, for example, most -/// attributes are strings, but fixed types have a size attribute, which is -/// integer. -/// -/// Since compound types are composed of other types, the leaf attribute -/// concepts extend a NodeImpl to include leaf nodes, and attributes for leaf -/// nodes, which are used to build parse trees. -/// -/// - -namespace concepts { - -template <typename Attribute> -struct NoAttribute -{ - static const bool hasAttribute = false; - - size_t size() const { - return 0; - } - +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_NodeConcepts_hh__ +#define avro_NodeConcepts_hh__ + +#include "Config.hh" + +#include <vector> +#include <map> +#include "Exception.hh" + +namespace avro { + + +/// +/// The concept classes are used to simplify NodeImpl. Since different types +/// of avro types carry different attributes, such as names, or field names for +/// record members. Using the concept class of NoAttribute vs Attribute, the +/// NodeImpl object can enable/disable the attribute, but the code is the same +/// in either case. +/// +/// Furthermore, attributes may have different types, for example, most +/// attributes are strings, but fixed types have a size attribute, which is +/// integer. +/// +/// Since compound types are composed of other types, the leaf attribute +/// concepts extend a NodeImpl to include leaf nodes, and attributes for leaf +/// nodes, which are used to build parse trees. +/// +/// + +namespace concepts { + +template <typename Attribute> +struct NoAttribute +{ + static const bool hasAttribute = false; + + size_t size() const { + return 0; + } + void add( const Attribute &) { - // There must be an add function for the generic NodeImpl, but the - // Node APIs ensure that it is never called, the throw here is - // just in case - throw Exception("This type does not have attribute"); - } - + // There must be an add function for the generic NodeImpl, but the + // Node APIs ensure that it is never called, the throw here is + // just in case + throw Exception("This type does not have attribute"); + } + const Attribute &get(size_t = 0) const { - // There must be an get function for the generic NodeImpl, but the - // Node APIs ensure that it is never called, the throw here is - // just in case - throw Exception("This type does not have attribute"); - // even though this code is unreachable the compiler requires it - static const Attribute empty = Attribute(); - return empty; - } - + // There must be an get function for the generic NodeImpl, but the + // Node APIs ensure that it is never called, the throw here is + // just in case + throw Exception("This type does not have attribute"); + // even though this code is unreachable the compiler requires it + static const Attribute empty = Attribute(); + return empty; + } + Attribute &get(size_t = 0) { - // There must be an get function for the generic NodeImpl, but the - // Node APIs ensure that it is never called, the throw here is - // just in case - throw Exception("This type does not have attribute"); - } - -}; - -template<typename Attribute> -struct SingleAttribute -{ - static const bool hasAttribute = true; - - SingleAttribute() : attr_() - { } - - SingleAttribute(const Attribute& a) : attr_(a) { } - // copy constructing from another single attribute is allowed - SingleAttribute(const SingleAttribute<Attribute> &rhs) : - attr_(rhs.attr_) - { } - - // copy constructing from a no attribute is allowed + // There must be an get function for the generic NodeImpl, but the + // Node APIs ensure that it is never called, the throw here is + // just in case + throw Exception("This type does not have attribute"); + } + +}; + +template<typename Attribute> +struct SingleAttribute +{ + static const bool hasAttribute = true; + + SingleAttribute() : attr_() + { } + + SingleAttribute(const Attribute& a) : attr_(a) { } + // copy constructing from another single attribute is allowed + SingleAttribute(const SingleAttribute<Attribute> &rhs) : + attr_(rhs.attr_) + { } + + // copy constructing from a no attribute is allowed SingleAttribute(const NoAttribute<Attribute> &) : - attr_() - { } - - size_t size() const { - return 1; - } - - void add(const Attribute &attr) { - attr_ = attr; - } - - const Attribute &get(size_t index = 0) const { - if (index != 0) { - throw Exception("SingleAttribute has only 1 value"); - } - return attr_; - } - - Attribute &get(size_t index = 0) { - if (index != 0) { - throw Exception("SingleAttribute has only 1 value"); - } - return attr_; - } - -private: - template<typename T> friend struct MultiAttribute; - Attribute attr_; -}; - -template<typename Attribute> -struct MultiAttribute -{ - static const bool hasAttribute = true; - - MultiAttribute() - { } - - // copy constructing from another single attribute is allowed, it - // pushes the attribute - MultiAttribute(const SingleAttribute<Attribute> &rhs) - { - // since map is the only type that does this we know it's - // final size will be two, so reserve - attrs_.reserve(2); - attrs_.push_back(rhs.attr_); - } - - MultiAttribute(const MultiAttribute<Attribute> &rhs) : - attrs_(rhs.attrs_) - { } - + attr_() + { } + + size_t size() const { + return 1; + } + + void add(const Attribute &attr) { + attr_ = attr; + } + + const Attribute &get(size_t index = 0) const { + if (index != 0) { + throw Exception("SingleAttribute has only 1 value"); + } + return attr_; + } + + Attribute &get(size_t index = 0) { + if (index != 0) { + throw Exception("SingleAttribute has only 1 value"); + } + return attr_; + } + +private: + template<typename T> friend struct MultiAttribute; + Attribute attr_; +}; + +template<typename Attribute> +struct MultiAttribute +{ + static const bool hasAttribute = true; + + MultiAttribute() + { } + + // copy constructing from another single attribute is allowed, it + // pushes the attribute + MultiAttribute(const SingleAttribute<Attribute> &rhs) + { + // since map is the only type that does this we know it's + // final size will be two, so reserve + attrs_.reserve(2); + attrs_.push_back(rhs.attr_); + } + + MultiAttribute(const MultiAttribute<Attribute> &rhs) : + attrs_(rhs.attrs_) + { } + MultiAttribute(const NoAttribute<Attribute> &) - {} - - size_t size() const { - return attrs_.size(); - } - - void add(const Attribute &attr) { - attrs_.push_back(attr); - } - - const Attribute &get(size_t index = 0) const { - return attrs_.at(index); - } - - Attribute &get(size_t index) { - return attrs_.at(index); - } - - private: - - std::vector<Attribute> attrs_; -}; - - -template<typename T> -struct NameIndexConcept { - + {} + + size_t size() const { + return attrs_.size(); + } + + void add(const Attribute &attr) { + attrs_.push_back(attr); + } + + const Attribute &get(size_t index = 0) const { + return attrs_.at(index); + } + + Attribute &get(size_t index) { + return attrs_.at(index); + } + + private: + + std::vector<Attribute> attrs_; +}; + + +template<typename T> +struct NameIndexConcept { + bool lookup(const std::string &, size_t &) const { - throw Exception("Name index does not exist"); - return 0; - } - + throw Exception("Name index does not exist"); + return 0; + } + bool add(const::std::string &, size_t) { - throw Exception("Name index does not exist"); - return false; - } -}; - -template<> -struct NameIndexConcept < MultiAttribute<std::string> > -{ - typedef std::map<std::string, size_t> IndexMap; - - bool lookup(const std::string &name, size_t &index) const { - IndexMap::const_iterator iter = map_.find(name); - if(iter == map_.end()) { - return false; - } - index = iter->second; - return true; - } - - bool add(const::std::string &name, size_t index) { - bool added = false; - IndexMap::iterator lb = map_.lower_bound(name); - if(lb == map_.end() || map_.key_comp()(name, lb->first)) { - map_.insert(lb, IndexMap::value_type(name, index)); - added = true; - } - return added; - } - - private: - - IndexMap map_; -}; - -} // namespace concepts -} // namespace avro - -#endif + throw Exception("Name index does not exist"); + return false; + } +}; + +template<> +struct NameIndexConcept < MultiAttribute<std::string> > +{ + typedef std::map<std::string, size_t> IndexMap; + + bool lookup(const std::string &name, size_t &index) const { + IndexMap::const_iterator iter = map_.find(name); + if(iter == map_.end()) { + return false; + } + index = iter->second; + return true; + } + + bool add(const::std::string &name, size_t index) { + bool added = false; + IndexMap::iterator lb = map_.lower_bound(name); + if(lb == map_.end() || map_.key_comp()(name, lb->first)) { + map_.insert(lb, IndexMap::value_type(name, index)); + added = true; + } + return added; + } + + private: + + IndexMap map_; +}; + +} // namespace concepts +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/NodeImpl.hh b/contrib/libs/apache/avro/api/NodeImpl.hh index f0f1cd9def..debce720a6 100644 --- a/contrib/libs/apache/avro/api/NodeImpl.hh +++ b/contrib/libs/apache/avro/api/NodeImpl.hh @@ -1,619 +1,619 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_NodeImpl_hh__ -#define avro_NodeImpl_hh__ - -#include "Config.hh" -#include "GenericDatum.hh" - -#include <limits> -#include <set> -#include <iomanip> -#include <iostream> -#include <sstream> -#include <memory> - -#include "Node.hh" -#include "NodeConcepts.hh" - -namespace avro { - -/// Implementation details for Node. NodeImpl represents all the avro types, -/// whose properties are enabled and disabled by selecting concept classes. - -template -< - class NameConcept, - class LeavesConcept, - class LeafNamesConcept, - class SizeConcept -> -class NodeImpl : public Node -{ - - protected: - - NodeImpl(Type type) : - Node(type), - nameAttribute_(), - docAttribute_(), - leafAttributes_(), - leafNameAttributes_(), - sizeAttribute_() - { } - - NodeImpl(Type type, - const NameConcept &name, - const LeavesConcept &leaves, - const LeafNamesConcept &leafNames, - const SizeConcept &size) : - Node(type), - nameAttribute_(name), - docAttribute_(), - leafAttributes_(leaves), - leafNameAttributes_(leafNames), - sizeAttribute_(size) - { } - - // Ctor with "doc" - NodeImpl(Type type, - const NameConcept &name, - const concepts::SingleAttribute<std::string> &doc, - const LeavesConcept &leaves, - const LeafNamesConcept &leafNames, - const SizeConcept &size) : - Node(type), - nameAttribute_(name), - docAttribute_(doc), - leafAttributes_(leaves), - leafNameAttributes_(leafNames), - sizeAttribute_(size) - {} - - void swap(NodeImpl& impl) { - std::swap(nameAttribute_, impl.nameAttribute_); - std::swap(docAttribute_, impl.docAttribute_); - std::swap(leafAttributes_, impl.leafAttributes_); - std::swap(leafNameAttributes_, impl.leafNameAttributes_); - std::swap(sizeAttribute_, impl.sizeAttribute_); - std::swap(nameIndex_, impl.nameIndex_); - } - - bool hasName() const { - // e.g.: true for single and multiattributes, false for noattributes. - return NameConcept::hasAttribute; - } - - void doSetName(const Name &name) { - nameAttribute_.add(name); - } - - const Name &name() const { - return nameAttribute_.get(); - } - - void doSetDoc(const std::string &doc) { - docAttribute_.add(doc); - } - - const std::string &getDoc() const { - return docAttribute_.get(); - } - - void doAddLeaf(const NodePtr &newLeaf) { - leafAttributes_.add(newLeaf); - } - - size_t leaves() const { - return leafAttributes_.size(); - } - - const NodePtr &leafAt(int index) const { - return leafAttributes_.get(index); - } - - void doAddName(const std::string &name) { - if (! nameIndex_.add(name, leafNameAttributes_.size())) { - throw Exception(boost::format("Cannot add duplicate name: %1%") % name); - } - leafNameAttributes_.add(name); - } - - size_t names() const { - return leafNameAttributes_.size(); - } - - const std::string &nameAt(int index) const { - return leafNameAttributes_.get(index); - } - - bool nameIndex(const std::string &name, size_t &index) const { - return nameIndex_.lookup(name, index); - } - - void doSetFixedSize(int size) { - sizeAttribute_.add(size); - } - - int fixedSize() const { - return sizeAttribute_.get(); - } - - virtual bool isValid() const = 0; - - void printBasicInfo(std::ostream &os) const; - - void setLeafToSymbolic(int index, const NodePtr &node); - - SchemaResolution furtherResolution(const Node &reader) const { - SchemaResolution match = RESOLVE_NO_MATCH; - - if (reader.type() == AVRO_SYMBOLIC) { - - // resolve the symbolic type, and check again - const NodePtr &node = reader.leafAt(0); - match = resolve(*node); - } - else if(reader.type() == AVRO_UNION) { - - // in this case, need to see if there is an exact match for the - // writer's type, or if not, the first one that can be promoted to a - // match - - for(size_t i= 0; i < reader.leaves(); ++i) { - - const NodePtr &node = reader.leafAt(i); - SchemaResolution thisMatch = resolve(*node); - - // if matched then the search is done - if(thisMatch == RESOLVE_MATCH) { - match = thisMatch; - break; - } - - // thisMatch is either no match, or promotable, this will set match to - // promotable if it hasn't been set already - if (match == RESOLVE_NO_MATCH) { - match = thisMatch; - } - } - } - - return match; - } - - NameConcept nameAttribute_; - - // Rem: NameConcept type is HasName (= SingleAttribute<Name>), we use std::string instead - concepts::SingleAttribute<std::string> docAttribute_; /** Doc used to compare schemas */ - - LeavesConcept leafAttributes_; - LeafNamesConcept leafNameAttributes_; - SizeConcept sizeAttribute_; - concepts::NameIndexConcept<LeafNamesConcept> nameIndex_; -}; - -typedef concepts::NoAttribute<Name> NoName; -typedef concepts::SingleAttribute<Name> HasName; - -typedef concepts::SingleAttribute<std::string> HasDoc; - -typedef concepts::NoAttribute<NodePtr> NoLeaves; -typedef concepts::SingleAttribute<NodePtr> SingleLeaf; -typedef concepts::MultiAttribute<NodePtr> MultiLeaves; - -typedef concepts::NoAttribute<std::string> NoLeafNames; -typedef concepts::MultiAttribute<std::string> LeafNames; - -typedef concepts::NoAttribute<int> NoSize; -typedef concepts::SingleAttribute<int> HasSize; - -typedef NodeImpl< NoName, NoLeaves, NoLeafNames, NoSize > NodeImplPrimitive; -typedef NodeImpl< HasName, NoLeaves, NoLeafNames, NoSize > NodeImplSymbolic; - -typedef NodeImpl< HasName, MultiLeaves, LeafNames, NoSize > NodeImplRecord; -typedef NodeImpl< HasName, NoLeaves, LeafNames, NoSize > NodeImplEnum; -typedef NodeImpl< NoName, SingleLeaf, NoLeafNames, NoSize > NodeImplArray; -typedef NodeImpl< NoName, MultiLeaves, NoLeafNames, NoSize > NodeImplMap; -typedef NodeImpl< NoName, MultiLeaves, NoLeafNames, NoSize > NodeImplUnion; -typedef NodeImpl< HasName, NoLeaves, NoLeafNames, HasSize > NodeImplFixed; - -class AVRO_DECL NodePrimitive : public NodeImplPrimitive -{ - public: - - explicit NodePrimitive(Type type) : - NodeImplPrimitive(type) - { } - - SchemaResolution resolve(const Node &reader) const; - - void printJson(std::ostream &os, int depth) const; - - bool isValid() const { - return true; - } - - void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; -}; - -class AVRO_DECL NodeSymbolic : public NodeImplSymbolic -{ - typedef std::weak_ptr<Node> NodeWeakPtr; - - public: - - NodeSymbolic() : - NodeImplSymbolic(AVRO_SYMBOLIC) - { } - - explicit NodeSymbolic(const HasName &name) : - NodeImplSymbolic(AVRO_SYMBOLIC, name, NoLeaves(), NoLeafNames(), NoSize()) - { } - - NodeSymbolic(const HasName &name, const NodePtr n) : - NodeImplSymbolic(AVRO_SYMBOLIC, name, NoLeaves(), NoLeafNames(), NoSize()), actualNode_(n) - { } - SchemaResolution resolve(const Node &reader) const; - - void printJson(std::ostream &os, int depth) const; - - bool isValid() const { - return (nameAttribute_.size() == 1); - } - - void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; - - bool isSet() const { - return (actualNode_.lock() != 0); - } - - NodePtr getNode() const { - NodePtr node = actualNode_.lock(); - if(!node) { - throw Exception(boost::format("Could not follow symbol %1%") % name()); - } - return node; - } - - void setNode(const NodePtr &node) { - actualNode_ = node; - } - - protected: - - NodeWeakPtr actualNode_; - -}; - -class AVRO_DECL NodeRecord : public NodeImplRecord { - std::vector<GenericDatum> defaultValues; -public: - NodeRecord() : NodeImplRecord(AVRO_RECORD) { } - NodeRecord(const HasName &name, const MultiLeaves &fields, - const LeafNames &fieldsNames, - const std::vector<GenericDatum>& dv) : - NodeImplRecord(AVRO_RECORD, name, fields, fieldsNames, NoSize()), - defaultValues(dv) { - for (size_t i = 0; i < leafNameAttributes_.size(); ++i) { - if (!nameIndex_.add(leafNameAttributes_.get(i), i)) { - throw Exception(boost::format( - "Cannot add duplicate field: %1%") % - leafNameAttributes_.get(i)); - } - } - } - - NodeRecord(const HasName &name, const HasDoc &doc, const MultiLeaves &fields, - const LeafNames &fieldsNames, - const std::vector<GenericDatum> &dv) : - NodeImplRecord(AVRO_RECORD, name, doc, fields, fieldsNames, NoSize()), - defaultValues(dv) { - for (size_t i = 0; i < leafNameAttributes_.size(); ++i) { - if (!nameIndex_.add(leafNameAttributes_.get(i), i)) { - throw Exception(boost::format( - "Cannot add duplicate field: %1%") % - leafNameAttributes_.get(i)); - } - } - } - - void swap(NodeRecord& r) { - NodeImplRecord::swap(r); - defaultValues.swap(r.defaultValues); - } - - SchemaResolution resolve(const Node &reader) const; - - void printJson(std::ostream &os, int depth) const; - - bool isValid() const { - return ((nameAttribute_.size() == 1) && - (leafAttributes_.size() == leafNameAttributes_.size())); - } - - const GenericDatum& defaultValueAt(int index) { - return defaultValues[index]; - } - - void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; -}; - -class AVRO_DECL NodeEnum : public NodeImplEnum -{ - public: - - NodeEnum() : - NodeImplEnum(AVRO_ENUM) - { } - - NodeEnum(const HasName &name, const LeafNames &symbols) : - NodeImplEnum(AVRO_ENUM, name, NoLeaves(), symbols, NoSize()) - { - for(size_t i=0; i < leafNameAttributes_.size(); ++i) { - if(!nameIndex_.add(leafNameAttributes_.get(i), i)) { - throw Exception(boost::format("Cannot add duplicate enum: %1%") % leafNameAttributes_.get(i)); - } - } - } - - SchemaResolution resolve(const Node &reader) const; - - void printJson(std::ostream &os, int depth) const; - - bool isValid() const { - return ( - (nameAttribute_.size() == 1) && - (leafNameAttributes_.size() > 0) - ); - } - - void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; -}; - -class AVRO_DECL NodeArray : public NodeImplArray -{ - public: - - NodeArray() : - NodeImplArray(AVRO_ARRAY) - { } - - explicit NodeArray(const SingleLeaf &items) : - NodeImplArray(AVRO_ARRAY, NoName(), items, NoLeafNames(), NoSize()) - { } - - SchemaResolution resolve(const Node &reader) const; - - void printJson(std::ostream &os, int depth) const; - - bool isValid() const { - return (leafAttributes_.size() == 1); - } - - void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; -}; - -class AVRO_DECL NodeMap : public NodeImplMap -{ - public: - - NodeMap() : - NodeImplMap(AVRO_MAP) - { - NodePtr key(new NodePrimitive(AVRO_STRING)); - doAddLeaf(key); - } - - explicit NodeMap(const SingleLeaf &values) : - NodeImplMap(AVRO_MAP, NoName(), values, NoLeafNames(), NoSize()) - { - // need to add the key for the map too - NodePtr key(new NodePrimitive(AVRO_STRING)); - doAddLeaf(key); - - // key goes before value - std::swap(leafAttributes_.get(0), leafAttributes_.get(1)); - } - - SchemaResolution resolve(const Node &reader) const; - - void printJson(std::ostream &os, int depth) const; - - bool isValid() const { - return (leafAttributes_.size() == 2); - } - - void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; -}; - -class AVRO_DECL NodeUnion : public NodeImplUnion -{ - public: - - NodeUnion() : - NodeImplUnion(AVRO_UNION) - { } - - explicit NodeUnion(const MultiLeaves &types) : - NodeImplUnion(AVRO_UNION, NoName(), types, NoLeafNames(), NoSize()) - { } - - SchemaResolution resolve(const Node &reader) const; - - void printJson(std::ostream &os, int depth) const; - - bool isValid() const { - std::set<std::string> seen; - if (leafAttributes_.size() >= 1) { - for (size_t i = 0; i < leafAttributes_.size(); ++i) { - std::string name; - const NodePtr& n = leafAttributes_.get(i); - switch (n->type()) { - case AVRO_STRING: - name = "string"; - break; - case AVRO_BYTES: - name = "bytes"; - break; - case AVRO_INT: - name = "int"; - break; - case AVRO_LONG: - name = "long"; - break; - case AVRO_FLOAT: - name = "float"; - break; - case AVRO_DOUBLE: - name = "double"; - break; - case AVRO_BOOL: - name = "bool"; - break; - case AVRO_NULL: - name = "null"; - break; - case AVRO_ARRAY: - name = "array"; - break; - case AVRO_MAP: - name = "map"; - break; - case AVRO_RECORD: - case AVRO_ENUM: - case AVRO_UNION: - case AVRO_FIXED: - case AVRO_SYMBOLIC: - name = n->name().fullname(); - break; - default: - return false; - } - if (seen.find(name) != seen.end()) { - return false; - } - seen.insert(name); - } - return true; - } - return false; - } - - void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; -}; - -class AVRO_DECL NodeFixed : public NodeImplFixed -{ - public: - - NodeFixed() : - NodeImplFixed(AVRO_FIXED) - { } - - NodeFixed(const HasName &name, const HasSize &size) : - NodeImplFixed(AVRO_FIXED, name, NoLeaves(), NoLeafNames(), size) - { } - - SchemaResolution resolve(const Node &reader) const; - - void printJson(std::ostream &os, int depth) const; - - bool isValid() const { - return ( - (nameAttribute_.size() == 1) && - (sizeAttribute_.size() == 1) - ); - } - - void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; -}; - -template < class A, class B, class C, class D > -inline void -NodeImpl<A,B,C,D>::setLeafToSymbolic(int index, const NodePtr &node) -{ - if(!B::hasAttribute) { - throw Exception("Cannot change leaf node for nonexistent leaf"); - } - - NodePtr &replaceNode = const_cast<NodePtr &>(leafAttributes_.get(index)); - if(replaceNode->name() != node->name()) { - throw Exception("Symbolic name does not match the name of the schema it references"); - } - - NodePtr symbol(new NodeSymbolic); - NodeSymbolic *ptr = static_cast<NodeSymbolic *> (symbol.get()); - - ptr->setName(node->name()); - ptr->setNode(node); - replaceNode.swap(symbol); -} - -template < class A, class B, class C, class D > -inline void -NodeImpl<A,B,C,D>::printBasicInfo(std::ostream &os) const -{ - os << type(); - if(hasName()) { - os << ' ' << nameAttribute_.get(); - } - - if(D::hasAttribute) { - os << " " << sizeAttribute_.get(); - } - os << '\n'; - int count = leaves(); - count = count ? count : names(); - for(int i= 0; i < count; ++i) { - if( C::hasAttribute ) { - os << "name " << nameAt(i) << '\n'; - } - if( type() != AVRO_SYMBOLIC && leafAttributes_.hasAttribute) { - leafAt(i)->printBasicInfo(os); - } - } - if(isCompound(type())) { - os << "end " << type() << '\n'; - } -} - - -inline NodePtr resolveSymbol(const NodePtr &node) -{ - if(node->type() != AVRO_SYMBOLIC) { - throw Exception("Only symbolic nodes may be resolved"); - } - std::shared_ptr<NodeSymbolic> symNode = std::static_pointer_cast<NodeSymbolic>(node); - return symNode->getNode(); -} - -template< typename T > -inline std::string intToHex(T i) -{ - std::stringstream stream; - stream << "\\u" - << std::setfill('0') << std::setw(sizeof(T)) - << std::hex << i; - return stream.str(); -} - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_NodeImpl_hh__ +#define avro_NodeImpl_hh__ + +#include "Config.hh" +#include "GenericDatum.hh" + +#include <limits> +#include <set> +#include <iomanip> +#include <iostream> +#include <sstream> +#include <memory> + +#include "Node.hh" +#include "NodeConcepts.hh" + +namespace avro { + +/// Implementation details for Node. NodeImpl represents all the avro types, +/// whose properties are enabled and disabled by selecting concept classes. + +template +< + class NameConcept, + class LeavesConcept, + class LeafNamesConcept, + class SizeConcept +> +class NodeImpl : public Node +{ + + protected: + + NodeImpl(Type type) : + Node(type), + nameAttribute_(), + docAttribute_(), + leafAttributes_(), + leafNameAttributes_(), + sizeAttribute_() + { } + + NodeImpl(Type type, + const NameConcept &name, + const LeavesConcept &leaves, + const LeafNamesConcept &leafNames, + const SizeConcept &size) : + Node(type), + nameAttribute_(name), + docAttribute_(), + leafAttributes_(leaves), + leafNameAttributes_(leafNames), + sizeAttribute_(size) + { } + + // Ctor with "doc" + NodeImpl(Type type, + const NameConcept &name, + const concepts::SingleAttribute<std::string> &doc, + const LeavesConcept &leaves, + const LeafNamesConcept &leafNames, + const SizeConcept &size) : + Node(type), + nameAttribute_(name), + docAttribute_(doc), + leafAttributes_(leaves), + leafNameAttributes_(leafNames), + sizeAttribute_(size) + {} + + void swap(NodeImpl& impl) { + std::swap(nameAttribute_, impl.nameAttribute_); + std::swap(docAttribute_, impl.docAttribute_); + std::swap(leafAttributes_, impl.leafAttributes_); + std::swap(leafNameAttributes_, impl.leafNameAttributes_); + std::swap(sizeAttribute_, impl.sizeAttribute_); + std::swap(nameIndex_, impl.nameIndex_); + } + + bool hasName() const { + // e.g.: true for single and multiattributes, false for noattributes. + return NameConcept::hasAttribute; + } + + void doSetName(const Name &name) { + nameAttribute_.add(name); + } + + const Name &name() const { + return nameAttribute_.get(); + } + + void doSetDoc(const std::string &doc) { + docAttribute_.add(doc); + } + + const std::string &getDoc() const { + return docAttribute_.get(); + } + + void doAddLeaf(const NodePtr &newLeaf) { + leafAttributes_.add(newLeaf); + } + + size_t leaves() const { + return leafAttributes_.size(); + } + + const NodePtr &leafAt(int index) const { + return leafAttributes_.get(index); + } + + void doAddName(const std::string &name) { + if (! nameIndex_.add(name, leafNameAttributes_.size())) { + throw Exception(boost::format("Cannot add duplicate name: %1%") % name); + } + leafNameAttributes_.add(name); + } + + size_t names() const { + return leafNameAttributes_.size(); + } + + const std::string &nameAt(int index) const { + return leafNameAttributes_.get(index); + } + + bool nameIndex(const std::string &name, size_t &index) const { + return nameIndex_.lookup(name, index); + } + + void doSetFixedSize(int size) { + sizeAttribute_.add(size); + } + + int fixedSize() const { + return sizeAttribute_.get(); + } + + virtual bool isValid() const = 0; + + void printBasicInfo(std::ostream &os) const; + + void setLeafToSymbolic(int index, const NodePtr &node); + + SchemaResolution furtherResolution(const Node &reader) const { + SchemaResolution match = RESOLVE_NO_MATCH; + + if (reader.type() == AVRO_SYMBOLIC) { + + // resolve the symbolic type, and check again + const NodePtr &node = reader.leafAt(0); + match = resolve(*node); + } + else if(reader.type() == AVRO_UNION) { + + // in this case, need to see if there is an exact match for the + // writer's type, or if not, the first one that can be promoted to a + // match + + for(size_t i= 0; i < reader.leaves(); ++i) { + + const NodePtr &node = reader.leafAt(i); + SchemaResolution thisMatch = resolve(*node); + + // if matched then the search is done + if(thisMatch == RESOLVE_MATCH) { + match = thisMatch; + break; + } + + // thisMatch is either no match, or promotable, this will set match to + // promotable if it hasn't been set already + if (match == RESOLVE_NO_MATCH) { + match = thisMatch; + } + } + } + + return match; + } + + NameConcept nameAttribute_; + + // Rem: NameConcept type is HasName (= SingleAttribute<Name>), we use std::string instead + concepts::SingleAttribute<std::string> docAttribute_; /** Doc used to compare schemas */ + + LeavesConcept leafAttributes_; + LeafNamesConcept leafNameAttributes_; + SizeConcept sizeAttribute_; + concepts::NameIndexConcept<LeafNamesConcept> nameIndex_; +}; + +typedef concepts::NoAttribute<Name> NoName; +typedef concepts::SingleAttribute<Name> HasName; + +typedef concepts::SingleAttribute<std::string> HasDoc; + +typedef concepts::NoAttribute<NodePtr> NoLeaves; +typedef concepts::SingleAttribute<NodePtr> SingleLeaf; +typedef concepts::MultiAttribute<NodePtr> MultiLeaves; + +typedef concepts::NoAttribute<std::string> NoLeafNames; +typedef concepts::MultiAttribute<std::string> LeafNames; + +typedef concepts::NoAttribute<int> NoSize; +typedef concepts::SingleAttribute<int> HasSize; + +typedef NodeImpl< NoName, NoLeaves, NoLeafNames, NoSize > NodeImplPrimitive; +typedef NodeImpl< HasName, NoLeaves, NoLeafNames, NoSize > NodeImplSymbolic; + +typedef NodeImpl< HasName, MultiLeaves, LeafNames, NoSize > NodeImplRecord; +typedef NodeImpl< HasName, NoLeaves, LeafNames, NoSize > NodeImplEnum; +typedef NodeImpl< NoName, SingleLeaf, NoLeafNames, NoSize > NodeImplArray; +typedef NodeImpl< NoName, MultiLeaves, NoLeafNames, NoSize > NodeImplMap; +typedef NodeImpl< NoName, MultiLeaves, NoLeafNames, NoSize > NodeImplUnion; +typedef NodeImpl< HasName, NoLeaves, NoLeafNames, HasSize > NodeImplFixed; + +class AVRO_DECL NodePrimitive : public NodeImplPrimitive +{ + public: + + explicit NodePrimitive(Type type) : + NodeImplPrimitive(type) + { } + + SchemaResolution resolve(const Node &reader) const; + + void printJson(std::ostream &os, int depth) const; + + bool isValid() const { + return true; + } + + void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; +}; + +class AVRO_DECL NodeSymbolic : public NodeImplSymbolic +{ + typedef std::weak_ptr<Node> NodeWeakPtr; + + public: + + NodeSymbolic() : + NodeImplSymbolic(AVRO_SYMBOLIC) + { } + + explicit NodeSymbolic(const HasName &name) : + NodeImplSymbolic(AVRO_SYMBOLIC, name, NoLeaves(), NoLeafNames(), NoSize()) + { } + + NodeSymbolic(const HasName &name, const NodePtr n) : + NodeImplSymbolic(AVRO_SYMBOLIC, name, NoLeaves(), NoLeafNames(), NoSize()), actualNode_(n) + { } + SchemaResolution resolve(const Node &reader) const; + + void printJson(std::ostream &os, int depth) const; + + bool isValid() const { + return (nameAttribute_.size() == 1); + } + + void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; + + bool isSet() const { + return (actualNode_.lock() != 0); + } + + NodePtr getNode() const { + NodePtr node = actualNode_.lock(); + if(!node) { + throw Exception(boost::format("Could not follow symbol %1%") % name()); + } + return node; + } + + void setNode(const NodePtr &node) { + actualNode_ = node; + } + + protected: + + NodeWeakPtr actualNode_; + +}; + +class AVRO_DECL NodeRecord : public NodeImplRecord { + std::vector<GenericDatum> defaultValues; +public: + NodeRecord() : NodeImplRecord(AVRO_RECORD) { } + NodeRecord(const HasName &name, const MultiLeaves &fields, + const LeafNames &fieldsNames, + const std::vector<GenericDatum>& dv) : + NodeImplRecord(AVRO_RECORD, name, fields, fieldsNames, NoSize()), + defaultValues(dv) { + for (size_t i = 0; i < leafNameAttributes_.size(); ++i) { + if (!nameIndex_.add(leafNameAttributes_.get(i), i)) { + throw Exception(boost::format( + "Cannot add duplicate field: %1%") % + leafNameAttributes_.get(i)); + } + } + } + + NodeRecord(const HasName &name, const HasDoc &doc, const MultiLeaves &fields, + const LeafNames &fieldsNames, + const std::vector<GenericDatum> &dv) : + NodeImplRecord(AVRO_RECORD, name, doc, fields, fieldsNames, NoSize()), + defaultValues(dv) { + for (size_t i = 0; i < leafNameAttributes_.size(); ++i) { + if (!nameIndex_.add(leafNameAttributes_.get(i), i)) { + throw Exception(boost::format( + "Cannot add duplicate field: %1%") % + leafNameAttributes_.get(i)); + } + } + } + + void swap(NodeRecord& r) { + NodeImplRecord::swap(r); + defaultValues.swap(r.defaultValues); + } + + SchemaResolution resolve(const Node &reader) const; + + void printJson(std::ostream &os, int depth) const; + + bool isValid() const { + return ((nameAttribute_.size() == 1) && + (leafAttributes_.size() == leafNameAttributes_.size())); + } + + const GenericDatum& defaultValueAt(int index) { + return defaultValues[index]; + } + + void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; +}; + +class AVRO_DECL NodeEnum : public NodeImplEnum +{ + public: + + NodeEnum() : + NodeImplEnum(AVRO_ENUM) + { } + + NodeEnum(const HasName &name, const LeafNames &symbols) : + NodeImplEnum(AVRO_ENUM, name, NoLeaves(), symbols, NoSize()) + { + for(size_t i=0; i < leafNameAttributes_.size(); ++i) { + if(!nameIndex_.add(leafNameAttributes_.get(i), i)) { + throw Exception(boost::format("Cannot add duplicate enum: %1%") % leafNameAttributes_.get(i)); + } + } + } + + SchemaResolution resolve(const Node &reader) const; + + void printJson(std::ostream &os, int depth) const; + + bool isValid() const { + return ( + (nameAttribute_.size() == 1) && + (leafNameAttributes_.size() > 0) + ); + } + + void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; +}; + +class AVRO_DECL NodeArray : public NodeImplArray +{ + public: + + NodeArray() : + NodeImplArray(AVRO_ARRAY) + { } + + explicit NodeArray(const SingleLeaf &items) : + NodeImplArray(AVRO_ARRAY, NoName(), items, NoLeafNames(), NoSize()) + { } + + SchemaResolution resolve(const Node &reader) const; + + void printJson(std::ostream &os, int depth) const; + + bool isValid() const { + return (leafAttributes_.size() == 1); + } + + void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; +}; + +class AVRO_DECL NodeMap : public NodeImplMap +{ + public: + + NodeMap() : + NodeImplMap(AVRO_MAP) + { + NodePtr key(new NodePrimitive(AVRO_STRING)); + doAddLeaf(key); + } + + explicit NodeMap(const SingleLeaf &values) : + NodeImplMap(AVRO_MAP, NoName(), values, NoLeafNames(), NoSize()) + { + // need to add the key for the map too + NodePtr key(new NodePrimitive(AVRO_STRING)); + doAddLeaf(key); + + // key goes before value + std::swap(leafAttributes_.get(0), leafAttributes_.get(1)); + } + + SchemaResolution resolve(const Node &reader) const; + + void printJson(std::ostream &os, int depth) const; + + bool isValid() const { + return (leafAttributes_.size() == 2); + } + + void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; +}; + +class AVRO_DECL NodeUnion : public NodeImplUnion +{ + public: + + NodeUnion() : + NodeImplUnion(AVRO_UNION) + { } + + explicit NodeUnion(const MultiLeaves &types) : + NodeImplUnion(AVRO_UNION, NoName(), types, NoLeafNames(), NoSize()) + { } + + SchemaResolution resolve(const Node &reader) const; + + void printJson(std::ostream &os, int depth) const; + + bool isValid() const { + std::set<std::string> seen; + if (leafAttributes_.size() >= 1) { + for (size_t i = 0; i < leafAttributes_.size(); ++i) { + std::string name; + const NodePtr& n = leafAttributes_.get(i); + switch (n->type()) { + case AVRO_STRING: + name = "string"; + break; + case AVRO_BYTES: + name = "bytes"; + break; + case AVRO_INT: + name = "int"; + break; + case AVRO_LONG: + name = "long"; + break; + case AVRO_FLOAT: + name = "float"; + break; + case AVRO_DOUBLE: + name = "double"; + break; + case AVRO_BOOL: + name = "bool"; + break; + case AVRO_NULL: + name = "null"; + break; + case AVRO_ARRAY: + name = "array"; + break; + case AVRO_MAP: + name = "map"; + break; + case AVRO_RECORD: + case AVRO_ENUM: + case AVRO_UNION: + case AVRO_FIXED: + case AVRO_SYMBOLIC: + name = n->name().fullname(); + break; + default: + return false; + } + if (seen.find(name) != seen.end()) { + return false; + } + seen.insert(name); + } + return true; + } + return false; + } + + void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; +}; + +class AVRO_DECL NodeFixed : public NodeImplFixed +{ + public: + + NodeFixed() : + NodeImplFixed(AVRO_FIXED) + { } + + NodeFixed(const HasName &name, const HasSize &size) : + NodeImplFixed(AVRO_FIXED, name, NoLeaves(), NoLeafNames(), size) + { } + + SchemaResolution resolve(const Node &reader) const; + + void printJson(std::ostream &os, int depth) const; + + bool isValid() const { + return ( + (nameAttribute_.size() == 1) && + (sizeAttribute_.size() == 1) + ); + } + + void printDefaultToJson(const GenericDatum& g, std::ostream &os, int depth) const; +}; + +template < class A, class B, class C, class D > +inline void +NodeImpl<A,B,C,D>::setLeafToSymbolic(int index, const NodePtr &node) +{ + if(!B::hasAttribute) { + throw Exception("Cannot change leaf node for nonexistent leaf"); + } + + NodePtr &replaceNode = const_cast<NodePtr &>(leafAttributes_.get(index)); + if(replaceNode->name() != node->name()) { + throw Exception("Symbolic name does not match the name of the schema it references"); + } + + NodePtr symbol(new NodeSymbolic); + NodeSymbolic *ptr = static_cast<NodeSymbolic *> (symbol.get()); + + ptr->setName(node->name()); + ptr->setNode(node); + replaceNode.swap(symbol); +} + +template < class A, class B, class C, class D > +inline void +NodeImpl<A,B,C,D>::printBasicInfo(std::ostream &os) const +{ + os << type(); + if(hasName()) { + os << ' ' << nameAttribute_.get(); + } + + if(D::hasAttribute) { + os << " " << sizeAttribute_.get(); + } + os << '\n'; + int count = leaves(); + count = count ? count : names(); + for(int i= 0; i < count; ++i) { + if( C::hasAttribute ) { + os << "name " << nameAt(i) << '\n'; + } + if( type() != AVRO_SYMBOLIC && leafAttributes_.hasAttribute) { + leafAt(i)->printBasicInfo(os); + } + } + if(isCompound(type())) { + os << "end " << type() << '\n'; + } +} + + +inline NodePtr resolveSymbol(const NodePtr &node) +{ + if(node->type() != AVRO_SYMBOLIC) { + throw Exception("Only symbolic nodes may be resolved"); + } + std::shared_ptr<NodeSymbolic> symNode = std::static_pointer_cast<NodeSymbolic>(node); + return symNode->getNode(); +} + +template< typename T > +inline std::string intToHex(T i) +{ + std::stringstream stream; + stream << "\\u" + << std::setfill('0') << std::setw(sizeof(T)) + << std::hex << i; + return stream.str(); +} + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Parser.hh b/contrib/libs/apache/avro/api/Parser.hh index 3ba6af65e3..fdf28fb0d5 100644 --- a/contrib/libs/apache/avro/api/Parser.hh +++ b/contrib/libs/apache/avro/api/Parser.hh @@ -1,151 +1,151 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Parser_hh__ -#define avro_Parser_hh__ - -#include "Config.hh" -#include "Reader.hh" - -#include <array> - -namespace avro { - -/// -/// Class that wraps a reader or ValidatingReade with an interface that uses -/// explicit get* names instead of getValue -/// - -template<class Reader> -class Parser : private boost::noncopyable -{ - - public: - - // Constructor only works with Writer - explicit Parser(const InputBuffer &in) : - reader_(in) - {} - - /// Constructor only works with ValidatingWriter - Parser(const ValidSchema &schema, const InputBuffer &in) : - reader_(schema, in) - {} - - void readNull() { - Null null; - reader_.readValue(null); - } - - bool readBool() { - bool val; - reader_.readValue(val); - return val; - } - - int32_t readInt() { - int32_t val; - reader_.readValue(val); - return val; - } - - int64_t readLong() { - int64_t val; - reader_.readValue(val); - return val; - } - - float readFloat() { - float val; - reader_.readValue(val); - return val; - } - - double readDouble() { - double val; - reader_.readValue(val); - return val; - } - - void readString(std::string &val) { - reader_.readValue(val); - } - - void readBytes(std::vector<uint8_t> &val) { - reader_.readBytes(val); - } - - template <size_t N> - void readFixed(uint8_t (&val)[N]) { - reader_.readFixed(val); - } - - template<size_t N> - void readFixed(std::array<uint8_t, N> &val) { - reader_.readFixed(val); - } - - void readRecord() { - reader_.readRecord(); - } - - void readRecordEnd() { - reader_.readRecordEnd(); - } - - int64_t readArrayBlockSize() { - return reader_.readArrayBlockSize(); - } - - int64_t readUnion() { - return reader_.readUnion(); - } - - int64_t readEnum() { - return reader_.readEnum(); - } - - int64_t readMapBlockSize() { - return reader_.readMapBlockSize(); - } - - private: - - friend Type nextType(Parser<ValidatingReader> &p); - friend bool currentRecordName(Parser<ValidatingReader> &p, std::string &name); - friend bool nextFieldName(Parser<ValidatingReader> &p, std::string &name); - - Reader reader_; - -}; - -inline Type nextType(Parser<ValidatingReader> &p) { - return p.reader_.nextType(); -} - -inline bool currentRecordName(Parser<ValidatingReader> &p, std::string &name) { - return p.reader_.currentRecordName(name); -} - -inline bool nextFieldName(Parser<ValidatingReader> &p, std::string &name) { - return p.reader_.nextFieldName(name); -} - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Parser_hh__ +#define avro_Parser_hh__ + +#include "Config.hh" +#include "Reader.hh" + +#include <array> + +namespace avro { + +/// +/// Class that wraps a reader or ValidatingReade with an interface that uses +/// explicit get* names instead of getValue +/// + +template<class Reader> +class Parser : private boost::noncopyable +{ + + public: + + // Constructor only works with Writer + explicit Parser(const InputBuffer &in) : + reader_(in) + {} + + /// Constructor only works with ValidatingWriter + Parser(const ValidSchema &schema, const InputBuffer &in) : + reader_(schema, in) + {} + + void readNull() { + Null null; + reader_.readValue(null); + } + + bool readBool() { + bool val; + reader_.readValue(val); + return val; + } + + int32_t readInt() { + int32_t val; + reader_.readValue(val); + return val; + } + + int64_t readLong() { + int64_t val; + reader_.readValue(val); + return val; + } + + float readFloat() { + float val; + reader_.readValue(val); + return val; + } + + double readDouble() { + double val; + reader_.readValue(val); + return val; + } + + void readString(std::string &val) { + reader_.readValue(val); + } + + void readBytes(std::vector<uint8_t> &val) { + reader_.readBytes(val); + } + + template <size_t N> + void readFixed(uint8_t (&val)[N]) { + reader_.readFixed(val); + } + + template<size_t N> + void readFixed(std::array<uint8_t, N> &val) { + reader_.readFixed(val); + } + + void readRecord() { + reader_.readRecord(); + } + + void readRecordEnd() { + reader_.readRecordEnd(); + } + + int64_t readArrayBlockSize() { + return reader_.readArrayBlockSize(); + } + + int64_t readUnion() { + return reader_.readUnion(); + } + + int64_t readEnum() { + return reader_.readEnum(); + } + + int64_t readMapBlockSize() { + return reader_.readMapBlockSize(); + } + + private: + + friend Type nextType(Parser<ValidatingReader> &p); + friend bool currentRecordName(Parser<ValidatingReader> &p, std::string &name); + friend bool nextFieldName(Parser<ValidatingReader> &p, std::string &name); + + Reader reader_; + +}; + +inline Type nextType(Parser<ValidatingReader> &p) { + return p.reader_.nextType(); +} + +inline bool currentRecordName(Parser<ValidatingReader> &p, std::string &name) { + return p.reader_.currentRecordName(name); +} + +inline bool nextFieldName(Parser<ValidatingReader> &p, std::string &name) { + return p.reader_.nextFieldName(name); +} + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Reader.hh b/contrib/libs/apache/avro/api/Reader.hh index 26488145a2..4f514fbbe9 100644 --- a/contrib/libs/apache/avro/api/Reader.hh +++ b/contrib/libs/apache/avro/api/Reader.hh @@ -1,209 +1,209 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Reader_hh__ -#define avro_Reader_hh__ - -#include <stdint.h> -#include <vector> -#include <array> -#include <boost/noncopyable.hpp> - -#include "Config.hh" -#include "Zigzag.hh" -#include "Types.hh" -#include "Validator.hh" -#include "buffer/BufferReader.hh" - -namespace avro { - -/// -/// Parses from an avro encoding to the requested type. Assumes the next item -/// in the avro binary data is the expected type. -/// - -template<class ValidatorType> -class ReaderImpl : private boost::noncopyable -{ - - public: - - explicit ReaderImpl(const InputBuffer &buffer) : - reader_(buffer) - {} - - ReaderImpl(const ValidSchema &schema, const InputBuffer &buffer) : - validator_(schema), - reader_(buffer) - {} - - void readValue(Null &) { - validator_.checkTypeExpected(AVRO_NULL); - } - - void readValue(bool &val) { - validator_.checkTypeExpected(AVRO_BOOL); - uint8_t ival = 0; - reader_.read(ival); - val = (ival != 0); - } - - void readValue(int32_t &val) { - validator_.checkTypeExpected(AVRO_INT); - uint32_t encoded = static_cast<uint32_t>(readVarInt()); - val = decodeZigzag32(encoded); - } - - void readValue(int64_t &val) { - validator_.checkTypeExpected(AVRO_LONG); - uint64_t encoded = readVarInt(); - val = decodeZigzag64(encoded); - } - - void readValue(float &val) { - validator_.checkTypeExpected(AVRO_FLOAT); - union { - float f; - uint32_t i; - } v; - reader_.read(v.i); - val = v.f; - } - - void readValue(double &val) { - validator_.checkTypeExpected(AVRO_DOUBLE); - union { - double d; - uint64_t i; - } v; - reader_.read(v.i); - val = v.d; - } - - void readValue(std::string &val) { - validator_.checkTypeExpected(AVRO_STRING); - size_t size = static_cast<size_t>(readSize()); - reader_.read(val, size); - } - - void readBytes(std::vector<uint8_t> &val) { - validator_.checkTypeExpected(AVRO_BYTES); - size_t size = static_cast<size_t>(readSize()); - val.resize(size); - reader_.read(reinterpret_cast<char *>(val.data()), size); - } - - void readFixed(uint8_t *val, size_t size) { - validator_.checkFixedSizeExpected(size); - reader_.read(reinterpret_cast<char *>(val), size); - } - - template <size_t N> - void readFixed(uint8_t (&val)[N]) { - this->readFixed(val, N); - } - - template <size_t N> - void readFixed(std::array<uint8_t, N> &val) { - this->readFixed(val.data(), N); - } - - void readRecord() { - validator_.checkTypeExpected(AVRO_RECORD); - validator_.checkTypeExpected(AVRO_LONG); - validator_.setCount(1); - } - - void readRecordEnd() { - validator_.checkTypeExpected(AVRO_RECORD); - validator_.checkTypeExpected(AVRO_LONG); - validator_.setCount(0); - } - - int64_t readArrayBlockSize() { - validator_.checkTypeExpected(AVRO_ARRAY); - return readCount(); - } - - int64_t readUnion() { - validator_.checkTypeExpected(AVRO_UNION); - return readCount(); - } - - int64_t readEnum() { - validator_.checkTypeExpected(AVRO_ENUM); - return readCount(); - } - - int64_t readMapBlockSize() { - validator_.checkTypeExpected(AVRO_MAP); - return readCount(); - } - - Type nextType() const { - return validator_.nextTypeExpected(); - } - - bool currentRecordName(std::string &name) const { - return validator_.getCurrentRecordName(name); - } - - bool nextFieldName(std::string &name) const { - return validator_.getNextFieldName(name); - } - - private: - - uint64_t readVarInt() { - uint64_t encoded = 0; - uint8_t val = 0; - int shift = 0; - do { - reader_.read(val); - uint64_t newbits = static_cast<uint64_t>(val & 0x7f) << shift; - encoded |= newbits; - shift += 7; - } while (val & 0x80); - - return encoded; - } - - int64_t readSize() { - uint64_t encoded = readVarInt(); - int64_t size = decodeZigzag64(encoded); - return size; - } - - int64_t readCount() { - validator_.checkTypeExpected(AVRO_LONG); - int64_t count = readSize(); - validator_.setCount(count); - return count; - } - - ValidatorType validator_; - BufferReader reader_; - -}; - -typedef ReaderImpl<NullValidator> Reader; -typedef ReaderImpl<Validator> ValidatingReader; - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Reader_hh__ +#define avro_Reader_hh__ + +#include <stdint.h> +#include <vector> +#include <array> +#include <boost/noncopyable.hpp> + +#include "Config.hh" +#include "Zigzag.hh" +#include "Types.hh" +#include "Validator.hh" +#include "buffer/BufferReader.hh" + +namespace avro { + +/// +/// Parses from an avro encoding to the requested type. Assumes the next item +/// in the avro binary data is the expected type. +/// + +template<class ValidatorType> +class ReaderImpl : private boost::noncopyable +{ + + public: + + explicit ReaderImpl(const InputBuffer &buffer) : + reader_(buffer) + {} + + ReaderImpl(const ValidSchema &schema, const InputBuffer &buffer) : + validator_(schema), + reader_(buffer) + {} + + void readValue(Null &) { + validator_.checkTypeExpected(AVRO_NULL); + } + + void readValue(bool &val) { + validator_.checkTypeExpected(AVRO_BOOL); + uint8_t ival = 0; + reader_.read(ival); + val = (ival != 0); + } + + void readValue(int32_t &val) { + validator_.checkTypeExpected(AVRO_INT); + uint32_t encoded = static_cast<uint32_t>(readVarInt()); + val = decodeZigzag32(encoded); + } + + void readValue(int64_t &val) { + validator_.checkTypeExpected(AVRO_LONG); + uint64_t encoded = readVarInt(); + val = decodeZigzag64(encoded); + } + + void readValue(float &val) { + validator_.checkTypeExpected(AVRO_FLOAT); + union { + float f; + uint32_t i; + } v; + reader_.read(v.i); + val = v.f; + } + + void readValue(double &val) { + validator_.checkTypeExpected(AVRO_DOUBLE); + union { + double d; + uint64_t i; + } v; + reader_.read(v.i); + val = v.d; + } + + void readValue(std::string &val) { + validator_.checkTypeExpected(AVRO_STRING); + size_t size = static_cast<size_t>(readSize()); + reader_.read(val, size); + } + + void readBytes(std::vector<uint8_t> &val) { + validator_.checkTypeExpected(AVRO_BYTES); + size_t size = static_cast<size_t>(readSize()); + val.resize(size); + reader_.read(reinterpret_cast<char *>(val.data()), size); + } + + void readFixed(uint8_t *val, size_t size) { + validator_.checkFixedSizeExpected(size); + reader_.read(reinterpret_cast<char *>(val), size); + } + + template <size_t N> + void readFixed(uint8_t (&val)[N]) { + this->readFixed(val, N); + } + + template <size_t N> + void readFixed(std::array<uint8_t, N> &val) { + this->readFixed(val.data(), N); + } + + void readRecord() { + validator_.checkTypeExpected(AVRO_RECORD); + validator_.checkTypeExpected(AVRO_LONG); + validator_.setCount(1); + } + + void readRecordEnd() { + validator_.checkTypeExpected(AVRO_RECORD); + validator_.checkTypeExpected(AVRO_LONG); + validator_.setCount(0); + } + + int64_t readArrayBlockSize() { + validator_.checkTypeExpected(AVRO_ARRAY); + return readCount(); + } + + int64_t readUnion() { + validator_.checkTypeExpected(AVRO_UNION); + return readCount(); + } + + int64_t readEnum() { + validator_.checkTypeExpected(AVRO_ENUM); + return readCount(); + } + + int64_t readMapBlockSize() { + validator_.checkTypeExpected(AVRO_MAP); + return readCount(); + } + + Type nextType() const { + return validator_.nextTypeExpected(); + } + + bool currentRecordName(std::string &name) const { + return validator_.getCurrentRecordName(name); + } + + bool nextFieldName(std::string &name) const { + return validator_.getNextFieldName(name); + } + + private: + + uint64_t readVarInt() { + uint64_t encoded = 0; + uint8_t val = 0; + int shift = 0; + do { + reader_.read(val); + uint64_t newbits = static_cast<uint64_t>(val & 0x7f) << shift; + encoded |= newbits; + shift += 7; + } while (val & 0x80); + + return encoded; + } + + int64_t readSize() { + uint64_t encoded = readVarInt(); + int64_t size = decodeZigzag64(encoded); + return size; + } + + int64_t readCount() { + validator_.checkTypeExpected(AVRO_LONG); + int64_t count = readSize(); + validator_.setCount(count); + return count; + } + + ValidatorType validator_; + BufferReader reader_; + +}; + +typedef ReaderImpl<NullValidator> Reader; +typedef ReaderImpl<Validator> ValidatingReader; + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Resolver.hh b/contrib/libs/apache/avro/api/Resolver.hh index 91c6de1cb7..a0ffcbeac7 100644 --- a/contrib/libs/apache/avro/api/Resolver.hh +++ b/contrib/libs/apache/avro/api/Resolver.hh @@ -1,57 +1,57 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Resolver_hh__ -#define avro_Resolver_hh__ - -#include <memory> -#include <boost/noncopyable.hpp> -#include <stdint.h> - -#include "Config.hh" -#include "Reader.hh" - -/// \file Resolver.hh -/// - -namespace avro { - -class ValidSchema; -class Layout; - -class AVRO_DECL Resolver : private boost::noncopyable -{ - - public: - - virtual void parse(Reader &reader, uint8_t *address) const = 0; - virtual ~Resolver() {} - -}; - -std::unique_ptr<Resolver> constructResolver( - const ValidSchema &rwriterSchema, - const ValidSchema &readerSchema, - const Layout &readerLayout - ); - - - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Resolver_hh__ +#define avro_Resolver_hh__ + +#include <memory> +#include <boost/noncopyable.hpp> +#include <stdint.h> + +#include "Config.hh" +#include "Reader.hh" + +/// \file Resolver.hh +/// + +namespace avro { + +class ValidSchema; +class Layout; + +class AVRO_DECL Resolver : private boost::noncopyable +{ + + public: + + virtual void parse(Reader &reader, uint8_t *address) const = 0; + virtual ~Resolver() {} + +}; + +std::unique_ptr<Resolver> constructResolver( + const ValidSchema &rwriterSchema, + const ValidSchema &readerSchema, + const Layout &readerLayout + ); + + + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/ResolverSchema.hh b/contrib/libs/apache/avro/api/ResolverSchema.hh index f9420cdfa5..9048a22b9a 100644 --- a/contrib/libs/apache/avro/api/ResolverSchema.hh +++ b/contrib/libs/apache/avro/api/ResolverSchema.hh @@ -1,56 +1,56 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_ResolverSchema_hh__ -#define avro_ResolverSchema_hh__ - -#include <boost/noncopyable.hpp> -#include <stdint.h> -#include <memory> - -#include "Config.hh" -#include "Reader.hh" - -/// \file ResolverSchema.hh -/// - -namespace avro { - -class ValidSchema; -class Layout; -class Resolver; - -class AVRO_DECL ResolverSchema { - - public: - - ResolverSchema(const ValidSchema &writer, const ValidSchema &reader, const Layout &readerLayout); - - private: - - friend class ResolvingReader; - - void parse(Reader &reader, uint8_t *address); - - std::shared_ptr<Resolver> resolver_; - -}; - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_ResolverSchema_hh__ +#define avro_ResolverSchema_hh__ + +#include <boost/noncopyable.hpp> +#include <stdint.h> +#include <memory> + +#include "Config.hh" +#include "Reader.hh" + +/// \file ResolverSchema.hh +/// + +namespace avro { + +class ValidSchema; +class Layout; +class Resolver; + +class AVRO_DECL ResolverSchema { + + public: + + ResolverSchema(const ValidSchema &writer, const ValidSchema &reader, const Layout &readerLayout); + + private: + + friend class ResolvingReader; + + void parse(Reader &reader, uint8_t *address); + + std::shared_ptr<Resolver> resolver_; + +}; + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/ResolvingReader.hh b/contrib/libs/apache/avro/api/ResolvingReader.hh index 7588e5e08d..806e64da56 100644 --- a/contrib/libs/apache/avro/api/ResolvingReader.hh +++ b/contrib/libs/apache/avro/api/ResolvingReader.hh @@ -1,54 +1,54 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_ResolvingReader_hh__ -#define avro_ResolvingReader_hh__ - -#include <stdint.h> -#include <boost/noncopyable.hpp> - -#include "Config.hh" -#include "ResolverSchema.hh" -#include "Reader.hh" - -namespace avro { - -class AVRO_DECL ResolvingReader : private boost::noncopyable -{ - - public: - - ResolvingReader(const ResolverSchema &schema, const InputBuffer &in) : - reader_(in), - schema_(schema) - {} - - template<typename T> - void parse(T &object) { - schema_.parse(reader_, reinterpret_cast<uint8_t *>(&object)); - } - - private: - - Reader reader_; - ResolverSchema schema_; -}; - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_ResolvingReader_hh__ +#define avro_ResolvingReader_hh__ + +#include <stdint.h> +#include <boost/noncopyable.hpp> + +#include "Config.hh" +#include "ResolverSchema.hh" +#include "Reader.hh" + +namespace avro { + +class AVRO_DECL ResolvingReader : private boost::noncopyable +{ + + public: + + ResolvingReader(const ResolverSchema &schema, const InputBuffer &in) : + reader_(in), + schema_(schema) + {} + + template<typename T> + void parse(T &object) { + schema_.parse(reader_, reinterpret_cast<uint8_t *>(&object)); + } + + private: + + Reader reader_; + ResolverSchema schema_; +}; + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Schema.hh b/contrib/libs/apache/avro/api/Schema.hh index ee0e79c6eb..b8ad92c825 100644 --- a/contrib/libs/apache/avro/api/Schema.hh +++ b/contrib/libs/apache/avro/api/Schema.hh @@ -1,146 +1,146 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Schema_hh__ -#define avro_Schema_hh__ - -#include "Config.hh" -#include "NodeImpl.hh" -#include <string> - -/// \file -/// -/// Schemas for representing all the avro types. The compound schema objects -/// allow composition from other schemas. -/// - -namespace avro { - - -/// The root Schema object is a base class. Nobody constructs this class directly. - -class AVRO_DECL Schema { -public: - - virtual ~Schema(); - - Type type() const { - return node_->type(); - } - - const NodePtr &root() const { - return node_; - } - - NodePtr &root() { - return node_; - } - - protected: - Schema(); - explicit Schema(const NodePtr &node); - explicit Schema(Node *node); - - NodePtr node_; -}; - -class AVRO_DECL NullSchema : public Schema { -public: - NullSchema(): Schema(new NodePrimitive(AVRO_NULL)) {} -}; - -class AVRO_DECL BoolSchema : public Schema { -public: - BoolSchema(): Schema(new NodePrimitive(AVRO_BOOL)) {} -}; - -class AVRO_DECL IntSchema : public Schema { -public: - IntSchema(): Schema(new NodePrimitive(AVRO_INT)) {} -}; - -class AVRO_DECL LongSchema : public Schema { -public: - LongSchema(): Schema(new NodePrimitive(AVRO_LONG)) {} -}; - -class AVRO_DECL FloatSchema : public Schema { -public: - FloatSchema(): Schema(new NodePrimitive(AVRO_FLOAT)) {} -}; - -class AVRO_DECL DoubleSchema : public Schema { -public: - DoubleSchema(): Schema(new NodePrimitive(AVRO_DOUBLE)) {} -}; - -class AVRO_DECL StringSchema : public Schema { -public: - StringSchema(): Schema(new NodePrimitive(AVRO_STRING)) {} -}; - -class AVRO_DECL BytesSchema : public Schema { -public: - BytesSchema(): Schema(new NodePrimitive(AVRO_BYTES)) {} -}; - -class AVRO_DECL RecordSchema : public Schema { -public: - RecordSchema(const std::string &name); - void addField(const std::string &name, const Schema &fieldSchema); - - std::string getDoc() const; - void setDoc(const std::string &); -}; - -class AVRO_DECL EnumSchema : public Schema { -public: - EnumSchema(const std::string &name); - void addSymbol(const std::string &symbol); -}; - -class AVRO_DECL ArraySchema : public Schema { -public: - ArraySchema(const Schema &itemsSchema); - ArraySchema(const ArraySchema &itemsSchema); -}; - -class AVRO_DECL MapSchema : public Schema { -public: - MapSchema(const Schema &valuesSchema); - MapSchema(const MapSchema &itemsSchema); -}; - -class AVRO_DECL UnionSchema : public Schema { -public: - UnionSchema(); - void addType(const Schema &typeSchema); -}; - -class AVRO_DECL FixedSchema : public Schema { -public: - FixedSchema(int size, const std::string &name); -}; - -class AVRO_DECL SymbolicSchema : public Schema { -public: - SymbolicSchema(const Name& name, const NodePtr& link); -}; -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Schema_hh__ +#define avro_Schema_hh__ + +#include "Config.hh" +#include "NodeImpl.hh" +#include <string> + +/// \file +/// +/// Schemas for representing all the avro types. The compound schema objects +/// allow composition from other schemas. +/// + +namespace avro { + + +/// The root Schema object is a base class. Nobody constructs this class directly. + +class AVRO_DECL Schema { +public: + + virtual ~Schema(); + + Type type() const { + return node_->type(); + } + + const NodePtr &root() const { + return node_; + } + + NodePtr &root() { + return node_; + } + + protected: + Schema(); + explicit Schema(const NodePtr &node); + explicit Schema(Node *node); + + NodePtr node_; +}; + +class AVRO_DECL NullSchema : public Schema { +public: + NullSchema(): Schema(new NodePrimitive(AVRO_NULL)) {} +}; + +class AVRO_DECL BoolSchema : public Schema { +public: + BoolSchema(): Schema(new NodePrimitive(AVRO_BOOL)) {} +}; + +class AVRO_DECL IntSchema : public Schema { +public: + IntSchema(): Schema(new NodePrimitive(AVRO_INT)) {} +}; + +class AVRO_DECL LongSchema : public Schema { +public: + LongSchema(): Schema(new NodePrimitive(AVRO_LONG)) {} +}; + +class AVRO_DECL FloatSchema : public Schema { +public: + FloatSchema(): Schema(new NodePrimitive(AVRO_FLOAT)) {} +}; + +class AVRO_DECL DoubleSchema : public Schema { +public: + DoubleSchema(): Schema(new NodePrimitive(AVRO_DOUBLE)) {} +}; + +class AVRO_DECL StringSchema : public Schema { +public: + StringSchema(): Schema(new NodePrimitive(AVRO_STRING)) {} +}; + +class AVRO_DECL BytesSchema : public Schema { +public: + BytesSchema(): Schema(new NodePrimitive(AVRO_BYTES)) {} +}; + +class AVRO_DECL RecordSchema : public Schema { +public: + RecordSchema(const std::string &name); + void addField(const std::string &name, const Schema &fieldSchema); + + std::string getDoc() const; + void setDoc(const std::string &); +}; + +class AVRO_DECL EnumSchema : public Schema { +public: + EnumSchema(const std::string &name); + void addSymbol(const std::string &symbol); +}; + +class AVRO_DECL ArraySchema : public Schema { +public: + ArraySchema(const Schema &itemsSchema); + ArraySchema(const ArraySchema &itemsSchema); +}; + +class AVRO_DECL MapSchema : public Schema { +public: + MapSchema(const Schema &valuesSchema); + MapSchema(const MapSchema &itemsSchema); +}; + +class AVRO_DECL UnionSchema : public Schema { +public: + UnionSchema(); + void addType(const Schema &typeSchema); +}; + +class AVRO_DECL FixedSchema : public Schema { +public: + FixedSchema(int size, const std::string &name); +}; + +class AVRO_DECL SymbolicSchema : public Schema { +public: + SymbolicSchema(const Name& name, const NodePtr& link); +}; +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/SchemaResolution.hh b/contrib/libs/apache/avro/api/SchemaResolution.hh index c3a39e1237..765347d9de 100644 --- a/contrib/libs/apache/avro/api/SchemaResolution.hh +++ b/contrib/libs/apache/avro/api/SchemaResolution.hh @@ -1,55 +1,55 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_SchemaResolution_hh__ -#define avro_SchemaResolution_hh__ - -#include "Config.hh" - -namespace avro { - - -enum SchemaResolution { - - /// The schemas definitely do not match - - RESOLVE_NO_MATCH, - - /// The schemas match at a cursory level - /// - /// For records and enums, this means the name is the same, but it does not - /// necessarily mean that every symbol or field is an exact match. - - RESOLVE_MATCH, - - /// For primitives, the matching may occur if the type is promotable. This means that the - /// writer matches reader if the writer's type is promoted the specified type. - - //@{ - - RESOLVE_PROMOTABLE_TO_LONG, - RESOLVE_PROMOTABLE_TO_FLOAT, - RESOLVE_PROMOTABLE_TO_DOUBLE, - - //@} - -}; - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_SchemaResolution_hh__ +#define avro_SchemaResolution_hh__ + +#include "Config.hh" + +namespace avro { + + +enum SchemaResolution { + + /// The schemas definitely do not match + + RESOLVE_NO_MATCH, + + /// The schemas match at a cursory level + /// + /// For records and enums, this means the name is the same, but it does not + /// necessarily mean that every symbol or field is an exact match. + + RESOLVE_MATCH, + + /// For primitives, the matching may occur if the type is promotable. This means that the + /// writer matches reader if the writer's type is promoted the specified type. + + //@{ + + RESOLVE_PROMOTABLE_TO_LONG, + RESOLVE_PROMOTABLE_TO_FLOAT, + RESOLVE_PROMOTABLE_TO_DOUBLE, + + //@} + +}; + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Serializer.hh b/contrib/libs/apache/avro/api/Serializer.hh index 3cc15b5a95..15a8878586 100644 --- a/contrib/libs/apache/avro/api/Serializer.hh +++ b/contrib/libs/apache/avro/api/Serializer.hh @@ -1,135 +1,135 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Serializer_hh__ -#define avro_Serializer_hh__ - -#include <array> -#include <boost/noncopyable.hpp> - -#include "Config.hh" -#include "Writer.hh" - -namespace avro { - -/// Class that wraps a Writer or ValidatingWriter with an interface that uses -/// explicit write* names instead of writeValue - -template<class Writer> -class Serializer : private boost::noncopyable -{ - - public: - - /// Constructor only works with Writer - explicit Serializer() : - writer_() - {} - - /// Constructor only works with ValidatingWriter - Serializer(const ValidSchema &schema) : - writer_(schema) - {} - - void writeNull() { - writer_.writeValue(Null()); - } - - void writeBool(bool val) { - writer_.writeValue(val); - } - - void writeInt(int32_t val) { - writer_.writeValue(val); - } - - void writeLong(int64_t val) { - writer_.writeValue(val); - } - - void writeFloat(float val) { - writer_.writeValue(val); - } - - void writeDouble(double val) { - writer_.writeValue(val); - } - - void writeBytes(const void *val, size_t size) { - writer_.writeBytes(val); - } - - template <size_t N> - void writeFixed(const uint8_t (&val)[N]) { - writer_.writeFixed(val); - } - - template <size_t N> - void writeFixed(const std::array<uint8_t, N> &val) { - writer_.writeFixed(val); - } - - void writeString(const std::string &val) { - writer_.writeValue(val); - } - - void writeRecord() { - writer_.writeRecord(); - } - - void writeRecordEnd() { - writer_.writeRecordEnd(); - } - - void writeArrayBlock(int64_t size) { - writer_.writeArrayBlock(size); - } - - void writeArrayEnd() { - writer_.writeArrayEnd(); - } - - void writeMapBlock(int64_t size) { - writer_.writeMapBlock(size); - } - - void writeMapEnd() { - writer_.writeMapEnd(); - } - - void writeUnion(int64_t choice) { - writer_.writeUnion(choice); - } - - void writeEnum(int64_t choice) { - writer_.writeEnum(choice); - } - - InputBuffer buffer() const { - return writer_.buffer(); - } - - private: - - Writer writer_; - -}; - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Serializer_hh__ +#define avro_Serializer_hh__ + +#include <array> +#include <boost/noncopyable.hpp> + +#include "Config.hh" +#include "Writer.hh" + +namespace avro { + +/// Class that wraps a Writer or ValidatingWriter with an interface that uses +/// explicit write* names instead of writeValue + +template<class Writer> +class Serializer : private boost::noncopyable +{ + + public: + + /// Constructor only works with Writer + explicit Serializer() : + writer_() + {} + + /// Constructor only works with ValidatingWriter + Serializer(const ValidSchema &schema) : + writer_(schema) + {} + + void writeNull() { + writer_.writeValue(Null()); + } + + void writeBool(bool val) { + writer_.writeValue(val); + } + + void writeInt(int32_t val) { + writer_.writeValue(val); + } + + void writeLong(int64_t val) { + writer_.writeValue(val); + } + + void writeFloat(float val) { + writer_.writeValue(val); + } + + void writeDouble(double val) { + writer_.writeValue(val); + } + + void writeBytes(const void *val, size_t size) { + writer_.writeBytes(val); + } + + template <size_t N> + void writeFixed(const uint8_t (&val)[N]) { + writer_.writeFixed(val); + } + + template <size_t N> + void writeFixed(const std::array<uint8_t, N> &val) { + writer_.writeFixed(val); + } + + void writeString(const std::string &val) { + writer_.writeValue(val); + } + + void writeRecord() { + writer_.writeRecord(); + } + + void writeRecordEnd() { + writer_.writeRecordEnd(); + } + + void writeArrayBlock(int64_t size) { + writer_.writeArrayBlock(size); + } + + void writeArrayEnd() { + writer_.writeArrayEnd(); + } + + void writeMapBlock(int64_t size) { + writer_.writeMapBlock(size); + } + + void writeMapEnd() { + writer_.writeMapEnd(); + } + + void writeUnion(int64_t choice) { + writer_.writeUnion(choice); + } + + void writeEnum(int64_t choice) { + writer_.writeEnum(choice); + } + + InputBuffer buffer() const { + return writer_.buffer(); + } + + private: + + Writer writer_; + +}; + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Specific.hh b/contrib/libs/apache/avro/api/Specific.hh index 8572ffae48..53741be480 100644 --- a/contrib/libs/apache/avro/api/Specific.hh +++ b/contrib/libs/apache/avro/api/Specific.hh @@ -1,348 +1,348 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Codec_hh__ -#define avro_Codec_hh__ - -#include <string> -#include <vector> -#include <map> -#include <algorithm> -#include "array" - -#include "boost/blank.hpp" - -#include "AvroTraits.hh" -#include "Config.hh" -#include "Encoder.hh" -#include "Decoder.hh" - -/** - * A bunch of templates and specializations for encoding and decoding - * specific types. - * - * Primitive AVRO types BOOLEAN, INT, LONG, FLOAT, DOUBLE, STRING and BYTES - * get decoded to and encoded from C++ types bool, int32_t, int64_t, float, - * double, std::string and std::vector<uint8_t> respectively. In addition, - * std::vector<T> for aribtrary type T gets encoded as an Avro array of T. - * Similarly, std::map<std::string, T> for arbitrary type T gets encoded - * as an Avro map with value type T. - * - * Users can have their custom types encoded/decoded by specializing - * avro::codec_traits class for their types. - */ -namespace avro { - -typedef boost::blank null; - -template <typename T> void encode(Encoder& e, const T& t); -template <typename T> void decode(Decoder& d, T& t); - -/** - * Codec_traits tells avro how to encode and decode an object of given type. - * - * The class is expected to have two static methods: - * \li static void encode(Encoder& e, const T& value); - * \li static void decode(Decoder& e, T& value); - * The default is empty. - */ -template <typename T> -struct codec_traits; - -/** - * codec_traits for Avro boolean. - */ -template <> struct codec_traits<bool> { - /** - * Encodes a given value. - */ - static void encode(Encoder& e, bool b) { - e.encodeBool(b); - } - - /** - * Decodes into a given value. - */ - static void decode(Decoder& d, bool& b) { - b = d.decodeBool(); - } -}; - -/** - * codec_traits for Avro int. - */ -template <> struct codec_traits<int32_t> { - /** - * Encodes a given value. - */ - static void encode(Encoder& e, int32_t i) { - e.encodeInt(i); - } - - /** - * Decodes into a given value. - */ - static void decode(Decoder& d, int32_t& i) { - i = d.decodeInt(); - } -}; - -/** - * codec_traits for Avro long. - */ -template <> struct codec_traits<int64_t> { - /** - * Encodes a given value. - */ - static void encode(Encoder& e, int64_t l) { - e.encodeLong(l); - } - - /** - * Decodes into a given value. - */ - static void decode(Decoder& d, int64_t& l) { - l = d.decodeLong(); - } -}; - -/** - * codec_traits for Avro float. - */ -template <> struct codec_traits<float> { - /** - * Encodes a given value. - */ - static void encode(Encoder& e, float f) { - e.encodeFloat(f); - } - - /** - * Decodes into a given value. - */ - static void decode(Decoder& d, float& f) { - f = d.decodeFloat(); - } -}; - -/** - * codec_traits for Avro double. - */ -template <> struct codec_traits<double> { - /** - * Encodes a given value. - */ - static void encode(Encoder& e, double d) { - e.encodeDouble(d); - } - - /** - * Decodes into a given value. - */ - static void decode(Decoder& d, double& dbl) { - dbl = d.decodeDouble(); - } -}; - -/** - * codec_traits for Avro string. - */ -template <> struct codec_traits<std::string> { - /** - * Encodes a given value. - */ - static void encode(Encoder& e, const std::string& s) { - e.encodeString(s); - } - - /** - * Decodes into a given value. - */ - static void decode(Decoder& d, std::string& s) { - s = d.decodeString(); - } -}; - -/** - * codec_traits for Avro bytes. - */ -template <> struct codec_traits<std::vector<uint8_t> > { - /** - * Encodes a given value. - */ - static void encode(Encoder& e, const std::vector<uint8_t>& b) { - e.encodeBytes(b); - } - - /** - * Decodes into a given value. - */ - static void decode(Decoder& d, std::vector<uint8_t>& s) { - d.decodeBytes(s); - } -}; - -/** - * codec_traits for Avro fixed. - */ -template <size_t N> struct codec_traits<std::array<uint8_t, N> > { - /** - * Encodes a given value. - */ - static void encode(Encoder& e, const std::array<uint8_t, N>& b) { - e.encodeFixed(b.data(), N); - } - - /** - * Decodes into a given value. - */ - static void decode(Decoder& d, std::array<uint8_t, N>& s) { - std::vector<uint8_t> v(N); - d.decodeFixed(N, v); - std::copy(v.data(), v.data() + N, s.data()); - } -}; - -/** - * codec_traits for Avro arrays. - */ -template <typename T> struct codec_traits<std::vector<T> > { - /** - * Encodes a given value. - */ - static void encode(Encoder& e, const std::vector<T>& b) { - e.arrayStart(); - if (! b.empty()) { - e.setItemCount(b.size()); - for (typename std::vector<T>::const_iterator it = b.begin(); - it != b.end(); ++it) { - e.startItem(); - avro::encode(e, *it); - } - } - e.arrayEnd(); - } - - /** - * Decodes into a given value. - */ - static void decode(Decoder& d, std::vector<T>& s) { - s.clear(); - for (size_t n = d.arrayStart(); n != 0; n = d.arrayNext()) { - for (size_t i = 0; i < n; ++i) { - T t; - avro::decode(d, t); - s.push_back(std::move(t)); - } - } - } -}; - -typedef codec_traits<std::vector<bool>::const_reference> bool_codec_traits; - -template <> struct codec_traits<std::conditional<avro::is_not_defined<bool_codec_traits>::value, - std::vector<bool>::const_reference, void>::type> { - /** - * Encodes a given value. - */ - static void encode(Encoder& e, std::vector<bool>::const_reference b) { - e.encodeBool(b); - } -}; - -/** - * codec_traits for Avro maps. - */ -template <typename T> struct codec_traits<std::map<std::string, T> > { - /** - * Encodes a given value. - */ - static void encode(Encoder& e, const std::map<std::string, T>& b) { - e.mapStart(); - if (! b.empty()) { - e.setItemCount(b.size()); - for (typename std::map<std::string, T>::const_iterator - it = b.begin(); - it != b.end(); ++it) { - e.startItem(); - avro::encode(e, it->first); - avro::encode(e, it->second); - } - } - e.mapEnd(); - } - - /** - * Decodes into a given value. - */ - static void decode(Decoder& d, std::map<std::string, T>& s) { - s.clear(); - for (size_t n = d.mapStart(); n != 0; n = d.mapNext()) { - for (size_t i = 0; i < n; ++i) { - std::string k; - avro::decode(d, k); - T& t = s[std::move(k)]; - avro::decode(d, t); - } - } - } -}; - -/** -* codec_traits for Avro null. -*/ -template <> struct codec_traits<avro::null> { - /** - * Encodes a given value. - */ - static void encode(Encoder& e, const avro::null&) { - e.encodeNull(); - } - - /** - * Decodes into a given value. - */ - static void decode(Decoder& d, avro::null&) { - d.decodeNull(); - } -}; - - - -/** - * Generic encoder function that makes use of the codec_traits. - */ -template <typename T> -void encode(Encoder& e, const T& t) { - codec_traits<T>::encode(e, t); -} - -/** - * Generic decoder function that makes use of the codec_traits. - */ -template <typename T> -void decode(Decoder& d, T& t) { - codec_traits<T>::decode(d, t); -} - -} // namespace avro - -#endif // avro_Codec_hh__ - - - +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Codec_hh__ +#define avro_Codec_hh__ + +#include <string> +#include <vector> +#include <map> +#include <algorithm> +#include "array" + +#include "boost/blank.hpp" + +#include "AvroTraits.hh" +#include "Config.hh" +#include "Encoder.hh" +#include "Decoder.hh" + +/** + * A bunch of templates and specializations for encoding and decoding + * specific types. + * + * Primitive AVRO types BOOLEAN, INT, LONG, FLOAT, DOUBLE, STRING and BYTES + * get decoded to and encoded from C++ types bool, int32_t, int64_t, float, + * double, std::string and std::vector<uint8_t> respectively. In addition, + * std::vector<T> for aribtrary type T gets encoded as an Avro array of T. + * Similarly, std::map<std::string, T> for arbitrary type T gets encoded + * as an Avro map with value type T. + * + * Users can have their custom types encoded/decoded by specializing + * avro::codec_traits class for their types. + */ +namespace avro { + +typedef boost::blank null; + +template <typename T> void encode(Encoder& e, const T& t); +template <typename T> void decode(Decoder& d, T& t); + +/** + * Codec_traits tells avro how to encode and decode an object of given type. + * + * The class is expected to have two static methods: + * \li static void encode(Encoder& e, const T& value); + * \li static void decode(Decoder& e, T& value); + * The default is empty. + */ +template <typename T> +struct codec_traits; + +/** + * codec_traits for Avro boolean. + */ +template <> struct codec_traits<bool> { + /** + * Encodes a given value. + */ + static void encode(Encoder& e, bool b) { + e.encodeBool(b); + } + + /** + * Decodes into a given value. + */ + static void decode(Decoder& d, bool& b) { + b = d.decodeBool(); + } +}; + +/** + * codec_traits for Avro int. + */ +template <> struct codec_traits<int32_t> { + /** + * Encodes a given value. + */ + static void encode(Encoder& e, int32_t i) { + e.encodeInt(i); + } + + /** + * Decodes into a given value. + */ + static void decode(Decoder& d, int32_t& i) { + i = d.decodeInt(); + } +}; + +/** + * codec_traits for Avro long. + */ +template <> struct codec_traits<int64_t> { + /** + * Encodes a given value. + */ + static void encode(Encoder& e, int64_t l) { + e.encodeLong(l); + } + + /** + * Decodes into a given value. + */ + static void decode(Decoder& d, int64_t& l) { + l = d.decodeLong(); + } +}; + +/** + * codec_traits for Avro float. + */ +template <> struct codec_traits<float> { + /** + * Encodes a given value. + */ + static void encode(Encoder& e, float f) { + e.encodeFloat(f); + } + + /** + * Decodes into a given value. + */ + static void decode(Decoder& d, float& f) { + f = d.decodeFloat(); + } +}; + +/** + * codec_traits for Avro double. + */ +template <> struct codec_traits<double> { + /** + * Encodes a given value. + */ + static void encode(Encoder& e, double d) { + e.encodeDouble(d); + } + + /** + * Decodes into a given value. + */ + static void decode(Decoder& d, double& dbl) { + dbl = d.decodeDouble(); + } +}; + +/** + * codec_traits for Avro string. + */ +template <> struct codec_traits<std::string> { + /** + * Encodes a given value. + */ + static void encode(Encoder& e, const std::string& s) { + e.encodeString(s); + } + + /** + * Decodes into a given value. + */ + static void decode(Decoder& d, std::string& s) { + s = d.decodeString(); + } +}; + +/** + * codec_traits for Avro bytes. + */ +template <> struct codec_traits<std::vector<uint8_t> > { + /** + * Encodes a given value. + */ + static void encode(Encoder& e, const std::vector<uint8_t>& b) { + e.encodeBytes(b); + } + + /** + * Decodes into a given value. + */ + static void decode(Decoder& d, std::vector<uint8_t>& s) { + d.decodeBytes(s); + } +}; + +/** + * codec_traits for Avro fixed. + */ +template <size_t N> struct codec_traits<std::array<uint8_t, N> > { + /** + * Encodes a given value. + */ + static void encode(Encoder& e, const std::array<uint8_t, N>& b) { + e.encodeFixed(b.data(), N); + } + + /** + * Decodes into a given value. + */ + static void decode(Decoder& d, std::array<uint8_t, N>& s) { + std::vector<uint8_t> v(N); + d.decodeFixed(N, v); + std::copy(v.data(), v.data() + N, s.data()); + } +}; + +/** + * codec_traits for Avro arrays. + */ +template <typename T> struct codec_traits<std::vector<T> > { + /** + * Encodes a given value. + */ + static void encode(Encoder& e, const std::vector<T>& b) { + e.arrayStart(); + if (! b.empty()) { + e.setItemCount(b.size()); + for (typename std::vector<T>::const_iterator it = b.begin(); + it != b.end(); ++it) { + e.startItem(); + avro::encode(e, *it); + } + } + e.arrayEnd(); + } + + /** + * Decodes into a given value. + */ + static void decode(Decoder& d, std::vector<T>& s) { + s.clear(); + for (size_t n = d.arrayStart(); n != 0; n = d.arrayNext()) { + for (size_t i = 0; i < n; ++i) { + T t; + avro::decode(d, t); + s.push_back(std::move(t)); + } + } + } +}; + +typedef codec_traits<std::vector<bool>::const_reference> bool_codec_traits; + +template <> struct codec_traits<std::conditional<avro::is_not_defined<bool_codec_traits>::value, + std::vector<bool>::const_reference, void>::type> { + /** + * Encodes a given value. + */ + static void encode(Encoder& e, std::vector<bool>::const_reference b) { + e.encodeBool(b); + } +}; + +/** + * codec_traits for Avro maps. + */ +template <typename T> struct codec_traits<std::map<std::string, T> > { + /** + * Encodes a given value. + */ + static void encode(Encoder& e, const std::map<std::string, T>& b) { + e.mapStart(); + if (! b.empty()) { + e.setItemCount(b.size()); + for (typename std::map<std::string, T>::const_iterator + it = b.begin(); + it != b.end(); ++it) { + e.startItem(); + avro::encode(e, it->first); + avro::encode(e, it->second); + } + } + e.mapEnd(); + } + + /** + * Decodes into a given value. + */ + static void decode(Decoder& d, std::map<std::string, T>& s) { + s.clear(); + for (size_t n = d.mapStart(); n != 0; n = d.mapNext()) { + for (size_t i = 0; i < n; ++i) { + std::string k; + avro::decode(d, k); + T& t = s[std::move(k)]; + avro::decode(d, t); + } + } + } +}; + +/** +* codec_traits for Avro null. +*/ +template <> struct codec_traits<avro::null> { + /** + * Encodes a given value. + */ + static void encode(Encoder& e, const avro::null&) { + e.encodeNull(); + } + + /** + * Decodes into a given value. + */ + static void decode(Decoder& d, avro::null&) { + d.decodeNull(); + } +}; + + + +/** + * Generic encoder function that makes use of the codec_traits. + */ +template <typename T> +void encode(Encoder& e, const T& t) { + codec_traits<T>::encode(e, t); +} + +/** + * Generic decoder function that makes use of the codec_traits. + */ +template <typename T> +void decode(Decoder& d, T& t) { + codec_traits<T>::decode(d, t); +} + +} // namespace avro + +#endif // avro_Codec_hh__ + + + diff --git a/contrib/libs/apache/avro/api/Stream.hh b/contrib/libs/apache/avro/api/Stream.hh index 6693fd5c24..508cb03980 100644 --- a/contrib/libs/apache/avro/api/Stream.hh +++ b/contrib/libs/apache/avro/api/Stream.hh @@ -1,483 +1,483 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Stream_hh__ -#define avro_Stream_hh__ - -#include <memory> -#include <string.h> -#include <stdint.h> - -#include "boost/utility.hpp" - -#include "Config.hh" -#include "Exception.hh" - -namespace avro { - -/** - * A no-copy input stream. - */ -class AVRO_DECL InputStream : boost::noncopyable { -protected: - - /** - * An empty constuctor. - */ - InputStream() { } - -public: - /** - * Destructor. - */ - virtual ~InputStream() { } - - /** - * Returns some of available data. - * - * Returns true if some data is available, false if no more data is - * available or an error has occurred. - */ - virtual bool next(const uint8_t** data, size_t* len) = 0; - - /** - * "Returns" back some of the data to the stream. The returned - * data must be less than what was obtained in the last call to - * next(). - */ - virtual void backup(size_t len) = 0; - - /** - * Skips number of bytes specified by len. - */ - virtual void skip(size_t len) = 0; - - /** - * Returns the number of bytes read from this stream so far. - * All the bytes made available through next are considered - * to be used unless, retutned back using backup. - */ - virtual size_t byteCount() const = 0; -}; - -typedef std::unique_ptr<InputStream> InputStreamPtr; - -/** - * An InputStream which also supports seeking to a specific offset. - */ -class AVRO_DECL SeekableInputStream : public InputStream { -protected: - - /** - * An empty constuctor. - */ - SeekableInputStream() { } - -public: - /** - * Destructor. - */ - virtual ~SeekableInputStream() { } - - /** - * Seek to a specific position in the stream. This may invalidate pointers - * returned from next(). This will also reset byteCount() to the given - * position. - */ - virtual void seek(int64_t position) = 0; -}; - -typedef std::unique_ptr<SeekableInputStream> SeekableInputStreamPtr; - -/** - * A no-copy output stream. - */ -class AVRO_DECL OutputStream : boost::noncopyable { -protected: - - /** - * An empty constuctor. - */ - OutputStream() { } -public: - - /** - * Destructor. - */ - virtual ~OutputStream() { } - - /** - * Returns a buffer that can be written into. - * On successful return, data has the pointer to the buffer - * and len has the number of bytes available at data. - */ - virtual bool next(uint8_t** data, size_t* len) = 0; - - /** - * "Returns" back to the stream some of the buffer obtained - * from in the last call to next(). - */ - virtual void backup(size_t len) = 0; - - /** - * Number of bytes written so far into this stream. The whole buffer - * returned by next() is assumed to be written unless some of - * it was retutned using backup(). - */ - virtual uint64_t byteCount() const = 0; - - /** - * Flushes any data remaining in the buffer to the stream's underlying - * store, if any. - */ - virtual void flush() = 0; -}; - -typedef std::unique_ptr<OutputStream> OutputStreamPtr; - -/** - * Returns a new OutputStream, which grows in memory chunks of specified size. - */ -AVRO_DECL OutputStreamPtr memoryOutputStream(size_t chunkSize = 4 * 1024); - -/** - * Returns a new InputStream, with the data from the given byte array. - * It does not copy the data, the byte array should remain valid - * until the InputStream is used. - */ -AVRO_DECL InputStreamPtr memoryInputStream(const uint8_t* data, size_t len); - -/** - * Returns a new InputStream with the contents written into an - * outputstream. The output stream must have been returned by - * an earlier call to memoryOutputStream(). The contents for the new - * input stream are the snapshot of the outputstream. One can construct - * any number of memory input stream from a single memory output stream. - */ -AVRO_DECL InputStreamPtr memoryInputStream(const OutputStream& source); - -/** - * Returns the contents written so far into the output stream, which should - * be a memory output stream. That is it must have been returned by a pervious - * call to memoryOutputStream(). - */ -AVRO_DECL std::shared_ptr<std::vector<uint8_t> > snapshot(const OutputStream& source); - -/** - * Returns a new OutputStream whose contents would be stored in a file. - * Data is written in chunks of given buffer size. - * - * If there is a file with the given name, it is truncated and overwritten. - * If there is no file with the given name, it is created. - */ -AVRO_DECL OutputStreamPtr fileOutputStream(const char* filename, - size_t bufferSize = 8 * 1024); - -/** - * Returns a new InputStream whose contents come from the given file. - * Data is read in chunks of given buffer size. - */ -AVRO_DECL InputStreamPtr fileInputStream( - const char *filename, size_t bufferSize = 8 * 1024); -AVRO_DECL SeekableInputStreamPtr fileSeekableInputStream( - const char *filename, size_t bufferSize = 8 * 1024); - -/** - * Returns a new OutputStream whose contents will be sent to the given - * std::ostream. The std::ostream object should outlive the returned - * OutputStream. - */ -AVRO_DECL OutputStreamPtr ostreamOutputStream(std::ostream& os, - size_t bufferSize = 8 * 1024); - -/** - * Returns a new InputStream whose contents come from the given - * std::istream. The std::istream object should outlive the returned - * InputStream. - */ -AVRO_DECL InputStreamPtr istreamInputStream( - std::istream &in, size_t bufferSize = 8 * 1024); - -/** - * Returns a new InputStream whose contents come from the given - * std::istream. Use this instead of istreamInputStream if - * the istream does not support seekg (e.g. compressed streams). - * The returned InputStream would read off bytes instead of seeking. - * Of, course it has a performance penalty when reading instead of seeking; - * So, use this only when seekg does not work. - * The std::istream object should outlive the returned - * InputStream. - */ -AVRO_DECL InputStreamPtr nonSeekableIstreamInputStream( - std::istream& is, size_t bufferSize = 8 * 1024); - - -/** A convenience class for reading from an InputStream */ -struct StreamReader { - /** - * The underlying input stream. - */ - InputStream* in_; - - /** - * The next location to read from. - */ - const uint8_t* next_; - - /** - * One past the last valid location. - */ - const uint8_t* end_; - - /** - * Constructs an empty reader. - */ - StreamReader() : in_(0), next_(0), end_(0) { } - - /** - * Constructs a reader with the given underlying stream. - */ - StreamReader(InputStream& in) : in_(0), next_(0), end_(0) { reset(in); } - - /** - * Replaces the current input stream with the given one after backing up - * the original one if required. - */ - void reset(InputStream& is) { - if (in_ != 0 && end_ != next_) { - in_->backup(end_ - next_); - } - in_ = &is; - next_ = end_ = 0; - } - - /** - * Read just one byte from the underlying stream. If there are no - * more data, throws an exception. - */ - uint8_t read() { - if (next_ == end_) { - more(); - } - return *next_++; - } - - /** - * Reads the given number of bytes from the underlying stream. - * If there are not that many bytes, throws an exception. - */ - void readBytes(uint8_t* b, size_t n) { - while (n > 0) { - if (next_ == end_) { - more(); - } - size_t q = end_ - next_; - if (q > n) { - q = n; - } - ::memcpy(b, next_, q); - next_ += q; - b += q; - n -= q; - } - } - - /** - * Skips the given number of bytes. Of there are not so that many - * bytes, throws an exception. - */ - void skipBytes(size_t n) { - if (n > static_cast<size_t>(end_ - next_)) { - n -= end_ - next_; - next_ = end_; - in_->skip(n); - } else { - next_ += n; - } - } - - /** - * Get as many byes from the underlying stream as possible in a single - * chunk. - * \return true if some data could be obtained. False is no more - * data is available on the stream. - */ - bool fill() { - size_t n = 0; - while (in_->next(&next_, &n)) { - if (n != 0) { - end_ = next_ + n; - return true; - } - } - return false; - } - - /** - * Tries to get more data and if it cannot, throws an exception. - */ - void more() { - if (! fill()) { - throw Exception("EOF reached"); - } - } - - /** - * Returns true if and only if the end of stream is not reached. - */ - bool hasMore() { - return (next_ == end_) ? fill() : true; - } - - /** - * Returns unused bytes back to the underlying stream. - * If unRead is true the last byte read is also pushed back. - */ - void drain(bool unRead) { - if (unRead) { - --next_; - } - in_->backup(end_ - next_); - end_ = next_; - } -}; - -/** - * A convinience class to write data into an OutputStream. - */ -struct StreamWriter { - /** - * The underlying output stream for this writer. - */ - OutputStream* out_; - - /** - * The next location to write to. - */ - uint8_t* next_; - - /** - * One past the last location one can write to. - */ - uint8_t* end_; - - /** - * Constructs a writer with no underlying stream. - */ - StreamWriter() : out_(0), next_(0), end_(0) { } - - /** - * Constructs a new writer with the given underlying stream. - */ - StreamWriter(OutputStream& out) : out_(0), next_(0), end_(0) { reset(out); } - - /** - * Replaces the current underlying stream with a new one. - * If required, it backs up unused bytes in the previous stream. - */ - void reset(OutputStream& os) { - if (out_ != 0 && end_ != next_) { - out_->backup(end_ - next_); - } - out_ = &os; - next_ = end_; - } - - /** - * Writes a single byte. - */ - void write(uint8_t c) { - if (next_ == end_) { - more(); - } - *next_++ = c; - } - - /** - * Writes the specified number of bytes starting at \p b. - */ - void writeBytes(const uint8_t* b, size_t n) { - while (n > 0) { - if (next_ == end_) { - more(); - } - size_t q = end_ - next_; - if (q > n) { - q = n; - } - ::memcpy(next_, b, q); - next_ += q; - b += q; - n -= q; - } - } - - /** - * backs up upto the currently written data and flushes the - * underlying stream. - */ - void flush() { - if (next_ != end_) { - out_->backup(end_ - next_); - next_ = end_; - } - out_->flush(); - } - - /** - * Return the number of bytes written so far. For a meaningful - * result, call this after a flush(). - */ - int64_t byteCount() const { - return out_->byteCount(); - } - - /** - * Gets more space to write to. Throws an exception it cannot. - */ - void more() { - size_t n = 0; - while (out_->next(&next_, &n)) { - if (n != 0) { - end_ = next_ + n; - return; - } - } - throw Exception("EOF reached"); - } -}; - -/** - * A convenience function to copy all the contents of an input stream into - * an output stream. - */ -inline void copy(InputStream& in, OutputStream& out) -{ - const uint8_t *p = 0; - size_t n = 0; - StreamWriter w(out); - while (in.next(&p, &n)) { - w.writeBytes(p, n); - } - w.flush(); -} - -} // namespace avro -#endif - - +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Stream_hh__ +#define avro_Stream_hh__ + +#include <memory> +#include <string.h> +#include <stdint.h> + +#include "boost/utility.hpp" + +#include "Config.hh" +#include "Exception.hh" + +namespace avro { + +/** + * A no-copy input stream. + */ +class AVRO_DECL InputStream : boost::noncopyable { +protected: + + /** + * An empty constuctor. + */ + InputStream() { } + +public: + /** + * Destructor. + */ + virtual ~InputStream() { } + + /** + * Returns some of available data. + * + * Returns true if some data is available, false if no more data is + * available or an error has occurred. + */ + virtual bool next(const uint8_t** data, size_t* len) = 0; + + /** + * "Returns" back some of the data to the stream. The returned + * data must be less than what was obtained in the last call to + * next(). + */ + virtual void backup(size_t len) = 0; + + /** + * Skips number of bytes specified by len. + */ + virtual void skip(size_t len) = 0; + + /** + * Returns the number of bytes read from this stream so far. + * All the bytes made available through next are considered + * to be used unless, retutned back using backup. + */ + virtual size_t byteCount() const = 0; +}; + +typedef std::unique_ptr<InputStream> InputStreamPtr; + +/** + * An InputStream which also supports seeking to a specific offset. + */ +class AVRO_DECL SeekableInputStream : public InputStream { +protected: + + /** + * An empty constuctor. + */ + SeekableInputStream() { } + +public: + /** + * Destructor. + */ + virtual ~SeekableInputStream() { } + + /** + * Seek to a specific position in the stream. This may invalidate pointers + * returned from next(). This will also reset byteCount() to the given + * position. + */ + virtual void seek(int64_t position) = 0; +}; + +typedef std::unique_ptr<SeekableInputStream> SeekableInputStreamPtr; + +/** + * A no-copy output stream. + */ +class AVRO_DECL OutputStream : boost::noncopyable { +protected: + + /** + * An empty constuctor. + */ + OutputStream() { } +public: + + /** + * Destructor. + */ + virtual ~OutputStream() { } + + /** + * Returns a buffer that can be written into. + * On successful return, data has the pointer to the buffer + * and len has the number of bytes available at data. + */ + virtual bool next(uint8_t** data, size_t* len) = 0; + + /** + * "Returns" back to the stream some of the buffer obtained + * from in the last call to next(). + */ + virtual void backup(size_t len) = 0; + + /** + * Number of bytes written so far into this stream. The whole buffer + * returned by next() is assumed to be written unless some of + * it was retutned using backup(). + */ + virtual uint64_t byteCount() const = 0; + + /** + * Flushes any data remaining in the buffer to the stream's underlying + * store, if any. + */ + virtual void flush() = 0; +}; + +typedef std::unique_ptr<OutputStream> OutputStreamPtr; + +/** + * Returns a new OutputStream, which grows in memory chunks of specified size. + */ +AVRO_DECL OutputStreamPtr memoryOutputStream(size_t chunkSize = 4 * 1024); + +/** + * Returns a new InputStream, with the data from the given byte array. + * It does not copy the data, the byte array should remain valid + * until the InputStream is used. + */ +AVRO_DECL InputStreamPtr memoryInputStream(const uint8_t* data, size_t len); + +/** + * Returns a new InputStream with the contents written into an + * outputstream. The output stream must have been returned by + * an earlier call to memoryOutputStream(). The contents for the new + * input stream are the snapshot of the outputstream. One can construct + * any number of memory input stream from a single memory output stream. + */ +AVRO_DECL InputStreamPtr memoryInputStream(const OutputStream& source); + +/** + * Returns the contents written so far into the output stream, which should + * be a memory output stream. That is it must have been returned by a pervious + * call to memoryOutputStream(). + */ +AVRO_DECL std::shared_ptr<std::vector<uint8_t> > snapshot(const OutputStream& source); + +/** + * Returns a new OutputStream whose contents would be stored in a file. + * Data is written in chunks of given buffer size. + * + * If there is a file with the given name, it is truncated and overwritten. + * If there is no file with the given name, it is created. + */ +AVRO_DECL OutputStreamPtr fileOutputStream(const char* filename, + size_t bufferSize = 8 * 1024); + +/** + * Returns a new InputStream whose contents come from the given file. + * Data is read in chunks of given buffer size. + */ +AVRO_DECL InputStreamPtr fileInputStream( + const char *filename, size_t bufferSize = 8 * 1024); +AVRO_DECL SeekableInputStreamPtr fileSeekableInputStream( + const char *filename, size_t bufferSize = 8 * 1024); + +/** + * Returns a new OutputStream whose contents will be sent to the given + * std::ostream. The std::ostream object should outlive the returned + * OutputStream. + */ +AVRO_DECL OutputStreamPtr ostreamOutputStream(std::ostream& os, + size_t bufferSize = 8 * 1024); + +/** + * Returns a new InputStream whose contents come from the given + * std::istream. The std::istream object should outlive the returned + * InputStream. + */ +AVRO_DECL InputStreamPtr istreamInputStream( + std::istream &in, size_t bufferSize = 8 * 1024); + +/** + * Returns a new InputStream whose contents come from the given + * std::istream. Use this instead of istreamInputStream if + * the istream does not support seekg (e.g. compressed streams). + * The returned InputStream would read off bytes instead of seeking. + * Of, course it has a performance penalty when reading instead of seeking; + * So, use this only when seekg does not work. + * The std::istream object should outlive the returned + * InputStream. + */ +AVRO_DECL InputStreamPtr nonSeekableIstreamInputStream( + std::istream& is, size_t bufferSize = 8 * 1024); + + +/** A convenience class for reading from an InputStream */ +struct StreamReader { + /** + * The underlying input stream. + */ + InputStream* in_; + + /** + * The next location to read from. + */ + const uint8_t* next_; + + /** + * One past the last valid location. + */ + const uint8_t* end_; + + /** + * Constructs an empty reader. + */ + StreamReader() : in_(0), next_(0), end_(0) { } + + /** + * Constructs a reader with the given underlying stream. + */ + StreamReader(InputStream& in) : in_(0), next_(0), end_(0) { reset(in); } + + /** + * Replaces the current input stream with the given one after backing up + * the original one if required. + */ + void reset(InputStream& is) { + if (in_ != 0 && end_ != next_) { + in_->backup(end_ - next_); + } + in_ = &is; + next_ = end_ = 0; + } + + /** + * Read just one byte from the underlying stream. If there are no + * more data, throws an exception. + */ + uint8_t read() { + if (next_ == end_) { + more(); + } + return *next_++; + } + + /** + * Reads the given number of bytes from the underlying stream. + * If there are not that many bytes, throws an exception. + */ + void readBytes(uint8_t* b, size_t n) { + while (n > 0) { + if (next_ == end_) { + more(); + } + size_t q = end_ - next_; + if (q > n) { + q = n; + } + ::memcpy(b, next_, q); + next_ += q; + b += q; + n -= q; + } + } + + /** + * Skips the given number of bytes. Of there are not so that many + * bytes, throws an exception. + */ + void skipBytes(size_t n) { + if (n > static_cast<size_t>(end_ - next_)) { + n -= end_ - next_; + next_ = end_; + in_->skip(n); + } else { + next_ += n; + } + } + + /** + * Get as many byes from the underlying stream as possible in a single + * chunk. + * \return true if some data could be obtained. False is no more + * data is available on the stream. + */ + bool fill() { + size_t n = 0; + while (in_->next(&next_, &n)) { + if (n != 0) { + end_ = next_ + n; + return true; + } + } + return false; + } + + /** + * Tries to get more data and if it cannot, throws an exception. + */ + void more() { + if (! fill()) { + throw Exception("EOF reached"); + } + } + + /** + * Returns true if and only if the end of stream is not reached. + */ + bool hasMore() { + return (next_ == end_) ? fill() : true; + } + + /** + * Returns unused bytes back to the underlying stream. + * If unRead is true the last byte read is also pushed back. + */ + void drain(bool unRead) { + if (unRead) { + --next_; + } + in_->backup(end_ - next_); + end_ = next_; + } +}; + +/** + * A convinience class to write data into an OutputStream. + */ +struct StreamWriter { + /** + * The underlying output stream for this writer. + */ + OutputStream* out_; + + /** + * The next location to write to. + */ + uint8_t* next_; + + /** + * One past the last location one can write to. + */ + uint8_t* end_; + + /** + * Constructs a writer with no underlying stream. + */ + StreamWriter() : out_(0), next_(0), end_(0) { } + + /** + * Constructs a new writer with the given underlying stream. + */ + StreamWriter(OutputStream& out) : out_(0), next_(0), end_(0) { reset(out); } + + /** + * Replaces the current underlying stream with a new one. + * If required, it backs up unused bytes in the previous stream. + */ + void reset(OutputStream& os) { + if (out_ != 0 && end_ != next_) { + out_->backup(end_ - next_); + } + out_ = &os; + next_ = end_; + } + + /** + * Writes a single byte. + */ + void write(uint8_t c) { + if (next_ == end_) { + more(); + } + *next_++ = c; + } + + /** + * Writes the specified number of bytes starting at \p b. + */ + void writeBytes(const uint8_t* b, size_t n) { + while (n > 0) { + if (next_ == end_) { + more(); + } + size_t q = end_ - next_; + if (q > n) { + q = n; + } + ::memcpy(next_, b, q); + next_ += q; + b += q; + n -= q; + } + } + + /** + * backs up upto the currently written data and flushes the + * underlying stream. + */ + void flush() { + if (next_ != end_) { + out_->backup(end_ - next_); + next_ = end_; + } + out_->flush(); + } + + /** + * Return the number of bytes written so far. For a meaningful + * result, call this after a flush(). + */ + int64_t byteCount() const { + return out_->byteCount(); + } + + /** + * Gets more space to write to. Throws an exception it cannot. + */ + void more() { + size_t n = 0; + while (out_->next(&next_, &n)) { + if (n != 0) { + end_ = next_ + n; + return; + } + } + throw Exception("EOF reached"); + } +}; + +/** + * A convenience function to copy all the contents of an input stream into + * an output stream. + */ +inline void copy(InputStream& in, OutputStream& out) +{ + const uint8_t *p = 0; + size_t n = 0; + StreamWriter w(out); + while (in.next(&p, &n)) { + w.writeBytes(p, n); + } + w.flush(); +} + +} // namespace avro +#endif + + diff --git a/contrib/libs/apache/avro/api/Types.hh b/contrib/libs/apache/avro/api/Types.hh index a72647c483..f42399e96b 100644 --- a/contrib/libs/apache/avro/api/Types.hh +++ b/contrib/libs/apache/avro/api/Types.hh @@ -1,114 +1,114 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Types_hh__ -#define avro_Types_hh__ - -#include <iostream> - -#include "Config.hh" - -namespace avro { - -/** - * The "type" for the schema. - */ -enum Type { - - AVRO_STRING, /*!< String */ - AVRO_BYTES, /*!< Sequence of variable length bytes data */ - AVRO_INT, /*!< 32-bit integer */ - AVRO_LONG, /*!< 64-bit integer */ - AVRO_FLOAT, /*!< Floating point number */ - AVRO_DOUBLE, /*!< Double precision floating point number */ - AVRO_BOOL, /*!< Boolean value */ - AVRO_NULL, /*!< Null */ - - AVRO_RECORD, /*!< Record, a sequence of fields */ - AVRO_ENUM, /*!< Enumeration */ - AVRO_ARRAY, /*!< Homogeneous array of some specific type */ - AVRO_MAP, /*!< Homogeneous map from string to some specific type */ - AVRO_UNION, /*!< Union of one or more types */ - AVRO_FIXED, /*!< Fixed number of bytes */ - - AVRO_NUM_TYPES, /*!< Marker */ - - // The following is a pseudo-type used in implementation - - AVRO_SYMBOLIC = AVRO_NUM_TYPES, /*!< User internally to avoid circular references. */ - AVRO_UNKNOWN = -1 /*!< Used internally. */ - -}; - -/** - * Returns true if and only if the given type is a primitive. - * Primitive types are: string, bytes, int, long, float, double, boolean - * and null - */ -inline bool isPrimitive(Type t) { - return (t >= AVRO_STRING) && (t < AVRO_RECORD); -} - -/** - * Returns true if and only if the given type is a non primitive valid type. - * Primitive types are: string, bytes, int, long, float, double, boolean - * and null - */ -inline bool isCompound(Type t) { - return (t>= AVRO_RECORD) && (t < AVRO_NUM_TYPES); -} - -/** - * Returns true if and only if the given type is a valid avro type. - */ -inline bool isAvroType(Type t) { - return (t >= AVRO_STRING) && (t < AVRO_NUM_TYPES); -} - -/** - * Returns true if and only if the given type is within the valid range - * of enumeration. - */ -inline bool isAvroTypeOrPseudoType(Type t) { - return (t >= AVRO_STRING) && (t <= AVRO_NUM_TYPES); -} - -/** - * Converts the given type into a string. Useful for generating messages. - */ -AVRO_DECL const std::string& toString(Type type); - -/** - * Writes a string form of the given type into the given ostream. - */ -AVRO_DECL std::ostream &operator<< (std::ostream &os, avro::Type type); - -/// define a type to identify Null in template functions -struct AVRO_DECL Null { }; - -/** - * Writes schema for null \p null type to \p os. - * \param os The ostream to write to. - * \param null The value to be written. - */ -std::ostream& operator<< (std::ostream &os, const Null &null); - -} // namespace avro - - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Types_hh__ +#define avro_Types_hh__ + +#include <iostream> + +#include "Config.hh" + +namespace avro { + +/** + * The "type" for the schema. + */ +enum Type { + + AVRO_STRING, /*!< String */ + AVRO_BYTES, /*!< Sequence of variable length bytes data */ + AVRO_INT, /*!< 32-bit integer */ + AVRO_LONG, /*!< 64-bit integer */ + AVRO_FLOAT, /*!< Floating point number */ + AVRO_DOUBLE, /*!< Double precision floating point number */ + AVRO_BOOL, /*!< Boolean value */ + AVRO_NULL, /*!< Null */ + + AVRO_RECORD, /*!< Record, a sequence of fields */ + AVRO_ENUM, /*!< Enumeration */ + AVRO_ARRAY, /*!< Homogeneous array of some specific type */ + AVRO_MAP, /*!< Homogeneous map from string to some specific type */ + AVRO_UNION, /*!< Union of one or more types */ + AVRO_FIXED, /*!< Fixed number of bytes */ + + AVRO_NUM_TYPES, /*!< Marker */ + + // The following is a pseudo-type used in implementation + + AVRO_SYMBOLIC = AVRO_NUM_TYPES, /*!< User internally to avoid circular references. */ + AVRO_UNKNOWN = -1 /*!< Used internally. */ + +}; + +/** + * Returns true if and only if the given type is a primitive. + * Primitive types are: string, bytes, int, long, float, double, boolean + * and null + */ +inline bool isPrimitive(Type t) { + return (t >= AVRO_STRING) && (t < AVRO_RECORD); +} + +/** + * Returns true if and only if the given type is a non primitive valid type. + * Primitive types are: string, bytes, int, long, float, double, boolean + * and null + */ +inline bool isCompound(Type t) { + return (t>= AVRO_RECORD) && (t < AVRO_NUM_TYPES); +} + +/** + * Returns true if and only if the given type is a valid avro type. + */ +inline bool isAvroType(Type t) { + return (t >= AVRO_STRING) && (t < AVRO_NUM_TYPES); +} + +/** + * Returns true if and only if the given type is within the valid range + * of enumeration. + */ +inline bool isAvroTypeOrPseudoType(Type t) { + return (t >= AVRO_STRING) && (t <= AVRO_NUM_TYPES); +} + +/** + * Converts the given type into a string. Useful for generating messages. + */ +AVRO_DECL const std::string& toString(Type type); + +/** + * Writes a string form of the given type into the given ostream. + */ +AVRO_DECL std::ostream &operator<< (std::ostream &os, avro::Type type); + +/// define a type to identify Null in template functions +struct AVRO_DECL Null { }; + +/** + * Writes schema for null \p null type to \p os. + * \param os The ostream to write to. + * \param null The value to be written. + */ +std::ostream& operator<< (std::ostream &os, const Null &null); + +} // namespace avro + + +#endif diff --git a/contrib/libs/apache/avro/api/ValidSchema.hh b/contrib/libs/apache/avro/api/ValidSchema.hh index eeff4ef610..7b0ec28bed 100644 --- a/contrib/libs/apache/avro/api/ValidSchema.hh +++ b/contrib/libs/apache/avro/api/ValidSchema.hh @@ -1,66 +1,66 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_ValidSchema_hh__ -#define avro_ValidSchema_hh__ - -#include "Config.hh" -#include "Node.hh" - -namespace avro { - -class AVRO_DECL Schema; - -/// A ValidSchema is basically a non-mutable Schema that has passed some -/// minimum of sanity checks. Once validated, any Schema that is part of -/// this ValidSchema is considered locked, and cannot be modified (an attempt -/// to modify a locked Schema will throw). Also, as it is validated, any -/// recursive duplications of schemas are replaced with symbolic links to the -/// original. -/// -/// Once a Schema is converted to a valid schema it can be used in validating -/// parsers/serializers, converted to a json schema, etc. -/// - -class AVRO_DECL ValidSchema { -public: - explicit ValidSchema(const NodePtr &root); - explicit ValidSchema(const Schema &schema); - ValidSchema(); - - void setSchema(const Schema &schema); - - const NodePtr &root() const { - return root_; - } - - void toJson(std::ostream &os) const; - std::string toJson(bool prettyPrint = true) const; - - void toFlatList(std::ostream &os) const; - - protected: - NodePtr root_; - - private: - static std::string compactSchema(const std::string &schema); -}; - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_ValidSchema_hh__ +#define avro_ValidSchema_hh__ + +#include "Config.hh" +#include "Node.hh" + +namespace avro { + +class AVRO_DECL Schema; + +/// A ValidSchema is basically a non-mutable Schema that has passed some +/// minimum of sanity checks. Once validated, any Schema that is part of +/// this ValidSchema is considered locked, and cannot be modified (an attempt +/// to modify a locked Schema will throw). Also, as it is validated, any +/// recursive duplications of schemas are replaced with symbolic links to the +/// original. +/// +/// Once a Schema is converted to a valid schema it can be used in validating +/// parsers/serializers, converted to a json schema, etc. +/// + +class AVRO_DECL ValidSchema { +public: + explicit ValidSchema(const NodePtr &root); + explicit ValidSchema(const Schema &schema); + ValidSchema(); + + void setSchema(const Schema &schema); + + const NodePtr &root() const { + return root_; + } + + void toJson(std::ostream &os) const; + std::string toJson(bool prettyPrint = true) const; + + void toFlatList(std::ostream &os) const; + + protected: + NodePtr root_; + + private: + static std::string compactSchema(const std::string &schema); +}; + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Validator.hh b/contrib/libs/apache/avro/api/Validator.hh index 8000f6fd22..3f542d611a 100644 --- a/contrib/libs/apache/avro/api/Validator.hh +++ b/contrib/libs/apache/avro/api/Validator.hh @@ -1,161 +1,161 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Validating_hh__ -#define avro_Validating_hh__ - -#include <boost/noncopyable.hpp> -#include <vector> -#include <stdint.h> - -#include "Config.hh" -#include "Types.hh" -#include "ValidSchema.hh" - -namespace avro { - -class AVRO_DECL NullValidator : private boost::noncopyable -{ - public: - +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Validating_hh__ +#define avro_Validating_hh__ + +#include <boost/noncopyable.hpp> +#include <vector> +#include <stdint.h> + +#include "Config.hh" +#include "Types.hh" +#include "ValidSchema.hh" + +namespace avro { + +class AVRO_DECL NullValidator : private boost::noncopyable +{ + public: + explicit NullValidator(const ValidSchema &) {} - NullValidator() {} - + NullValidator() {} + void setCount(int64_t) {} - + bool typeIsExpected(Type) const { - return true; - } - - Type nextTypeExpected() const { - return AVRO_UNKNOWN; - } - - int nextSizeExpected() const { - return 0; - } - + return true; + } + + Type nextTypeExpected() const { + return AVRO_UNKNOWN; + } + + int nextSizeExpected() const { + return 0; + } + bool getCurrentRecordName(std::string &) const { - return true; - } - + return true; + } + bool getNextFieldName(std::string &) const { - return true; - } - + return true; + } + void checkTypeExpected(Type) { } void checkFixedSizeExpected(int) { } - - -}; - -/// This class is used by both the ValidatingSerializer and ValidationParser -/// objects. It advances the parse tree (containing logic how to advance -/// through the various compound types, for example a record must advance -/// through all leaf nodes but a union only skips to one), and reports which -/// type is next. - -class AVRO_DECL Validator : private boost::noncopyable -{ - public: - - explicit Validator(const ValidSchema &schema); - - void setCount(int64_t val); - - bool typeIsExpected(Type type) const { - return (expectedTypesFlag_ & typeToFlag(type)) != 0; - } - - Type nextTypeExpected() const { - return nextType_; - } - - int nextSizeExpected() const; - - bool getCurrentRecordName(std::string &name) const; - bool getNextFieldName(std::string &name) const; - - void checkTypeExpected(Type type) { - if(! typeIsExpected(type)) { - throw Exception( - boost::format("Type %1% does not match schema %2%") - % type % nextType_ - ); - } - advance(); - } - - void checkFixedSizeExpected(int size) { - if( nextSizeExpected() != size) { - throw Exception( - boost::format("Wrong size for fixed, got %1%, expected %2%") - % size % nextSizeExpected() - ); - } - checkTypeExpected(AVRO_FIXED); - } - - private: - - typedef uint32_t flag_t; - - flag_t typeToFlag(Type type) const { - flag_t flag = (1L << type); - return flag; - } - - void setupOperation(const NodePtr &node); - - void setWaitingForCount(); - - void advance(); - void doAdvance(); - - void enumAdvance(); - bool countingSetup(); - void countingAdvance(); - void unionAdvance(); - void fixedAdvance(); - - void setupFlag(Type type); - - const ValidSchema schema_; - - Type nextType_; - flag_t expectedTypesFlag_; - bool compoundStarted_; - bool waitingForCount_; - int64_t count_; - - struct CompoundType { - explicit CompoundType(const NodePtr &n) : - node(n), pos(0) - {} - NodePtr node; ///< save the node - size_t pos; ///< track the leaf position to visit - }; - - std::vector<CompoundType> compoundStack_; - std::vector<size_t> counters_; - -}; - -} // namespace avro - -#endif + + +}; + +/// This class is used by both the ValidatingSerializer and ValidationParser +/// objects. It advances the parse tree (containing logic how to advance +/// through the various compound types, for example a record must advance +/// through all leaf nodes but a union only skips to one), and reports which +/// type is next. + +class AVRO_DECL Validator : private boost::noncopyable +{ + public: + + explicit Validator(const ValidSchema &schema); + + void setCount(int64_t val); + + bool typeIsExpected(Type type) const { + return (expectedTypesFlag_ & typeToFlag(type)) != 0; + } + + Type nextTypeExpected() const { + return nextType_; + } + + int nextSizeExpected() const; + + bool getCurrentRecordName(std::string &name) const; + bool getNextFieldName(std::string &name) const; + + void checkTypeExpected(Type type) { + if(! typeIsExpected(type)) { + throw Exception( + boost::format("Type %1% does not match schema %2%") + % type % nextType_ + ); + } + advance(); + } + + void checkFixedSizeExpected(int size) { + if( nextSizeExpected() != size) { + throw Exception( + boost::format("Wrong size for fixed, got %1%, expected %2%") + % size % nextSizeExpected() + ); + } + checkTypeExpected(AVRO_FIXED); + } + + private: + + typedef uint32_t flag_t; + + flag_t typeToFlag(Type type) const { + flag_t flag = (1L << type); + return flag; + } + + void setupOperation(const NodePtr &node); + + void setWaitingForCount(); + + void advance(); + void doAdvance(); + + void enumAdvance(); + bool countingSetup(); + void countingAdvance(); + void unionAdvance(); + void fixedAdvance(); + + void setupFlag(Type type); + + const ValidSchema schema_; + + Type nextType_; + flag_t expectedTypesFlag_; + bool compoundStarted_; + bool waitingForCount_; + int64_t count_; + + struct CompoundType { + explicit CompoundType(const NodePtr &n) : + node(n), pos(0) + {} + NodePtr node; ///< save the node + size_t pos; ///< track the leaf position to visit + }; + + std::vector<CompoundType> compoundStack_; + std::vector<size_t> counters_; + +}; + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Writer.hh b/contrib/libs/apache/avro/api/Writer.hh index 7e74ce403e..74b057ce65 100644 --- a/contrib/libs/apache/avro/api/Writer.hh +++ b/contrib/libs/apache/avro/api/Writer.hh @@ -1,186 +1,186 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Writer_hh__ -#define avro_Writer_hh__ - -#include <array> -#include <boost/noncopyable.hpp> - -#include "Config.hh" -#include "buffer/Buffer.hh" -#include "Zigzag.hh" -#include "Types.hh" -#include "Validator.hh" - -namespace avro { - -/// Class for writing avro data to a stream. - -template<class ValidatorType> -class WriterImpl : private boost::noncopyable -{ - - public: - - WriterImpl() {} - - explicit WriterImpl(const ValidSchema &schema) : - validator_(schema) - {} - - void writeValue(const Null &) { - validator_.checkTypeExpected(AVRO_NULL); - } - - void writeValue(bool val) { - validator_.checkTypeExpected(AVRO_BOOL); - int8_t byte = (val != 0); - buffer_.writeTo(byte); - } - - void writeValue(int32_t val) { - validator_.checkTypeExpected(AVRO_INT); - std::array<uint8_t, 5> bytes; - size_t size = encodeInt32(val, bytes); - buffer_.writeTo(reinterpret_cast<const char *>(bytes.data()), size); - } - - void writeValue(int64_t val) { - validator_.checkTypeExpected(AVRO_LONG); - putLong(val); - } - - void writeValue(float val) { - validator_.checkTypeExpected(AVRO_FLOAT); - union { - float f; - int32_t i; - } v; - - v.f = val; - buffer_.writeTo(v.i); - } - - void writeValue(double val) { - validator_.checkTypeExpected(AVRO_DOUBLE); - union { - double d; - int64_t i; - } v; - - v.d = val; - buffer_.writeTo(v.i); - } - - void writeValue(const std::string &val) { - validator_.checkTypeExpected(AVRO_STRING); - putBytes(val.c_str(), val.size()); - } - - void writeBytes(const void *val, size_t size) { - validator_.checkTypeExpected(AVRO_BYTES); - putBytes(val, size); - } - - template <size_t N> - void writeFixed(const uint8_t (&val)[N]) { - validator_.checkFixedSizeExpected(N); - buffer_.writeTo(reinterpret_cast<const char *>(val), N); - } - - template <size_t N> - void writeFixed(const std::array<uint8_t, N> &val) { - validator_.checkFixedSizeExpected(val.size()); - buffer_.writeTo(reinterpret_cast<const char *>(val.data()), val.size()); - } - - void writeRecord() { - validator_.checkTypeExpected(AVRO_RECORD); - validator_.checkTypeExpected(AVRO_LONG); - validator_.setCount(1); - } - - void writeRecordEnd() { - validator_.checkTypeExpected(AVRO_RECORD); - validator_.checkTypeExpected(AVRO_LONG); - validator_.setCount(0); - } - - void writeArrayBlock(int64_t size) { - validator_.checkTypeExpected(AVRO_ARRAY); - writeCount(size); - } - - void writeArrayEnd() { - writeArrayBlock(0); - } - - void writeMapBlock(int64_t size) { - validator_.checkTypeExpected(AVRO_MAP); - writeCount(size); - } - - void writeMapEnd() { - writeMapBlock(0); - } - - void writeUnion(int64_t choice) { - validator_.checkTypeExpected(AVRO_UNION); - writeCount(choice); - } - - void writeEnum(int64_t choice) { - validator_.checkTypeExpected(AVRO_ENUM); - writeCount(choice); - } - - InputBuffer buffer() const { - return buffer_; - } - - private: - - void putLong(int64_t val) { - std::array<uint8_t, 10> bytes; - size_t size = encodeInt64(val, bytes); - buffer_.writeTo(reinterpret_cast<const char *>(bytes.data()), size); - } - - void putBytes(const void *val, size_t size) { - putLong(size); - buffer_.writeTo(reinterpret_cast<const char *>(val), size); - } - - void writeCount(int64_t count) { - validator_.checkTypeExpected(AVRO_LONG); - validator_.setCount(count); - putLong(count); - } - - ValidatorType validator_; - OutputBuffer buffer_; - -}; - -typedef WriterImpl<NullValidator> Writer; -typedef WriterImpl<Validator> ValidatingWriter; - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Writer_hh__ +#define avro_Writer_hh__ + +#include <array> +#include <boost/noncopyable.hpp> + +#include "Config.hh" +#include "buffer/Buffer.hh" +#include "Zigzag.hh" +#include "Types.hh" +#include "Validator.hh" + +namespace avro { + +/// Class for writing avro data to a stream. + +template<class ValidatorType> +class WriterImpl : private boost::noncopyable +{ + + public: + + WriterImpl() {} + + explicit WriterImpl(const ValidSchema &schema) : + validator_(schema) + {} + + void writeValue(const Null &) { + validator_.checkTypeExpected(AVRO_NULL); + } + + void writeValue(bool val) { + validator_.checkTypeExpected(AVRO_BOOL); + int8_t byte = (val != 0); + buffer_.writeTo(byte); + } + + void writeValue(int32_t val) { + validator_.checkTypeExpected(AVRO_INT); + std::array<uint8_t, 5> bytes; + size_t size = encodeInt32(val, bytes); + buffer_.writeTo(reinterpret_cast<const char *>(bytes.data()), size); + } + + void writeValue(int64_t val) { + validator_.checkTypeExpected(AVRO_LONG); + putLong(val); + } + + void writeValue(float val) { + validator_.checkTypeExpected(AVRO_FLOAT); + union { + float f; + int32_t i; + } v; + + v.f = val; + buffer_.writeTo(v.i); + } + + void writeValue(double val) { + validator_.checkTypeExpected(AVRO_DOUBLE); + union { + double d; + int64_t i; + } v; + + v.d = val; + buffer_.writeTo(v.i); + } + + void writeValue(const std::string &val) { + validator_.checkTypeExpected(AVRO_STRING); + putBytes(val.c_str(), val.size()); + } + + void writeBytes(const void *val, size_t size) { + validator_.checkTypeExpected(AVRO_BYTES); + putBytes(val, size); + } + + template <size_t N> + void writeFixed(const uint8_t (&val)[N]) { + validator_.checkFixedSizeExpected(N); + buffer_.writeTo(reinterpret_cast<const char *>(val), N); + } + + template <size_t N> + void writeFixed(const std::array<uint8_t, N> &val) { + validator_.checkFixedSizeExpected(val.size()); + buffer_.writeTo(reinterpret_cast<const char *>(val.data()), val.size()); + } + + void writeRecord() { + validator_.checkTypeExpected(AVRO_RECORD); + validator_.checkTypeExpected(AVRO_LONG); + validator_.setCount(1); + } + + void writeRecordEnd() { + validator_.checkTypeExpected(AVRO_RECORD); + validator_.checkTypeExpected(AVRO_LONG); + validator_.setCount(0); + } + + void writeArrayBlock(int64_t size) { + validator_.checkTypeExpected(AVRO_ARRAY); + writeCount(size); + } + + void writeArrayEnd() { + writeArrayBlock(0); + } + + void writeMapBlock(int64_t size) { + validator_.checkTypeExpected(AVRO_MAP); + writeCount(size); + } + + void writeMapEnd() { + writeMapBlock(0); + } + + void writeUnion(int64_t choice) { + validator_.checkTypeExpected(AVRO_UNION); + writeCount(choice); + } + + void writeEnum(int64_t choice) { + validator_.checkTypeExpected(AVRO_ENUM); + writeCount(choice); + } + + InputBuffer buffer() const { + return buffer_; + } + + private: + + void putLong(int64_t val) { + std::array<uint8_t, 10> bytes; + size_t size = encodeInt64(val, bytes); + buffer_.writeTo(reinterpret_cast<const char *>(bytes.data()), size); + } + + void putBytes(const void *val, size_t size) { + putLong(size); + buffer_.writeTo(reinterpret_cast<const char *>(val), size); + } + + void writeCount(int64_t count) { + validator_.checkTypeExpected(AVRO_LONG); + validator_.setCount(count); + putLong(count); + } + + ValidatorType validator_; + OutputBuffer buffer_; + +}; + +typedef WriterImpl<NullValidator> Writer; +typedef WriterImpl<Validator> ValidatingWriter; + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/Zigzag.hh b/contrib/libs/apache/avro/api/Zigzag.hh index ed76aae931..d0259b8d50 100644 --- a/contrib/libs/apache/avro/api/Zigzag.hh +++ b/contrib/libs/apache/avro/api/Zigzag.hh @@ -1,43 +1,43 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Encoding_hh__ -#define avro_Encoding_hh__ - -#include <stdint.h> -#include <array> -#include <cstddef> - -#include "Config.hh" -/// \file -/// Functions for encoding and decoding integers with zigzag compression - -namespace avro { - -AVRO_DECL uint64_t encodeZigzag64(int64_t input); -AVRO_DECL int64_t decodeZigzag64(uint64_t input); - -AVRO_DECL uint32_t encodeZigzag32(int32_t input); -AVRO_DECL int32_t decodeZigzag32(uint32_t input); - -AVRO_DECL size_t encodeInt32(int32_t input, std::array<uint8_t, 5> &output); -AVRO_DECL size_t encodeInt64(int64_t input, std::array<uint8_t, 10> &output); - -} // namespace avro - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Encoding_hh__ +#define avro_Encoding_hh__ + +#include <stdint.h> +#include <array> +#include <cstddef> + +#include "Config.hh" +/// \file +/// Functions for encoding and decoding integers with zigzag compression + +namespace avro { + +AVRO_DECL uint64_t encodeZigzag64(int64_t input); +AVRO_DECL int64_t decodeZigzag64(uint64_t input); + +AVRO_DECL uint32_t encodeZigzag32(int32_t input); +AVRO_DECL int32_t decodeZigzag32(uint32_t input); + +AVRO_DECL size_t encodeInt32(int32_t input, std::array<uint8_t, 5> &output); +AVRO_DECL size_t encodeInt64(int64_t input, std::array<uint8_t, 10> &output); + +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/api/buffer/Buffer.hh b/contrib/libs/apache/avro/api/buffer/Buffer.hh index b2a04dd683..7d7aaf8679 100644 --- a/contrib/libs/apache/avro/api/buffer/Buffer.hh +++ b/contrib/libs/apache/avro/api/buffer/Buffer.hh @@ -1,526 +1,526 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_Buffer_hh__ -#define avro_Buffer_hh__ - -#ifndef _WIN32 -#include <sys/uio.h> -#endif -#include <vector> - -#include "../Config.hh" -#include "detail/BufferDetail.hh" -#include "detail/BufferDetailIterator.hh" - -/** - * \file Buffer.hh - * - * \brief Definitions for InputBuffer and OutputBuffer classes - * - **/ - -namespace avro { - -class OutputBuffer; -class InputBuffer; - - -/** - * The OutputBuffer (write-only buffer) - * - * Use cases for OutputBuffer - * - * - write message to buffer using ostream class or directly - * - append messages to headers - * - building up streams of messages via append - * - converting to read-only buffers for sending - * - extracting parts of the messages into read-only buffers - * - * -# ASIO access: - * - write to a buffer(s) by asio using iterator - * - convert to read buffer for deserializing - * - * OutputBuffer is assignable and copy-constructable. On copy or assignment, - * only a pointer is copied, so the two resulting copies are identical, so - * modifying one will modify both. - **/ - -class AVRO_DECL OutputBuffer -{ - - public: - - typedef detail::size_type size_type; - typedef detail::data_type data_type; - - /** - * The asio library expects a const_iterator (the const-ness refers to the - * fact that the underlying avro of buffers will not be modified, even - * though the data in those buffers is being modified). The iterator - * provides the list of addresses an operation can write to. - **/ - - typedef detail::OutputBufferIterator const_iterator; - - /** - * Default constructor. Will pre-allocate at least the requested size, but - * can grow larger on demand. - * - * Destructor uses the default, which resets a shared pointer, deleting the - * underlying data if no other copies of exist. - * - * Copy and assignment operators are not explicitly provided because the - * default ones work fine. The default makes only a shallow copy, so the - * copies will refer to the same memory. This is required by asio - * functions, which will implicitly make copies for asynchronous - * operations. Therefore, the user must be careful that if they create - * multiple copies of the same OutputBuffer, only one is being modified - * otherwise undefined behavior may occur. - * - **/ - - OutputBuffer(size_type reserveSize = 0) : - pimpl_(new detail::BufferImpl) - { - if(reserveSize) { - reserve(reserveSize); - } - } - - /** - * Reserve enough space for a wroteTo() operation. When using writeTo(), - * the buffer will grow dynamically as needed. But when using the iterator - * to write (followed by wroteTo()), data may only be written to the space - * available, so this ensures there is enough room in the buffer before - * the write operation. - **/ - - void reserve(size_type reserveSize) - { - pimpl_->reserveFreeSpace(reserveSize); - } - - /** - * Write a block of data to the buffer. The buffer size will automatically - * grow if the size is larger than what is currently free. - **/ - - size_type writeTo(const data_type *data, size_type size) { - return pimpl_->writeTo(data, size); - } - - /** - * Write a single value to the buffer. The buffer size will automatically - * grow if there is not room for the byte. The value must be a - * "fundamental" type, e.g. int, float, etc. (otherwise use the other - * writeTo tests). - **/ - - template<typename T> - void writeTo(T val) { - pimpl_->writeTo(val, std::is_fundamental<T>()); - } - - /** - * Update the state of the buffer after writing through the iterator - * interface. This function exists primarily for the boost:asio which - * writes directly to the buffer using its iterator. In this case, the - * internal state of the buffer does not reflect that the data was written - * This informs the buffer how much data was written. - * - * The buffer does not automatically resize in this case, the bytes written - * cannot exceed the amount of free space. Attempting to write more will - * throw a std::length_error exception. - **/ - - size_type wroteTo(size_type size) - { - int wrote = 0; - if(size) { - if(size > freeSpace()) { - throw std::length_error("Impossible to write more data than free space"); - } - wrote = pimpl_->wroteTo(size); - } - return wrote; - } - - /** - * Does the buffer have any data? - **/ - - bool empty() const { - return (pimpl_->size()==0); - } - - /** - * Returns the size of the buffer, in bytes. - */ - - size_type size() const { - return pimpl_->size(); - } - - /** - * Returns the current free space that is available to write to in the - * buffer, in bytes. This is not a strict limit in size, as writeTo() can - * automatically increase capacity if necessary. - **/ - - size_type freeSpace() const { - return pimpl_->freeSpace(); - } - - /** - * Appends the data in the argument to the end of this buffer. The - * argument can be either an InputBuffer or OutputBuffer. - * - **/ - - template <class BufferType> - void append(const BufferType &buf) { - // don't append an empty buffer - if(buf.size()) { - pimpl_->append(*(buf.pimpl_.get())); - } - } - - /** - * Return an iterator pointing to the first data chunk of this buffer - * that may be written to. - **/ - - const_iterator begin() const { - return const_iterator(pimpl_->beginWrite()); - } - - /** - * Return the end iterator for writing. - **/ - - const_iterator end() const { - return const_iterator(pimpl_->endWrite()); - } - - /** - * Discard any data in this buffer. - **/ - - void discardData() - { - pimpl_->discardData(); - } - - /** - * Discard the specified number of bytes from this data, starting at the beginning. - * Throws if the size is greater than the number of bytes. - **/ - - void discardData(size_t bytes) - { - if(bytes > 0) { - if(bytes < pimpl_->size()) { - pimpl_->discardData(bytes); - } - else if(bytes == pimpl_->size()) { - pimpl_->discardData(); - } - else { - throw std::out_of_range("trying to discard more data than exists"); - } - } - } - - /** - * Remove bytes from this buffer, starting from the beginning, and place - * them into a new buffer. Throws if the number of requested bytes exceeds - * the size of the buffer. Data and freeSpace in the buffer after bytes - * remains in this buffer. - **/ - - InputBuffer extractData(size_type bytes); - - /** - * Remove all bytes from this buffer, returning them in a new buffer. - * After removing data, some freeSpace may remain in this buffer. - **/ - - InputBuffer extractData(); - - /** - * Clone this buffer, creating a copy that contains the same data. - **/ - - OutputBuffer clone() const - { - detail::BufferImpl::SharedPtr newImpl(new detail::BufferImpl(*pimpl_)); - return OutputBuffer(newImpl); - } - - /** - * Add unmanaged data to the buffer. The buffer will not automatically - * free the data, but it will call the supplied function when the data is - * no longer referenced by the buffer (or copies of the buffer). - **/ - - void appendForeignData(const data_type *data, size_type size, const detail::free_func &func) { - pimpl_->appendForeignData(data, size, func); - } - - /** - * Returns the number of chunks that contain free space. - **/ - - int numChunks() const { - return pimpl_->numFreeChunks(); - } - - /** - * Returns the number of chunks that contain data - **/ - - int numDataChunks() const { - return pimpl_->numDataChunks(); - } - - private: - - friend class InputBuffer; - friend class BufferReader; - - explicit OutputBuffer(const detail::BufferImpl::SharedPtr &pimpl) : - pimpl_(pimpl) - { } - - detail::BufferImpl::SharedPtr pimpl_; ///< Must never be null. -}; - -/** - * The InputBuffer (read-only buffer) - * - * InputBuffer is an immutable buffer which that may be constructed from an - * OutputBuffer, or several of OutputBuffer's methods. Once the data is - * transfered to an InputBuffer it cannot be modified, only read (via - * BufferReader, istream, or its iterator). - * - * Assignments and copies are shallow copies. - * - * -# ASIO access: - iterate using const_iterator for sending messages - * - **/ - -class AVRO_DECL InputBuffer -{ - - public: - - typedef detail::size_type size_type; - typedef detail::data_type data_type; - - // needed for asio - typedef detail::InputBufferIterator const_iterator; - - /** - * Default InputBuffer creates an empty buffer. - * - * Copy/assignment functions use the default ones. They will do a shallow - * copy, and because InputBuffer is immutable, the copies will be - * identical. - * - * Destructor also uses the default, which resets a shared pointer, - * deleting the underlying data if no other copies of exist. - **/ - - InputBuffer() : - pimpl_(new detail::BufferImpl) - { } - - /** - * Construct an InputBuffer that contains the contents of an OutputBuffer. - * The two buffers will have the same contents, but this copy will be - * immutable, while the the OutputBuffer may still be written to. - * - * If you wish to move the data from the OutputBuffer to a new InputBuffer - * (leaving only free space in the OutputBuffer), - * OutputBuffer::extractData() will do this more efficiently. - * - * Implicit conversion is allowed. - **/ - - InputBuffer(const OutputBuffer &src) : - pimpl_(new detail::BufferImpl(*src.pimpl_)) - { } - - /** - * Does the buffer have any data? - **/ - - bool empty() const { - return (pimpl_->size() == 0); - } - - /** - * Returns the size of the buffer, in bytes. - **/ - - size_type size() const { - return pimpl_->size(); - } - - /** - * Return an iterator pointing to the first data chunk of this buffer - * that contains data. - **/ - - const_iterator begin() const { - return const_iterator(pimpl_->beginRead()); - } - - /** - * Return the end iterator. - **/ - - const_iterator end() const { - return const_iterator(pimpl_->endRead()); - } - - /** - * Returns the number of chunks containing data. - **/ - - int numChunks() const { - return pimpl_->numDataChunks(); - } - - - private: - - friend class OutputBuffer; // for append function - friend class istreambuf; - friend class BufferReader; - - explicit InputBuffer(const detail::BufferImpl::SharedPtr &pimpl) : - pimpl_(pimpl) - { } - - /** - * Class to indicate that a copy of a OutputBuffer to InputBuffer should be - * a shallow copy, used to enable reading of the contents of an - * OutputBuffer without need to convert it to InputBuffer using a deep - * copy. It is private and only used by BufferReader and istreambuf - * classes. - * - * Writing to an OutputBuffer while it is being read may lead to undefined - * behavior. - **/ - - class ShallowCopy {}; - - /** - * Make a shallow copy of an OutputBuffer in order to read it without - * causing conversion overhead. - **/ - InputBuffer(const OutputBuffer &src, const ShallowCopy &) : - pimpl_(src.pimpl_) - { } - - /** - * Make a shallow copy of an InputBuffer. The default copy constructor - * already provides shallow copy, this is just provided for generic - * algorithms that wish to treat InputBuffer and OutputBuffer in the same - * manner. - **/ - - InputBuffer(const InputBuffer &src, const ShallowCopy &) : - pimpl_(src.pimpl_) - { } - - - detail::BufferImpl::ConstSharedPtr pimpl_; ///< Must never be null. -}; - - -/* - * Implementations of some OutputBuffer functions are inlined here - * because InputBuffer definition was required before. - */ - -inline InputBuffer OutputBuffer::extractData() -{ - detail::BufferImpl::SharedPtr newImpl(new detail::BufferImpl); - if(pimpl_->size()) { - pimpl_->extractData(*newImpl); - } - return InputBuffer(newImpl); -} - -inline InputBuffer OutputBuffer::extractData(size_type bytes) -{ - if(bytes > pimpl_->size()) { - throw std::out_of_range("trying to extract more data than exists"); - } - - detail::BufferImpl::SharedPtr newImpl(new detail::BufferImpl); - if(bytes > 0) { - if(bytes < pimpl_->size()) { - pimpl_->extractData(*newImpl, bytes); - } - else { - pimpl_->extractData(*newImpl); - } - } - - return InputBuffer(newImpl); -} - -#ifndef _WIN32 -/** - * Create an array of iovec structures from the buffer. This utility is used - * to support writev and readv function calls. The caller should ensure the - * buffer object is not deleted while using the iovec vector. - * - * If the BufferType is an InputBuffer, the iovec will point to the data that - * already exists in the buffer, for reading. - * - * If the BufferType is an OutputBuffer, the iovec will point to the free - * space, which may be written to. Before writing, the caller should call - * OutputBuffer::reserve() to create enough room for the desired write (which - * can be verified by calling OutputBuffer::freeSpace()), and after writing, - * they MUST call OutputBuffer::wroteTo(), otherwise the buffer will not know - * the space is not free anymore. - * - **/ - -template<class BufferType> -inline void toIovec(BufferType &buf, std::vector<struct iovec> &iov) -{ - const int chunks = buf.numChunks(); - iov.resize(chunks); - typename BufferType::const_iterator iter = buf.begin(); - for (int i = 0; i < chunks; ++i) { - iov[i].iov_base = const_cast<typename BufferType::data_type *>(iter->data()); - iov[i].iov_len = iter->size(); - ++iter; - } -} -#endif - -} // namespace - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Buffer_hh__ +#define avro_Buffer_hh__ + +#ifndef _WIN32 +#include <sys/uio.h> +#endif +#include <vector> + +#include "../Config.hh" +#include "detail/BufferDetail.hh" +#include "detail/BufferDetailIterator.hh" + +/** + * \file Buffer.hh + * + * \brief Definitions for InputBuffer and OutputBuffer classes + * + **/ + +namespace avro { + +class OutputBuffer; +class InputBuffer; + + +/** + * The OutputBuffer (write-only buffer) + * + * Use cases for OutputBuffer + * + * - write message to buffer using ostream class or directly + * - append messages to headers + * - building up streams of messages via append + * - converting to read-only buffers for sending + * - extracting parts of the messages into read-only buffers + * + * -# ASIO access: + * - write to a buffer(s) by asio using iterator + * - convert to read buffer for deserializing + * + * OutputBuffer is assignable and copy-constructable. On copy or assignment, + * only a pointer is copied, so the two resulting copies are identical, so + * modifying one will modify both. + **/ + +class AVRO_DECL OutputBuffer +{ + + public: + + typedef detail::size_type size_type; + typedef detail::data_type data_type; + + /** + * The asio library expects a const_iterator (the const-ness refers to the + * fact that the underlying avro of buffers will not be modified, even + * though the data in those buffers is being modified). The iterator + * provides the list of addresses an operation can write to. + **/ + + typedef detail::OutputBufferIterator const_iterator; + + /** + * Default constructor. Will pre-allocate at least the requested size, but + * can grow larger on demand. + * + * Destructor uses the default, which resets a shared pointer, deleting the + * underlying data if no other copies of exist. + * + * Copy and assignment operators are not explicitly provided because the + * default ones work fine. The default makes only a shallow copy, so the + * copies will refer to the same memory. This is required by asio + * functions, which will implicitly make copies for asynchronous + * operations. Therefore, the user must be careful that if they create + * multiple copies of the same OutputBuffer, only one is being modified + * otherwise undefined behavior may occur. + * + **/ + + OutputBuffer(size_type reserveSize = 0) : + pimpl_(new detail::BufferImpl) + { + if(reserveSize) { + reserve(reserveSize); + } + } + + /** + * Reserve enough space for a wroteTo() operation. When using writeTo(), + * the buffer will grow dynamically as needed. But when using the iterator + * to write (followed by wroteTo()), data may only be written to the space + * available, so this ensures there is enough room in the buffer before + * the write operation. + **/ + + void reserve(size_type reserveSize) + { + pimpl_->reserveFreeSpace(reserveSize); + } + + /** + * Write a block of data to the buffer. The buffer size will automatically + * grow if the size is larger than what is currently free. + **/ + + size_type writeTo(const data_type *data, size_type size) { + return pimpl_->writeTo(data, size); + } + + /** + * Write a single value to the buffer. The buffer size will automatically + * grow if there is not room for the byte. The value must be a + * "fundamental" type, e.g. int, float, etc. (otherwise use the other + * writeTo tests). + **/ + + template<typename T> + void writeTo(T val) { + pimpl_->writeTo(val, std::is_fundamental<T>()); + } + + /** + * Update the state of the buffer after writing through the iterator + * interface. This function exists primarily for the boost:asio which + * writes directly to the buffer using its iterator. In this case, the + * internal state of the buffer does not reflect that the data was written + * This informs the buffer how much data was written. + * + * The buffer does not automatically resize in this case, the bytes written + * cannot exceed the amount of free space. Attempting to write more will + * throw a std::length_error exception. + **/ + + size_type wroteTo(size_type size) + { + int wrote = 0; + if(size) { + if(size > freeSpace()) { + throw std::length_error("Impossible to write more data than free space"); + } + wrote = pimpl_->wroteTo(size); + } + return wrote; + } + + /** + * Does the buffer have any data? + **/ + + bool empty() const { + return (pimpl_->size()==0); + } + + /** + * Returns the size of the buffer, in bytes. + */ + + size_type size() const { + return pimpl_->size(); + } + + /** + * Returns the current free space that is available to write to in the + * buffer, in bytes. This is not a strict limit in size, as writeTo() can + * automatically increase capacity if necessary. + **/ + + size_type freeSpace() const { + return pimpl_->freeSpace(); + } + + /** + * Appends the data in the argument to the end of this buffer. The + * argument can be either an InputBuffer or OutputBuffer. + * + **/ + + template <class BufferType> + void append(const BufferType &buf) { + // don't append an empty buffer + if(buf.size()) { + pimpl_->append(*(buf.pimpl_.get())); + } + } + + /** + * Return an iterator pointing to the first data chunk of this buffer + * that may be written to. + **/ + + const_iterator begin() const { + return const_iterator(pimpl_->beginWrite()); + } + + /** + * Return the end iterator for writing. + **/ + + const_iterator end() const { + return const_iterator(pimpl_->endWrite()); + } + + /** + * Discard any data in this buffer. + **/ + + void discardData() + { + pimpl_->discardData(); + } + + /** + * Discard the specified number of bytes from this data, starting at the beginning. + * Throws if the size is greater than the number of bytes. + **/ + + void discardData(size_t bytes) + { + if(bytes > 0) { + if(bytes < pimpl_->size()) { + pimpl_->discardData(bytes); + } + else if(bytes == pimpl_->size()) { + pimpl_->discardData(); + } + else { + throw std::out_of_range("trying to discard more data than exists"); + } + } + } + + /** + * Remove bytes from this buffer, starting from the beginning, and place + * them into a new buffer. Throws if the number of requested bytes exceeds + * the size of the buffer. Data and freeSpace in the buffer after bytes + * remains in this buffer. + **/ + + InputBuffer extractData(size_type bytes); + + /** + * Remove all bytes from this buffer, returning them in a new buffer. + * After removing data, some freeSpace may remain in this buffer. + **/ + + InputBuffer extractData(); + + /** + * Clone this buffer, creating a copy that contains the same data. + **/ + + OutputBuffer clone() const + { + detail::BufferImpl::SharedPtr newImpl(new detail::BufferImpl(*pimpl_)); + return OutputBuffer(newImpl); + } + + /** + * Add unmanaged data to the buffer. The buffer will not automatically + * free the data, but it will call the supplied function when the data is + * no longer referenced by the buffer (or copies of the buffer). + **/ + + void appendForeignData(const data_type *data, size_type size, const detail::free_func &func) { + pimpl_->appendForeignData(data, size, func); + } + + /** + * Returns the number of chunks that contain free space. + **/ + + int numChunks() const { + return pimpl_->numFreeChunks(); + } + + /** + * Returns the number of chunks that contain data + **/ + + int numDataChunks() const { + return pimpl_->numDataChunks(); + } + + private: + + friend class InputBuffer; + friend class BufferReader; + + explicit OutputBuffer(const detail::BufferImpl::SharedPtr &pimpl) : + pimpl_(pimpl) + { } + + detail::BufferImpl::SharedPtr pimpl_; ///< Must never be null. +}; + +/** + * The InputBuffer (read-only buffer) + * + * InputBuffer is an immutable buffer which that may be constructed from an + * OutputBuffer, or several of OutputBuffer's methods. Once the data is + * transfered to an InputBuffer it cannot be modified, only read (via + * BufferReader, istream, or its iterator). + * + * Assignments and copies are shallow copies. + * + * -# ASIO access: - iterate using const_iterator for sending messages + * + **/ + +class AVRO_DECL InputBuffer +{ + + public: + + typedef detail::size_type size_type; + typedef detail::data_type data_type; + + // needed for asio + typedef detail::InputBufferIterator const_iterator; + + /** + * Default InputBuffer creates an empty buffer. + * + * Copy/assignment functions use the default ones. They will do a shallow + * copy, and because InputBuffer is immutable, the copies will be + * identical. + * + * Destructor also uses the default, which resets a shared pointer, + * deleting the underlying data if no other copies of exist. + **/ + + InputBuffer() : + pimpl_(new detail::BufferImpl) + { } + + /** + * Construct an InputBuffer that contains the contents of an OutputBuffer. + * The two buffers will have the same contents, but this copy will be + * immutable, while the the OutputBuffer may still be written to. + * + * If you wish to move the data from the OutputBuffer to a new InputBuffer + * (leaving only free space in the OutputBuffer), + * OutputBuffer::extractData() will do this more efficiently. + * + * Implicit conversion is allowed. + **/ + + InputBuffer(const OutputBuffer &src) : + pimpl_(new detail::BufferImpl(*src.pimpl_)) + { } + + /** + * Does the buffer have any data? + **/ + + bool empty() const { + return (pimpl_->size() == 0); + } + + /** + * Returns the size of the buffer, in bytes. + **/ + + size_type size() const { + return pimpl_->size(); + } + + /** + * Return an iterator pointing to the first data chunk of this buffer + * that contains data. + **/ + + const_iterator begin() const { + return const_iterator(pimpl_->beginRead()); + } + + /** + * Return the end iterator. + **/ + + const_iterator end() const { + return const_iterator(pimpl_->endRead()); + } + + /** + * Returns the number of chunks containing data. + **/ + + int numChunks() const { + return pimpl_->numDataChunks(); + } + + + private: + + friend class OutputBuffer; // for append function + friend class istreambuf; + friend class BufferReader; + + explicit InputBuffer(const detail::BufferImpl::SharedPtr &pimpl) : + pimpl_(pimpl) + { } + + /** + * Class to indicate that a copy of a OutputBuffer to InputBuffer should be + * a shallow copy, used to enable reading of the contents of an + * OutputBuffer without need to convert it to InputBuffer using a deep + * copy. It is private and only used by BufferReader and istreambuf + * classes. + * + * Writing to an OutputBuffer while it is being read may lead to undefined + * behavior. + **/ + + class ShallowCopy {}; + + /** + * Make a shallow copy of an OutputBuffer in order to read it without + * causing conversion overhead. + **/ + InputBuffer(const OutputBuffer &src, const ShallowCopy &) : + pimpl_(src.pimpl_) + { } + + /** + * Make a shallow copy of an InputBuffer. The default copy constructor + * already provides shallow copy, this is just provided for generic + * algorithms that wish to treat InputBuffer and OutputBuffer in the same + * manner. + **/ + + InputBuffer(const InputBuffer &src, const ShallowCopy &) : + pimpl_(src.pimpl_) + { } + + + detail::BufferImpl::ConstSharedPtr pimpl_; ///< Must never be null. +}; + + +/* + * Implementations of some OutputBuffer functions are inlined here + * because InputBuffer definition was required before. + */ + +inline InputBuffer OutputBuffer::extractData() +{ + detail::BufferImpl::SharedPtr newImpl(new detail::BufferImpl); + if(pimpl_->size()) { + pimpl_->extractData(*newImpl); + } + return InputBuffer(newImpl); +} + +inline InputBuffer OutputBuffer::extractData(size_type bytes) +{ + if(bytes > pimpl_->size()) { + throw std::out_of_range("trying to extract more data than exists"); + } + + detail::BufferImpl::SharedPtr newImpl(new detail::BufferImpl); + if(bytes > 0) { + if(bytes < pimpl_->size()) { + pimpl_->extractData(*newImpl, bytes); + } + else { + pimpl_->extractData(*newImpl); + } + } + + return InputBuffer(newImpl); +} + +#ifndef _WIN32 +/** + * Create an array of iovec structures from the buffer. This utility is used + * to support writev and readv function calls. The caller should ensure the + * buffer object is not deleted while using the iovec vector. + * + * If the BufferType is an InputBuffer, the iovec will point to the data that + * already exists in the buffer, for reading. + * + * If the BufferType is an OutputBuffer, the iovec will point to the free + * space, which may be written to. Before writing, the caller should call + * OutputBuffer::reserve() to create enough room for the desired write (which + * can be verified by calling OutputBuffer::freeSpace()), and after writing, + * they MUST call OutputBuffer::wroteTo(), otherwise the buffer will not know + * the space is not free anymore. + * + **/ + +template<class BufferType> +inline void toIovec(BufferType &buf, std::vector<struct iovec> &iov) +{ + const int chunks = buf.numChunks(); + iov.resize(chunks); + typename BufferType::const_iterator iter = buf.begin(); + for (int i = 0; i < chunks; ++i) { + iov[i].iov_base = const_cast<typename BufferType::data_type *>(iter->data()); + iov[i].iov_len = iter->size(); + ++iter; + } +} +#endif + +} // namespace + +#endif diff --git a/contrib/libs/apache/avro/api/buffer/BufferReader.hh b/contrib/libs/apache/avro/api/buffer/BufferReader.hh index cc1b05880b..83b6b4b324 100644 --- a/contrib/libs/apache/avro/api/buffer/BufferReader.hh +++ b/contrib/libs/apache/avro/api/buffer/BufferReader.hh @@ -1,289 +1,289 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_BufferReader_hh__ -#define avro_BufferReader_hh__ - -#include <type_traits> -#include "Buffer.hh" - -#ifdef min -#undef min -#endif -/** - * \file BufferReader.hh - * - * \brief Helper class for reading bytes from buffer in a streaming manner, - * without the overhead of istreams. - * - **/ - -namespace avro { - -/** - * Helper class for reading bytes from buffer without worrying about - * chunk boundaries. May read from an InputBuffer or OutputBuffer. - * - **/ -class AVRO_DECL BufferReader : private boost::noncopyable -{ - - public: - - typedef detail::data_type data_type; - typedef detail::size_type size_type; - - private: - - size_type chunkRemaining() const { - return iter_->dataSize() - chunkPos_; - } - - void incrementChunk(size_type howmuch) { - bytesRemaining_ -= howmuch; - chunkPos_ += howmuch; - if(chunkPos_ == iter_->dataSize()) { - chunkPos_ = 0; - ++iter_; - } - } - - void rewind() { - iter_ = bufferImpl_->beginRead(); - bytesRemaining_ = bytes_; - chunkPos_ = 0; - } - - const data_type *addr() const { - return iter_->tellReadPos() + chunkPos_; - } - - public: - - BufferReader(const InputBuffer &buf) : - bufferImpl_(buf.pimpl_), - iter_(bufferImpl_->beginRead()), - bytes_(bufferImpl_->size()), - bytesRemaining_(bytes_), - chunkPos_(0) - { } - - BufferReader(const OutputBuffer &buf) : - bufferImpl_(buf.pimpl_), - iter_(bufferImpl_->beginRead()), - bytes_(bufferImpl_->size()), - bytesRemaining_(bytes_), - chunkPos_(0) - { } - - /** - * How many bytes are still not read from this buffer. - **/ - - size_type bytesRemaining() const { - return bytesRemaining_; - } - - /** - * Read a block of data from the front of the buffer. - **/ - - size_type bytesRead() const { - return bytes_ - bytesRemaining_; - } - - /** - * Read a block of data from the buffer. - **/ - - size_type read(data_type *data, size_type size) { - - if(size > bytesRemaining_) { - size = bytesRemaining_; - } - size_type sizeToRead = size; - - while(sizeToRead) { - const size_type toRead = std::min(sizeToRead, chunkRemaining()); - memcpy(data, addr(), toRead); - sizeToRead -= toRead; - data += toRead; - incrementChunk(toRead); - } - - return size; - } - - /** - * Read a block of data from the buffer. - **/ - - bool read(std::string &str, size_type size) { - if(size > bytesRemaining_) { - return false; - } - - if(size <= chunkRemaining()) { - fastStringRead(str, size); - } - else { - slowStringRead(str, size); - } - - return true; - } - - - /** - * Read a single value from the buffer. The value must be a "fundamental" - * type, e.g. int, float, etc. (otherwise use the other writeTo tests). - * - **/ - - template<typename T> - bool read(T &val) { - return read(val, std::is_fundamental<T>()); - } - - /** - * Skips a block of data from the buffer. - **/ - - bool skip(size_type bytes) { - bool skipped = false; - if(bytes <= bytesRemaining_) { - doSkip(bytes); - skipped = true; - } - return skipped; - } - - /** - * Seek to a position in the buffer. - **/ - - bool seek(size_type pos) { - if(pos > bytes_) { - return false; - } - - size_type toSkip = pos; - size_type curPos = bytesRead(); - // if the seek position is ahead, we can use skip to get there - if(pos >= curPos) { - toSkip -= curPos; - } - // if the seek position is ahead of the start of the chunk we can back up to - // start of the chunk - else if(pos >= (curPos - chunkPos_)) { - curPos -= chunkPos_; - bytesRemaining_ += chunkPos_; - chunkPos_ = 0; - toSkip -= curPos; - } - else { - rewind(); - } - doSkip(toSkip); - return true; - } - - bool peek(char &val) { - bool ret = (bytesRemaining_ > 0); - if(ret) { - val = *(addr()); - } - return ret; - } - - InputBuffer copyData(size_type bytes) { - if(bytes > bytesRemaining_) { - // force no copy - bytes = 0; - } - detail::BufferImpl::SharedPtr newImpl(new detail::BufferImpl); - if(bytes) { - bufferImpl_->copyData(*newImpl, iter_, chunkPos_, bytes); - doSkip(bytes); - } - return InputBuffer(newImpl); - } - - private: - - void doSkip(size_type sizeToSkip) { - - while(sizeToSkip) { - const size_type toSkip = std::min(sizeToSkip, chunkRemaining()); - sizeToSkip -= toSkip; - incrementChunk(toSkip); - } - } - - template<typename T> - bool read(T &val, const std::true_type&) - { - if(sizeof(T) > bytesRemaining_) { - return false; - } - - if (sizeof(T) <= chunkRemaining()) { - val = *(reinterpret_cast<const T*> (addr())); - incrementChunk(sizeof(T)); - } - else { - read(reinterpret_cast<data_type *>(&val), sizeof(T)); - } - return true; - } - - /// An uninstantiable function, that is if boost::is_fundamental check fails - template<typename T> +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_BufferReader_hh__ +#define avro_BufferReader_hh__ + +#include <type_traits> +#include "Buffer.hh" + +#ifdef min +#undef min +#endif +/** + * \file BufferReader.hh + * + * \brief Helper class for reading bytes from buffer in a streaming manner, + * without the overhead of istreams. + * + **/ + +namespace avro { + +/** + * Helper class for reading bytes from buffer without worrying about + * chunk boundaries. May read from an InputBuffer or OutputBuffer. + * + **/ +class AVRO_DECL BufferReader : private boost::noncopyable +{ + + public: + + typedef detail::data_type data_type; + typedef detail::size_type size_type; + + private: + + size_type chunkRemaining() const { + return iter_->dataSize() - chunkPos_; + } + + void incrementChunk(size_type howmuch) { + bytesRemaining_ -= howmuch; + chunkPos_ += howmuch; + if(chunkPos_ == iter_->dataSize()) { + chunkPos_ = 0; + ++iter_; + } + } + + void rewind() { + iter_ = bufferImpl_->beginRead(); + bytesRemaining_ = bytes_; + chunkPos_ = 0; + } + + const data_type *addr() const { + return iter_->tellReadPos() + chunkPos_; + } + + public: + + BufferReader(const InputBuffer &buf) : + bufferImpl_(buf.pimpl_), + iter_(bufferImpl_->beginRead()), + bytes_(bufferImpl_->size()), + bytesRemaining_(bytes_), + chunkPos_(0) + { } + + BufferReader(const OutputBuffer &buf) : + bufferImpl_(buf.pimpl_), + iter_(bufferImpl_->beginRead()), + bytes_(bufferImpl_->size()), + bytesRemaining_(bytes_), + chunkPos_(0) + { } + + /** + * How many bytes are still not read from this buffer. + **/ + + size_type bytesRemaining() const { + return bytesRemaining_; + } + + /** + * Read a block of data from the front of the buffer. + **/ + + size_type bytesRead() const { + return bytes_ - bytesRemaining_; + } + + /** + * Read a block of data from the buffer. + **/ + + size_type read(data_type *data, size_type size) { + + if(size > bytesRemaining_) { + size = bytesRemaining_; + } + size_type sizeToRead = size; + + while(sizeToRead) { + const size_type toRead = std::min(sizeToRead, chunkRemaining()); + memcpy(data, addr(), toRead); + sizeToRead -= toRead; + data += toRead; + incrementChunk(toRead); + } + + return size; + } + + /** + * Read a block of data from the buffer. + **/ + + bool read(std::string &str, size_type size) { + if(size > bytesRemaining_) { + return false; + } + + if(size <= chunkRemaining()) { + fastStringRead(str, size); + } + else { + slowStringRead(str, size); + } + + return true; + } + + + /** + * Read a single value from the buffer. The value must be a "fundamental" + * type, e.g. int, float, etc. (otherwise use the other writeTo tests). + * + **/ + + template<typename T> + bool read(T &val) { + return read(val, std::is_fundamental<T>()); + } + + /** + * Skips a block of data from the buffer. + **/ + + bool skip(size_type bytes) { + bool skipped = false; + if(bytes <= bytesRemaining_) { + doSkip(bytes); + skipped = true; + } + return skipped; + } + + /** + * Seek to a position in the buffer. + **/ + + bool seek(size_type pos) { + if(pos > bytes_) { + return false; + } + + size_type toSkip = pos; + size_type curPos = bytesRead(); + // if the seek position is ahead, we can use skip to get there + if(pos >= curPos) { + toSkip -= curPos; + } + // if the seek position is ahead of the start of the chunk we can back up to + // start of the chunk + else if(pos >= (curPos - chunkPos_)) { + curPos -= chunkPos_; + bytesRemaining_ += chunkPos_; + chunkPos_ = 0; + toSkip -= curPos; + } + else { + rewind(); + } + doSkip(toSkip); + return true; + } + + bool peek(char &val) { + bool ret = (bytesRemaining_ > 0); + if(ret) { + val = *(addr()); + } + return ret; + } + + InputBuffer copyData(size_type bytes) { + if(bytes > bytesRemaining_) { + // force no copy + bytes = 0; + } + detail::BufferImpl::SharedPtr newImpl(new detail::BufferImpl); + if(bytes) { + bufferImpl_->copyData(*newImpl, iter_, chunkPos_, bytes); + doSkip(bytes); + } + return InputBuffer(newImpl); + } + + private: + + void doSkip(size_type sizeToSkip) { + + while(sizeToSkip) { + const size_type toSkip = std::min(sizeToSkip, chunkRemaining()); + sizeToSkip -= toSkip; + incrementChunk(toSkip); + } + } + + template<typename T> + bool read(T &val, const std::true_type&) + { + if(sizeof(T) > bytesRemaining_) { + return false; + } + + if (sizeof(T) <= chunkRemaining()) { + val = *(reinterpret_cast<const T*> (addr())); + incrementChunk(sizeof(T)); + } + else { + read(reinterpret_cast<data_type *>(&val), sizeof(T)); + } + return true; + } + + /// An uninstantiable function, that is if boost::is_fundamental check fails + template<typename T> bool read(T &, const std::false_type&) - { - static_assert(sizeof(T) == 0, "Not a valid type to read"); - return false; - } - - void fastStringRead(std::string &str, size_type sizeToCopy) { - str.assign(addr(), sizeToCopy); - incrementChunk(sizeToCopy); - } - - void slowStringRead(std::string &str, size_type sizeToCopy) { - str.clear(); - str.reserve(sizeToCopy); - while(sizeToCopy) { - const size_type toCopy = std::min(sizeToCopy, chunkRemaining()); - str.append(addr(), toCopy); - sizeToCopy -= toCopy; - incrementChunk(toCopy); - } - } - - detail::BufferImpl::ConstSharedPtr bufferImpl_; - detail::BufferImpl::ChunkList::const_iterator iter_; - size_type bytes_; - size_type bytesRemaining_; - size_type chunkPos_; -}; - - -} // namespace - -#endif + { + static_assert(sizeof(T) == 0, "Not a valid type to read"); + return false; + } + + void fastStringRead(std::string &str, size_type sizeToCopy) { + str.assign(addr(), sizeToCopy); + incrementChunk(sizeToCopy); + } + + void slowStringRead(std::string &str, size_type sizeToCopy) { + str.clear(); + str.reserve(sizeToCopy); + while(sizeToCopy) { + const size_type toCopy = std::min(sizeToCopy, chunkRemaining()); + str.append(addr(), toCopy); + sizeToCopy -= toCopy; + incrementChunk(toCopy); + } + } + + detail::BufferImpl::ConstSharedPtr bufferImpl_; + detail::BufferImpl::ChunkList::const_iterator iter_; + size_type bytes_; + size_type bytesRemaining_; + size_type chunkPos_; +}; + + +} // namespace + +#endif diff --git a/contrib/libs/apache/avro/api/buffer/detail/BufferDetail.hh b/contrib/libs/apache/avro/api/buffer/detail/BufferDetail.hh index bbc2948f26..29a2e00b4e 100644 --- a/contrib/libs/apache/avro/api/buffer/detail/BufferDetail.hh +++ b/contrib/libs/apache/avro/api/buffer/detail/BufferDetail.hh @@ -1,555 +1,555 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_BufferDetail_hh__ -#define avro_BufferDetail_hh__ - -#include <boost/shared_ptr.hpp> -#include <boost/shared_array.hpp> -#include <boost/static_assert.hpp> -#include <boost/function.hpp> -#include <boost/utility.hpp> -#ifdef HAVE_BOOST_ASIO -#include <boost/asio/buffer.hpp> -#endif -#include <exception> -#include <cassert> -#include <deque> - -/** - * \file BufferDetail.hh - * - * \brief The implementation details for the Buffer class. - * - **/ - -namespace avro { - -namespace detail { - -typedef char data_type; -typedef size_t size_type; -#ifdef HAVE_BOOST_ASIO -typedef boost::asio::const_buffer ConstAsioBuffer; -typedef boost::asio::mutable_buffer MutableAsioBuffer; -#endif - -/// The size in bytes for blocks backing buffer chunks. -const size_type kMinBlockSize = 4096; -const size_type kMaxBlockSize = 16384; -const size_type kDefaultBlockSize = kMinBlockSize; - -typedef boost::function<void(void)> free_func; - -/** - * Simple class to hold a functor that executes on delete - **/ -class CallOnDestroy { - public: - CallOnDestroy(const free_func &func) : func_(func) - { } - ~CallOnDestroy() { - if (func_) { - func_(); - } - } - private: - free_func func_; -}; - -/** - * \brief A chunk is the building block for buffers. - * - * A chunk is backed by a memory block, and internally it maintains information - * about which area of the block it may use, and the portion of this area that - * contains valid data. More than one chunk may share the same underlying - * block, but the areas should never overlap. Chunk holds a shared pointer to - * an array of bytes so that shared blocks are reference counted. - * - * When a chunk is copied, the copy shares the same underlying buffer, but the - * copy receives its own copies of the start/cursor/end pointers, so each copy - * can be manipulated independently. This allows different buffers to share - * the same non-overlapping parts of a chunk, or even overlapping parts of a - * chunk if the situation arises. - * - **/ - -class Chunk -{ - - public: - - typedef boost::shared_ptr<Chunk> SharedPtr; - - /// Default constructor, allocates a new underlying block for this chunk. - Chunk(size_type size) : - underlyingBlock_(new data_type[size]), - readPos_(underlyingBlock_.get()), - writePos_(readPos_), - endPos_(readPos_ + size) - { } - - /// Foreign buffer constructor, uses the supplied data for this chunk, and - /// only for reading. - Chunk(const data_type *data, size_type size, const free_func &func) : - callOnDestroy_(new CallOnDestroy(func)), - readPos_(const_cast<data_type *>(data)), - writePos_(readPos_ + size), - endPos_(writePos_) - { } - - private: - // reference counted object will call a functor when it's destroyed - boost::shared_ptr<CallOnDestroy> callOnDestroy_; - - public: - - /// Remove readable bytes from the front of the chunk by advancing the - /// chunk start position. - void truncateFront(size_type howMuch) { - readPos_ += howMuch; - assert(readPos_ <= writePos_); - } - - /// Remove readable bytes from the back of the chunk by moving the - /// chunk cursor position. - void truncateBack(size_type howMuch) { - writePos_ -= howMuch; - assert(readPos_ <= writePos_); - } - - /// Tell the position the next byte may be written to. - data_type *tellWritePos() const { - return writePos_; - } - - /// Tell the position of the first byte containing valid data. - const data_type *tellReadPos() const { - return readPos_; - } - - /// After a write operation, increment the write position. - void incrementCursor(size_type howMuch) { - writePos_ += howMuch; - assert(writePos_ <= endPos_); - } - - /// Tell how many bytes of data were written to this chunk. - size_type dataSize() const { - return (writePos_ - readPos_); - } - - /// Tell how many bytes this chunk has available to write to. - size_type freeSize() const { - return (endPos_ - writePos_); - } - - /// Tell how many bytes of data this chunk can hold (used and free). - size_type capacity() const { - return (endPos_ - readPos_); - } - - private: - - friend bool operator==(const Chunk &lhs, const Chunk &rhs); - friend bool operator!=(const Chunk &lhs, const Chunk &rhs); - - // more than one buffer can share an underlying block, so use SharedPtr - boost::shared_array<data_type> underlyingBlock_; - - data_type *readPos_; ///< The first readable byte in the block - data_type *writePos_; ///< The end of written data and start of free space - data_type *endPos_; ///< Marks the end of the usable block area -}; - -/** - * Compare underlying buffers and return true if they are equal - **/ -inline bool operator==(const Chunk &lhs, const Chunk &rhs) { - return lhs.underlyingBlock_ == rhs.underlyingBlock_; -} - -/** - * Compare underlying buffers and return true if they are unequal - **/ -inline bool operator!=(const Chunk &lhs, const Chunk &rhs) { - return lhs.underlyingBlock_ != rhs.underlyingBlock_; -} - - -/** - * \brief Implementation details for Buffer class - * - * Internally, BufferImpl keeps two lists of chunks, one list consists entirely of - * chunks containing data, and one list which contains chunks with free space. - * - * - */ - -class BufferImpl : boost::noncopyable -{ - - /// Add a new chunk to the list of chunks for this buffer, growing the - /// buffer by the default block size. - void allocChunkChecked(size_type size = kDefaultBlockSize) - { - writeChunks_.push_back(Chunk(size)); - freeSpace_ += writeChunks_.back().freeSize(); - } - - /// Add a new chunk to the list of chunks for this buffer, growing the - /// buffer by the requested size, but within the range of a minimum and - /// maximum. - void allocChunk(size_type size) - { - if(size < kMinBlockSize) { - size = kMinBlockSize; - } - else if (size > kMaxBlockSize) { - size = kMaxBlockSize; - } - allocChunkChecked(size); - } - - /// Update the state of the chunks after a write operation. This function - /// ensures the chunk states are consistent with the write. - void postWrite(size_type size) - { - - // precondition to this function is that the writeChunk_.front() - // contains the data that was just written, so make sure writeChunks_ - // is not empty: - - assert(size <= freeSpace_ && !writeChunks_.empty()); - - // This is probably the one tricky part of BufferImpl. The data that - // was written now exists in writeChunks_.front(). Now we must make - // sure that same data exists in readChunks_.back(). - // - // There are two cases: - // - // 1. readChunks_.last() and writeChunk_.front() refer to the same - // underlying block, in which case they both just need their cursor - // updated to reflect the new state. - // - // 2. readChunk_.last() is not the same block as writeChunks_.front(), - // in which case it should be, since the writeChunk.front() contains - // the next bit of data that will be appended to readChunks_, and - // therefore needs to be copied there so we can proceed with updating - // their state. - // - - // if readChunks_ is not the same as writeChunks_.front(), make a copy - // of it there - - if(readChunks_.empty() || (readChunks_.back() != writeChunks_.front())) { - const Chunk &curChunk = writeChunks_.front(); - readChunks_.push_back(curChunk); - - // Any data that existed in the write chunk previously doesn't - // belong to this buffer (otherwise it would have already been - // added to the readChunk_ list). Here, adjust the start of the - // readChunk to begin after any data already existing in curChunk - - readChunks_.back().truncateFront( curChunk.dataSize()); - } - - assert(readChunks_.back().freeSize() == writeChunks_.front().freeSize()); - - // update the states of both readChunks_ and writeChunks_ to indicate that they are - // holding the new data - - readChunks_.back().incrementCursor(size); - writeChunks_.front().incrementCursor(size); - size_ += size; - freeSpace_ -= size; - - // if there is no more free space in writeChunks_, the next write cannot use - // it, so dispose of it now - - if(writeChunks_.front().freeSize() == 0) { - writeChunks_.pop_front(); - } - } - - public: - - typedef std::deque<Chunk> ChunkList; - typedef boost::shared_ptr<BufferImpl> SharedPtr; - typedef boost::shared_ptr<const BufferImpl> ConstSharedPtr; - - /// Default constructor, creates a buffer without any chunks - BufferImpl() : - freeSpace_(0), - size_(0) - { } - - /// Copy constructor, gets a copy of all the chunks with data. - explicit BufferImpl(const BufferImpl &src) : - readChunks_(src.readChunks_), - freeSpace_(0), - size_(src.size_) - { } - - /// Amount of data held in this buffer. - size_type size() const { - return size_; - } - - /// Capacity that may be written before the buffer must allocate more memory. - size_type freeSpace() const { - return freeSpace_; - } - - /// Add enough free chunks to make the reservation size available. - /// Actual amount may be more (rounded up to next chunk). - void reserveFreeSpace(size_type reserveSize) { - while(freeSpace_ < reserveSize) { - allocChunk(reserveSize - freeSpace_); - } - } - - /// Return the chunk avro's begin iterator for reading. - ChunkList::const_iterator beginRead() const { - return readChunks_.begin(); - } - - /// Return the chunk avro's end iterator for reading. - ChunkList::const_iterator endRead() const { - return readChunks_.end(); - } - - /// Return the chunk avro's begin iterator for writing. - ChunkList::const_iterator beginWrite() const { - return writeChunks_.begin(); - } - - /// Return the chunk avro's end iterator for writing. - ChunkList::const_iterator endWrite() const { - return writeChunks_.end(); - } - - /// Write a single value to buffer, add a new chunk if necessary. - template<typename T> - void writeTo(T val, const std::true_type&) - { - if(freeSpace_ && (sizeof(T) <= writeChunks_.front().freeSize())) { - // fast path, there's enough room in the writeable chunk to just - // straight out copy it - *(reinterpret_cast <T*> ( writeChunks_.front().tellWritePos()) ) = val; - postWrite(sizeof(T)); - } - else { - // need to fixup chunks first, so use the regular memcpy - // writeTo method - writeTo(reinterpret_cast<data_type*>(&val), sizeof(T)); - } - } - - /// An uninstantiable function, this is if boost::is_fundamental check fails, - /// and will compile-time assert. - template<typename T> +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_BufferDetail_hh__ +#define avro_BufferDetail_hh__ + +#include <boost/shared_ptr.hpp> +#include <boost/shared_array.hpp> +#include <boost/static_assert.hpp> +#include <boost/function.hpp> +#include <boost/utility.hpp> +#ifdef HAVE_BOOST_ASIO +#include <boost/asio/buffer.hpp> +#endif +#include <exception> +#include <cassert> +#include <deque> + +/** + * \file BufferDetail.hh + * + * \brief The implementation details for the Buffer class. + * + **/ + +namespace avro { + +namespace detail { + +typedef char data_type; +typedef size_t size_type; +#ifdef HAVE_BOOST_ASIO +typedef boost::asio::const_buffer ConstAsioBuffer; +typedef boost::asio::mutable_buffer MutableAsioBuffer; +#endif + +/// The size in bytes for blocks backing buffer chunks. +const size_type kMinBlockSize = 4096; +const size_type kMaxBlockSize = 16384; +const size_type kDefaultBlockSize = kMinBlockSize; + +typedef boost::function<void(void)> free_func; + +/** + * Simple class to hold a functor that executes on delete + **/ +class CallOnDestroy { + public: + CallOnDestroy(const free_func &func) : func_(func) + { } + ~CallOnDestroy() { + if (func_) { + func_(); + } + } + private: + free_func func_; +}; + +/** + * \brief A chunk is the building block for buffers. + * + * A chunk is backed by a memory block, and internally it maintains information + * about which area of the block it may use, and the portion of this area that + * contains valid data. More than one chunk may share the same underlying + * block, but the areas should never overlap. Chunk holds a shared pointer to + * an array of bytes so that shared blocks are reference counted. + * + * When a chunk is copied, the copy shares the same underlying buffer, but the + * copy receives its own copies of the start/cursor/end pointers, so each copy + * can be manipulated independently. This allows different buffers to share + * the same non-overlapping parts of a chunk, or even overlapping parts of a + * chunk if the situation arises. + * + **/ + +class Chunk +{ + + public: + + typedef boost::shared_ptr<Chunk> SharedPtr; + + /// Default constructor, allocates a new underlying block for this chunk. + Chunk(size_type size) : + underlyingBlock_(new data_type[size]), + readPos_(underlyingBlock_.get()), + writePos_(readPos_), + endPos_(readPos_ + size) + { } + + /// Foreign buffer constructor, uses the supplied data for this chunk, and + /// only for reading. + Chunk(const data_type *data, size_type size, const free_func &func) : + callOnDestroy_(new CallOnDestroy(func)), + readPos_(const_cast<data_type *>(data)), + writePos_(readPos_ + size), + endPos_(writePos_) + { } + + private: + // reference counted object will call a functor when it's destroyed + boost::shared_ptr<CallOnDestroy> callOnDestroy_; + + public: + + /// Remove readable bytes from the front of the chunk by advancing the + /// chunk start position. + void truncateFront(size_type howMuch) { + readPos_ += howMuch; + assert(readPos_ <= writePos_); + } + + /// Remove readable bytes from the back of the chunk by moving the + /// chunk cursor position. + void truncateBack(size_type howMuch) { + writePos_ -= howMuch; + assert(readPos_ <= writePos_); + } + + /// Tell the position the next byte may be written to. + data_type *tellWritePos() const { + return writePos_; + } + + /// Tell the position of the first byte containing valid data. + const data_type *tellReadPos() const { + return readPos_; + } + + /// After a write operation, increment the write position. + void incrementCursor(size_type howMuch) { + writePos_ += howMuch; + assert(writePos_ <= endPos_); + } + + /// Tell how many bytes of data were written to this chunk. + size_type dataSize() const { + return (writePos_ - readPos_); + } + + /// Tell how many bytes this chunk has available to write to. + size_type freeSize() const { + return (endPos_ - writePos_); + } + + /// Tell how many bytes of data this chunk can hold (used and free). + size_type capacity() const { + return (endPos_ - readPos_); + } + + private: + + friend bool operator==(const Chunk &lhs, const Chunk &rhs); + friend bool operator!=(const Chunk &lhs, const Chunk &rhs); + + // more than one buffer can share an underlying block, so use SharedPtr + boost::shared_array<data_type> underlyingBlock_; + + data_type *readPos_; ///< The first readable byte in the block + data_type *writePos_; ///< The end of written data and start of free space + data_type *endPos_; ///< Marks the end of the usable block area +}; + +/** + * Compare underlying buffers and return true if they are equal + **/ +inline bool operator==(const Chunk &lhs, const Chunk &rhs) { + return lhs.underlyingBlock_ == rhs.underlyingBlock_; +} + +/** + * Compare underlying buffers and return true if they are unequal + **/ +inline bool operator!=(const Chunk &lhs, const Chunk &rhs) { + return lhs.underlyingBlock_ != rhs.underlyingBlock_; +} + + +/** + * \brief Implementation details for Buffer class + * + * Internally, BufferImpl keeps two lists of chunks, one list consists entirely of + * chunks containing data, and one list which contains chunks with free space. + * + * + */ + +class BufferImpl : boost::noncopyable +{ + + /// Add a new chunk to the list of chunks for this buffer, growing the + /// buffer by the default block size. + void allocChunkChecked(size_type size = kDefaultBlockSize) + { + writeChunks_.push_back(Chunk(size)); + freeSpace_ += writeChunks_.back().freeSize(); + } + + /// Add a new chunk to the list of chunks for this buffer, growing the + /// buffer by the requested size, but within the range of a minimum and + /// maximum. + void allocChunk(size_type size) + { + if(size < kMinBlockSize) { + size = kMinBlockSize; + } + else if (size > kMaxBlockSize) { + size = kMaxBlockSize; + } + allocChunkChecked(size); + } + + /// Update the state of the chunks after a write operation. This function + /// ensures the chunk states are consistent with the write. + void postWrite(size_type size) + { + + // precondition to this function is that the writeChunk_.front() + // contains the data that was just written, so make sure writeChunks_ + // is not empty: + + assert(size <= freeSpace_ && !writeChunks_.empty()); + + // This is probably the one tricky part of BufferImpl. The data that + // was written now exists in writeChunks_.front(). Now we must make + // sure that same data exists in readChunks_.back(). + // + // There are two cases: + // + // 1. readChunks_.last() and writeChunk_.front() refer to the same + // underlying block, in which case they both just need their cursor + // updated to reflect the new state. + // + // 2. readChunk_.last() is not the same block as writeChunks_.front(), + // in which case it should be, since the writeChunk.front() contains + // the next bit of data that will be appended to readChunks_, and + // therefore needs to be copied there so we can proceed with updating + // their state. + // + + // if readChunks_ is not the same as writeChunks_.front(), make a copy + // of it there + + if(readChunks_.empty() || (readChunks_.back() != writeChunks_.front())) { + const Chunk &curChunk = writeChunks_.front(); + readChunks_.push_back(curChunk); + + // Any data that existed in the write chunk previously doesn't + // belong to this buffer (otherwise it would have already been + // added to the readChunk_ list). Here, adjust the start of the + // readChunk to begin after any data already existing in curChunk + + readChunks_.back().truncateFront( curChunk.dataSize()); + } + + assert(readChunks_.back().freeSize() == writeChunks_.front().freeSize()); + + // update the states of both readChunks_ and writeChunks_ to indicate that they are + // holding the new data + + readChunks_.back().incrementCursor(size); + writeChunks_.front().incrementCursor(size); + size_ += size; + freeSpace_ -= size; + + // if there is no more free space in writeChunks_, the next write cannot use + // it, so dispose of it now + + if(writeChunks_.front().freeSize() == 0) { + writeChunks_.pop_front(); + } + } + + public: + + typedef std::deque<Chunk> ChunkList; + typedef boost::shared_ptr<BufferImpl> SharedPtr; + typedef boost::shared_ptr<const BufferImpl> ConstSharedPtr; + + /// Default constructor, creates a buffer without any chunks + BufferImpl() : + freeSpace_(0), + size_(0) + { } + + /// Copy constructor, gets a copy of all the chunks with data. + explicit BufferImpl(const BufferImpl &src) : + readChunks_(src.readChunks_), + freeSpace_(0), + size_(src.size_) + { } + + /// Amount of data held in this buffer. + size_type size() const { + return size_; + } + + /// Capacity that may be written before the buffer must allocate more memory. + size_type freeSpace() const { + return freeSpace_; + } + + /// Add enough free chunks to make the reservation size available. + /// Actual amount may be more (rounded up to next chunk). + void reserveFreeSpace(size_type reserveSize) { + while(freeSpace_ < reserveSize) { + allocChunk(reserveSize - freeSpace_); + } + } + + /// Return the chunk avro's begin iterator for reading. + ChunkList::const_iterator beginRead() const { + return readChunks_.begin(); + } + + /// Return the chunk avro's end iterator for reading. + ChunkList::const_iterator endRead() const { + return readChunks_.end(); + } + + /// Return the chunk avro's begin iterator for writing. + ChunkList::const_iterator beginWrite() const { + return writeChunks_.begin(); + } + + /// Return the chunk avro's end iterator for writing. + ChunkList::const_iterator endWrite() const { + return writeChunks_.end(); + } + + /// Write a single value to buffer, add a new chunk if necessary. + template<typename T> + void writeTo(T val, const std::true_type&) + { + if(freeSpace_ && (sizeof(T) <= writeChunks_.front().freeSize())) { + // fast path, there's enough room in the writeable chunk to just + // straight out copy it + *(reinterpret_cast <T*> ( writeChunks_.front().tellWritePos()) ) = val; + postWrite(sizeof(T)); + } + else { + // need to fixup chunks first, so use the regular memcpy + // writeTo method + writeTo(reinterpret_cast<data_type*>(&val), sizeof(T)); + } + } + + /// An uninstantiable function, this is if boost::is_fundamental check fails, + /// and will compile-time assert. + template<typename T> void writeTo(T, const std::false_type&) - { - BOOST_STATIC_ASSERT(sizeof(T)==0); - } - - /// Write a block of data to the buffer, adding new chunks if necessary. - size_type writeTo(const data_type *data, size_type size) - { - size_type bytesLeft = size; - while(bytesLeft) { - - if(freeSpace_ == 0) { - allocChunkChecked(); - } - - Chunk &chunk = writeChunks_.front(); - size_type toCopy = std::min<size_type>(chunk.freeSize(), bytesLeft); - assert(toCopy); - memcpy(chunk.tellWritePos(), data, toCopy); - postWrite(toCopy); - data += toCopy; - bytesLeft -= toCopy; - } - return size; - } - - /// Update internal status of chunks after data is written using iterator. - size_type wroteTo(size_type size) - { - assert(size <= freeSpace_); - size_type bytesLeft = size; - while (bytesLeft) { - - Chunk &chunk = writeChunks_.front(); - size_type wrote = std::min<size_type>(chunk.freeSize(), bytesLeft); - assert(wrote); - postWrite(wrote); - bytesLeft -= wrote; - } - return size; - } - - /// Append the chunks that have data in src to this buffer - void append(const BufferImpl &src) { - std::copy(src.readChunks_.begin(), src.readChunks_.end(), std::back_inserter(readChunks_)); - size_ += src.size_; - } - - /// Remove all the chunks that contain data from this buffer. - void discardData() { - readChunks_.clear(); - size_ = 0; - } - - /// Remove the specified amount of data from the chunks, starting at the front. - void discardData(size_type bytes) - { - assert(bytes && bytes <= size_); - - size_type bytesToDiscard = bytes; - while( bytesToDiscard ) { - - size_t currentSize = readChunks_.front().dataSize(); - - // see if entire chunk is discarded - if(currentSize <= bytesToDiscard) { - readChunks_.pop_front(); - bytesToDiscard -= currentSize; - } - else { - readChunks_.front().truncateFront(bytesToDiscard); - bytesToDiscard = 0; - } - } - - size_ -= bytes; - } - - /// Remove the specified amount of data from the chunks, moving the - /// data to dest's chunks - void extractData(BufferImpl &dest, size_type bytes) - { - assert(bytes && bytes <= size_); - - size_type bytesToExtract = bytes; - while( bytesToExtract ) { - - size_t currentSize = readChunks_.front().dataSize(); - dest.readChunks_.push_back(readChunks_.front()); - - // see if entire chunk was extracted - if(currentSize <= bytesToExtract) { - readChunks_.pop_front(); - bytesToExtract -= currentSize; - } - else { - readChunks_.front().truncateFront(bytesToExtract); - size_t excess = currentSize - bytesToExtract; - dest.readChunks_.back().truncateBack(excess); - bytesToExtract = 0; - } - } - - size_ -= bytes; - dest.size_ += bytes; - } - - /// Move data from this to the destination, leaving this buffer without data - void extractData(BufferImpl &dest) - { - assert(dest.readChunks_.empty()); - dest.readChunks_.swap(readChunks_); - dest.size_ = size_; - size_ = 0; - } - - /// Copy data to a different buffer by copying the chunks. It's - /// a bit like extract, but without modifying the source buffer. - void copyData(BufferImpl &dest, - ChunkList::const_iterator iter, - size_type offset, - size_type bytes) const - { - // now we are positioned to start the copying, copy as many - // chunks as we need, the first chunk may have a non-zero offset - // if the data to copy is not at the start of the chunk - size_type copied = 0; - while(copied < bytes) { - - dest.readChunks_.push_back(*iter); - - // offset only applies in the first chunk, - // all subsequent chunks are copied from the start - dest.readChunks_.back().truncateFront(offset); - offset = 0; - - copied += dest.readChunks_.back().dataSize(); - ++iter; - } - - // if the last chunk copied has more bytes than we need, truncate it - size_type excess = copied - bytes; - dest.readChunks_.back().truncateBack(excess); - - dest.size_ += bytes; - } - - /// The number of chunks containing data. Used for debugging. - int numDataChunks() const { - return readChunks_.size(); - } - - /// The number of chunks containing free space (note that an entire chunk - /// may not be free). Used for debugging. - int numFreeChunks() const { - return writeChunks_.size(); - } - - /// Add unmanaged data to the buffer. The buffer will not automatically - /// free the data, but it will call the supplied function when the data is - /// no longer referenced by the buffer (or copies of the buffer). - void appendForeignData(const data_type *data, size_type size, const free_func &func) { - readChunks_.push_back(Chunk(data, size, func)); - size_ += size; - } - - private: - - /// Assignment not allowed - BufferImpl& operator=(const BufferImpl &src); - /* { - readChunks_.assign(src.readChunks_.begin(), src.readChunks_.end()); - size_ = src.size(); - return *this; - } */ - - ChunkList readChunks_; ///< chunks of this buffer containing data - ChunkList writeChunks_; ///< chunks of this buffer containing free space - - size_type freeSpace_; ///< capacity of buffer before allocation required - size_type size_; ///< amount of data in buffer - -}; - -} // detail namespace - -} // namespace - -#endif + { + BOOST_STATIC_ASSERT(sizeof(T)==0); + } + + /// Write a block of data to the buffer, adding new chunks if necessary. + size_type writeTo(const data_type *data, size_type size) + { + size_type bytesLeft = size; + while(bytesLeft) { + + if(freeSpace_ == 0) { + allocChunkChecked(); + } + + Chunk &chunk = writeChunks_.front(); + size_type toCopy = std::min<size_type>(chunk.freeSize(), bytesLeft); + assert(toCopy); + memcpy(chunk.tellWritePos(), data, toCopy); + postWrite(toCopy); + data += toCopy; + bytesLeft -= toCopy; + } + return size; + } + + /// Update internal status of chunks after data is written using iterator. + size_type wroteTo(size_type size) + { + assert(size <= freeSpace_); + size_type bytesLeft = size; + while (bytesLeft) { + + Chunk &chunk = writeChunks_.front(); + size_type wrote = std::min<size_type>(chunk.freeSize(), bytesLeft); + assert(wrote); + postWrite(wrote); + bytesLeft -= wrote; + } + return size; + } + + /// Append the chunks that have data in src to this buffer + void append(const BufferImpl &src) { + std::copy(src.readChunks_.begin(), src.readChunks_.end(), std::back_inserter(readChunks_)); + size_ += src.size_; + } + + /// Remove all the chunks that contain data from this buffer. + void discardData() { + readChunks_.clear(); + size_ = 0; + } + + /// Remove the specified amount of data from the chunks, starting at the front. + void discardData(size_type bytes) + { + assert(bytes && bytes <= size_); + + size_type bytesToDiscard = bytes; + while( bytesToDiscard ) { + + size_t currentSize = readChunks_.front().dataSize(); + + // see if entire chunk is discarded + if(currentSize <= bytesToDiscard) { + readChunks_.pop_front(); + bytesToDiscard -= currentSize; + } + else { + readChunks_.front().truncateFront(bytesToDiscard); + bytesToDiscard = 0; + } + } + + size_ -= bytes; + } + + /// Remove the specified amount of data from the chunks, moving the + /// data to dest's chunks + void extractData(BufferImpl &dest, size_type bytes) + { + assert(bytes && bytes <= size_); + + size_type bytesToExtract = bytes; + while( bytesToExtract ) { + + size_t currentSize = readChunks_.front().dataSize(); + dest.readChunks_.push_back(readChunks_.front()); + + // see if entire chunk was extracted + if(currentSize <= bytesToExtract) { + readChunks_.pop_front(); + bytesToExtract -= currentSize; + } + else { + readChunks_.front().truncateFront(bytesToExtract); + size_t excess = currentSize - bytesToExtract; + dest.readChunks_.back().truncateBack(excess); + bytesToExtract = 0; + } + } + + size_ -= bytes; + dest.size_ += bytes; + } + + /// Move data from this to the destination, leaving this buffer without data + void extractData(BufferImpl &dest) + { + assert(dest.readChunks_.empty()); + dest.readChunks_.swap(readChunks_); + dest.size_ = size_; + size_ = 0; + } + + /// Copy data to a different buffer by copying the chunks. It's + /// a bit like extract, but without modifying the source buffer. + void copyData(BufferImpl &dest, + ChunkList::const_iterator iter, + size_type offset, + size_type bytes) const + { + // now we are positioned to start the copying, copy as many + // chunks as we need, the first chunk may have a non-zero offset + // if the data to copy is not at the start of the chunk + size_type copied = 0; + while(copied < bytes) { + + dest.readChunks_.push_back(*iter); + + // offset only applies in the first chunk, + // all subsequent chunks are copied from the start + dest.readChunks_.back().truncateFront(offset); + offset = 0; + + copied += dest.readChunks_.back().dataSize(); + ++iter; + } + + // if the last chunk copied has more bytes than we need, truncate it + size_type excess = copied - bytes; + dest.readChunks_.back().truncateBack(excess); + + dest.size_ += bytes; + } + + /// The number of chunks containing data. Used for debugging. + int numDataChunks() const { + return readChunks_.size(); + } + + /// The number of chunks containing free space (note that an entire chunk + /// may not be free). Used for debugging. + int numFreeChunks() const { + return writeChunks_.size(); + } + + /// Add unmanaged data to the buffer. The buffer will not automatically + /// free the data, but it will call the supplied function when the data is + /// no longer referenced by the buffer (or copies of the buffer). + void appendForeignData(const data_type *data, size_type size, const free_func &func) { + readChunks_.push_back(Chunk(data, size, func)); + size_ += size; + } + + private: + + /// Assignment not allowed + BufferImpl& operator=(const BufferImpl &src); + /* { + readChunks_.assign(src.readChunks_.begin(), src.readChunks_.end()); + size_ = src.size(); + return *this; + } */ + + ChunkList readChunks_; ///< chunks of this buffer containing data + ChunkList writeChunks_; ///< chunks of this buffer containing free space + + size_type freeSpace_; ///< capacity of buffer before allocation required + size_type size_; ///< amount of data in buffer + +}; + +} // detail namespace + +} // namespace + +#endif diff --git a/contrib/libs/apache/avro/api/buffer/detail/BufferDetailIterator.hh b/contrib/libs/apache/avro/api/buffer/detail/BufferDetailIterator.hh index ad1dee6fe2..c05f219a64 100644 --- a/contrib/libs/apache/avro/api/buffer/detail/BufferDetailIterator.hh +++ b/contrib/libs/apache/avro/api/buffer/detail/BufferDetailIterator.hh @@ -1,230 +1,230 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_BufferDetailIterator_hh__ -#define avro_BufferDetailIterator_hh__ - -#include "BufferDetail.hh" - -/** - * \file BufferDetailIterator.hh - * - * \brief The implementation details for the Buffer iterators. - **/ - -namespace avro { - -namespace detail { - -/** - * \brief Implements conversion from a chunk to asio::const_buffer - * - * Iterators for an InputBuffer will iterate over the avro of chunks, so - * internally they contain an iterator. But the iterator needs to be - * convertable to an asio buffer for use in boost::asio functions. This class - * wraps the iterator with a cast operator to do this conversion. - **/ - -struct InputIteratorHelper -{ - /// Construct a helper with an unnassigned iterator. - InputIteratorHelper() : - iter_() - {} - - /// Construct a helper with an iterator. - InputIteratorHelper(const BufferImpl::ChunkList::const_iterator &iter) : - iter_(iter) - {} - - /// The location of valid data in this chunk. - const data_type *data() const { - return iter_->tellReadPos(); - } - - /// The size of valid data in this chunk. - size_type size() const { - return iter_->dataSize(); - } - - /// Conversion operator. It doesn't check for null, because the only - /// the only time the chunk should be null is when it's the iterator - /// end(), which should never be dereferenced anyway. -#ifdef HAVE_BOOST_ASIO - operator ConstAsioBuffer() const { - return ConstAsioBuffer(data(), size()); - } -#endif - - BufferImpl::ChunkList::const_iterator iter_; ///< the current iterator -}; - -/** - * \brief Implements conversion from a chunk to asio::buffer - * - * Iterators for an OutputBuffer will iterate over the avro of chunks, so - * internally they contain an iterator. But the iterator needs to be - * convertable to an asio buffer for use in boost::asio functions. This class - * wraps the iterator with a cast operator to do this conversion. - */ - -struct OutputIteratorHelper -{ - /// Construct a helper with an unnassigned iterator. - OutputIteratorHelper() : - iter_() - {} - - /// Construct a helper with an iterator. - OutputIteratorHelper(const BufferImpl::ChunkList::const_iterator &iter) : - iter_(iter) - {} - - /// The location of the first writable byte in this chunk. - data_type *data() const { - return iter_->tellWritePos(); - } - - /// The size of area that can be written in this chunk. - size_type size() const { - return iter_->freeSize(); - } - - /// Conversion operator. It doesn't check for null, because the only - /// the only time the chunk should be null is when it's the iterator - /// end(), which should never be dereferenced anyway. -#ifdef HAVE_BOOST_ASIO - operator MutableAsioBuffer() const { - return MutableAsioBuffer(data(), size()); - } -#endif - - BufferImpl::ChunkList::const_iterator iter_; ///< the current iterator -}; - -/** - * \brief Implements the iterator for Buffer, that iterates through the - * buffer's chunks. - **/ - -template<typename Helper> -class BufferIterator -{ - - public: - - typedef BufferIterator<Helper> this_type; - - /** - * @name Typedefs - * - * STL iterators define the following declarations. According to - * boost::asio documentation, the library expects the iterator to be - * bidirectional, however this implements only the forward iterator type. - * So far this has not created any problems with asio, but may change if - * future versions of the asio require it. - **/ - - //@{ - typedef std::forward_iterator_tag iterator_category; // this is a lie to appease asio - typedef Helper value_type; - typedef std::ptrdiff_t difference_type; - typedef value_type* pointer; - typedef value_type& reference; - //@} - - /// Construct an unitialized iterator. - BufferIterator() : - helper_() - { } - - /* The default implementations are good here - /// Copy constructor. - BufferIterator(const BufferIterator &src) : - helper_(src.helper_) - { } - /// Assignment. - this_type& operator= (const this_type &rhs) { - helper_ = rhs.helper_; - return *this; - } - */ - - /// Construct iterator at the position in the buffer's chunk list. - explicit BufferIterator(BufferImpl::ChunkList::const_iterator iter) : - helper_(iter) - { } - - /// Dereference iterator, returns InputIteratorHelper or OutputIteratorHelper wrapper. - reference operator *() { - return helper_; - } - - /// Dereference iterator, returns const InputIteratorHelper or OutputIteratorHelper wrapper. - const value_type &operator *() const { - return helper_; - } - - /// Dereference iterator, returns InputIteratorHelper or OutputIteratorHelper wrapper. - pointer operator->() { - return &helper_; - } - - /// Dereference iterator, returns const InputIteratorHelper or OutputIteratorHelper wrapper. - const value_type *operator->() const { - return &helper_; - } - - /// Increment to next chunk in list, or to end() iterator. - this_type& operator++() - { - ++helper_.iter_; - return *this; - } - - /// Increment to next chunk in list, or to end() iterator. - this_type operator++(int) - { - this_type ret = *this; - ++helper_.iter_; - return ret; - } - - /// True if iterators point to same chunks. - bool operator==(const this_type &rhs) const { - return (helper_.iter_ == rhs.helper_.iter_); - } - - /// True if iterators point to different chunks. - bool operator!=(const this_type &rhs) const { - return (helper_.iter_ != rhs.helper_.iter_); - } - - private: - - Helper helper_; -}; - -typedef BufferIterator<InputIteratorHelper> InputBufferIterator; -typedef BufferIterator<OutputIteratorHelper> OutputBufferIterator; - -} // detail namespace - -} // namespace - -#endif +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_BufferDetailIterator_hh__ +#define avro_BufferDetailIterator_hh__ + +#include "BufferDetail.hh" + +/** + * \file BufferDetailIterator.hh + * + * \brief The implementation details for the Buffer iterators. + **/ + +namespace avro { + +namespace detail { + +/** + * \brief Implements conversion from a chunk to asio::const_buffer + * + * Iterators for an InputBuffer will iterate over the avro of chunks, so + * internally they contain an iterator. But the iterator needs to be + * convertable to an asio buffer for use in boost::asio functions. This class + * wraps the iterator with a cast operator to do this conversion. + **/ + +struct InputIteratorHelper +{ + /// Construct a helper with an unnassigned iterator. + InputIteratorHelper() : + iter_() + {} + + /// Construct a helper with an iterator. + InputIteratorHelper(const BufferImpl::ChunkList::const_iterator &iter) : + iter_(iter) + {} + + /// The location of valid data in this chunk. + const data_type *data() const { + return iter_->tellReadPos(); + } + + /// The size of valid data in this chunk. + size_type size() const { + return iter_->dataSize(); + } + + /// Conversion operator. It doesn't check for null, because the only + /// the only time the chunk should be null is when it's the iterator + /// end(), which should never be dereferenced anyway. +#ifdef HAVE_BOOST_ASIO + operator ConstAsioBuffer() const { + return ConstAsioBuffer(data(), size()); + } +#endif + + BufferImpl::ChunkList::const_iterator iter_; ///< the current iterator +}; + +/** + * \brief Implements conversion from a chunk to asio::buffer + * + * Iterators for an OutputBuffer will iterate over the avro of chunks, so + * internally they contain an iterator. But the iterator needs to be + * convertable to an asio buffer for use in boost::asio functions. This class + * wraps the iterator with a cast operator to do this conversion. + */ + +struct OutputIteratorHelper +{ + /// Construct a helper with an unnassigned iterator. + OutputIteratorHelper() : + iter_() + {} + + /// Construct a helper with an iterator. + OutputIteratorHelper(const BufferImpl::ChunkList::const_iterator &iter) : + iter_(iter) + {} + + /// The location of the first writable byte in this chunk. + data_type *data() const { + return iter_->tellWritePos(); + } + + /// The size of area that can be written in this chunk. + size_type size() const { + return iter_->freeSize(); + } + + /// Conversion operator. It doesn't check for null, because the only + /// the only time the chunk should be null is when it's the iterator + /// end(), which should never be dereferenced anyway. +#ifdef HAVE_BOOST_ASIO + operator MutableAsioBuffer() const { + return MutableAsioBuffer(data(), size()); + } +#endif + + BufferImpl::ChunkList::const_iterator iter_; ///< the current iterator +}; + +/** + * \brief Implements the iterator for Buffer, that iterates through the + * buffer's chunks. + **/ + +template<typename Helper> +class BufferIterator +{ + + public: + + typedef BufferIterator<Helper> this_type; + + /** + * @name Typedefs + * + * STL iterators define the following declarations. According to + * boost::asio documentation, the library expects the iterator to be + * bidirectional, however this implements only the forward iterator type. + * So far this has not created any problems with asio, but may change if + * future versions of the asio require it. + **/ + + //@{ + typedef std::forward_iterator_tag iterator_category; // this is a lie to appease asio + typedef Helper value_type; + typedef std::ptrdiff_t difference_type; + typedef value_type* pointer; + typedef value_type& reference; + //@} + + /// Construct an unitialized iterator. + BufferIterator() : + helper_() + { } + + /* The default implementations are good here + /// Copy constructor. + BufferIterator(const BufferIterator &src) : + helper_(src.helper_) + { } + /// Assignment. + this_type& operator= (const this_type &rhs) { + helper_ = rhs.helper_; + return *this; + } + */ + + /// Construct iterator at the position in the buffer's chunk list. + explicit BufferIterator(BufferImpl::ChunkList::const_iterator iter) : + helper_(iter) + { } + + /// Dereference iterator, returns InputIteratorHelper or OutputIteratorHelper wrapper. + reference operator *() { + return helper_; + } + + /// Dereference iterator, returns const InputIteratorHelper or OutputIteratorHelper wrapper. + const value_type &operator *() const { + return helper_; + } + + /// Dereference iterator, returns InputIteratorHelper or OutputIteratorHelper wrapper. + pointer operator->() { + return &helper_; + } + + /// Dereference iterator, returns const InputIteratorHelper or OutputIteratorHelper wrapper. + const value_type *operator->() const { + return &helper_; + } + + /// Increment to next chunk in list, or to end() iterator. + this_type& operator++() + { + ++helper_.iter_; + return *this; + } + + /// Increment to next chunk in list, or to end() iterator. + this_type operator++(int) + { + this_type ret = *this; + ++helper_.iter_; + return ret; + } + + /// True if iterators point to same chunks. + bool operator==(const this_type &rhs) const { + return (helper_.iter_ == rhs.helper_.iter_); + } + + /// True if iterators point to different chunks. + bool operator!=(const this_type &rhs) const { + return (helper_.iter_ != rhs.helper_.iter_); + } + + private: + + Helper helper_; +}; + +typedef BufferIterator<InputIteratorHelper> InputBufferIterator; +typedef BufferIterator<OutputIteratorHelper> OutputBufferIterator; + +} // detail namespace + +} // namespace + +#endif diff --git a/contrib/libs/apache/avro/avro/AvroParse.hh b/contrib/libs/apache/avro/avro/AvroParse.hh index 06b1930d17..1ce878e5d1 100644 --- a/contrib/libs/apache/avro/avro/AvroParse.hh +++ b/contrib/libs/apache/avro/avro/AvroParse.hh @@ -1 +1 @@ -#include "../api/AvroParse.hh" /* inclink generated by yamaker */ +#include "../api/AvroParse.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/AvroSerialize.hh b/contrib/libs/apache/avro/avro/AvroSerialize.hh index fd2c96a77b..17f3da8cc5 100644 --- a/contrib/libs/apache/avro/avro/AvroSerialize.hh +++ b/contrib/libs/apache/avro/avro/AvroSerialize.hh @@ -1 +1 @@ -#include "../api/AvroSerialize.hh" /* inclink generated by yamaker */ +#include "../api/AvroSerialize.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/AvroTraits.hh b/contrib/libs/apache/avro/avro/AvroTraits.hh index 7e6e49a183..edb19eb840 100644 --- a/contrib/libs/apache/avro/avro/AvroTraits.hh +++ b/contrib/libs/apache/avro/avro/AvroTraits.hh @@ -1 +1 @@ -#include "../api/AvroTraits.hh" /* inclink generated by yamaker */ +#include "../api/AvroTraits.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Compiler.hh b/contrib/libs/apache/avro/avro/Compiler.hh index 9e80353938..fa43ede18a 100644 --- a/contrib/libs/apache/avro/avro/Compiler.hh +++ b/contrib/libs/apache/avro/avro/Compiler.hh @@ -1 +1 @@ -#include "../api/Compiler.hh" /* inclink generated by yamaker */ +#include "../api/Compiler.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Config.hh b/contrib/libs/apache/avro/avro/Config.hh index 105b0f910c..ae19fb85b8 100644 --- a/contrib/libs/apache/avro/avro/Config.hh +++ b/contrib/libs/apache/avro/avro/Config.hh @@ -1 +1 @@ -#include "../api/Config.hh" /* inclink generated by yamaker */ +#include "../api/Config.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/DataFile.hh b/contrib/libs/apache/avro/avro/DataFile.hh index 1b7a87ca95..401cb5fc6d 100644 --- a/contrib/libs/apache/avro/avro/DataFile.hh +++ b/contrib/libs/apache/avro/avro/DataFile.hh @@ -1 +1 @@ -#include "../api/DataFile.hh" /* inclink generated by yamaker */ +#include "../api/DataFile.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Decoder.hh b/contrib/libs/apache/avro/avro/Decoder.hh index b6b77aac74..1d8d79c4e5 100644 --- a/contrib/libs/apache/avro/avro/Decoder.hh +++ b/contrib/libs/apache/avro/avro/Decoder.hh @@ -1 +1 @@ -#include "../api/Decoder.hh" /* inclink generated by yamaker */ +#include "../api/Decoder.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Encoder.hh b/contrib/libs/apache/avro/avro/Encoder.hh index 8d5d522549..b390aee4df 100644 --- a/contrib/libs/apache/avro/avro/Encoder.hh +++ b/contrib/libs/apache/avro/avro/Encoder.hh @@ -1 +1 @@ -#include "../api/Encoder.hh" /* inclink generated by yamaker */ +#include "../api/Encoder.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Exception.hh b/contrib/libs/apache/avro/avro/Exception.hh index 3037d8c087..63ad410a4c 100644 --- a/contrib/libs/apache/avro/avro/Exception.hh +++ b/contrib/libs/apache/avro/avro/Exception.hh @@ -1 +1 @@ -#include "../api/Exception.hh" /* inclink generated by yamaker */ +#include "../api/Exception.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Generic.hh b/contrib/libs/apache/avro/avro/Generic.hh index 29b4e93a16..8d64997183 100644 --- a/contrib/libs/apache/avro/avro/Generic.hh +++ b/contrib/libs/apache/avro/avro/Generic.hh @@ -1 +1 @@ -#include "../api/Generic.hh" /* inclink generated by yamaker */ +#include "../api/Generic.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/GenericDatum.hh b/contrib/libs/apache/avro/avro/GenericDatum.hh index b894d3d6c0..e98719027e 100644 --- a/contrib/libs/apache/avro/avro/GenericDatum.hh +++ b/contrib/libs/apache/avro/avro/GenericDatum.hh @@ -1 +1 @@ -#include "../api/GenericDatum.hh" /* inclink generated by yamaker */ +#include "../api/GenericDatum.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Layout.hh b/contrib/libs/apache/avro/avro/Layout.hh index 89c307fa3b..f0c927217f 100644 --- a/contrib/libs/apache/avro/avro/Layout.hh +++ b/contrib/libs/apache/avro/avro/Layout.hh @@ -1 +1 @@ -#include "../api/Layout.hh" /* inclink generated by yamaker */ +#include "../api/Layout.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/LogicalType.hh b/contrib/libs/apache/avro/avro/LogicalType.hh index ec7b1450ef..2a7eaa375a 100644 --- a/contrib/libs/apache/avro/avro/LogicalType.hh +++ b/contrib/libs/apache/avro/avro/LogicalType.hh @@ -1 +1 @@ -#include "../api/LogicalType.hh" /* inclink generated by yamaker */ +#include "../api/LogicalType.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Node.hh b/contrib/libs/apache/avro/avro/Node.hh index 2673bee1d4..600d9e8e22 100644 --- a/contrib/libs/apache/avro/avro/Node.hh +++ b/contrib/libs/apache/avro/avro/Node.hh @@ -1 +1 @@ -#include "../api/Node.hh" /* inclink generated by yamaker */ +#include "../api/Node.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/NodeConcepts.hh b/contrib/libs/apache/avro/avro/NodeConcepts.hh index 6a915727aa..5e532e20b1 100644 --- a/contrib/libs/apache/avro/avro/NodeConcepts.hh +++ b/contrib/libs/apache/avro/avro/NodeConcepts.hh @@ -1 +1 @@ -#include "../api/NodeConcepts.hh" /* inclink generated by yamaker */ +#include "../api/NodeConcepts.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/NodeImpl.hh b/contrib/libs/apache/avro/avro/NodeImpl.hh index 23fe131b4a..d4893d50a8 100644 --- a/contrib/libs/apache/avro/avro/NodeImpl.hh +++ b/contrib/libs/apache/avro/avro/NodeImpl.hh @@ -1 +1 @@ -#include "../api/NodeImpl.hh" /* inclink generated by yamaker */ +#include "../api/NodeImpl.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Parser.hh b/contrib/libs/apache/avro/avro/Parser.hh index 9c790dae75..28eb7c4e16 100644 --- a/contrib/libs/apache/avro/avro/Parser.hh +++ b/contrib/libs/apache/avro/avro/Parser.hh @@ -1 +1 @@ -#include "../api/Parser.hh" /* inclink generated by yamaker */ +#include "../api/Parser.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Reader.hh b/contrib/libs/apache/avro/avro/Reader.hh index 807fbc73a3..cdfa6dec44 100644 --- a/contrib/libs/apache/avro/avro/Reader.hh +++ b/contrib/libs/apache/avro/avro/Reader.hh @@ -1 +1 @@ -#include "../api/Reader.hh" /* inclink generated by yamaker */ +#include "../api/Reader.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Resolver.hh b/contrib/libs/apache/avro/avro/Resolver.hh index 40b8ced180..bc3b3fb08f 100644 --- a/contrib/libs/apache/avro/avro/Resolver.hh +++ b/contrib/libs/apache/avro/avro/Resolver.hh @@ -1 +1 @@ -#include "../api/Resolver.hh" /* inclink generated by yamaker */ +#include "../api/Resolver.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/ResolverSchema.hh b/contrib/libs/apache/avro/avro/ResolverSchema.hh index ad5ae6018d..f4d41e1de2 100644 --- a/contrib/libs/apache/avro/avro/ResolverSchema.hh +++ b/contrib/libs/apache/avro/avro/ResolverSchema.hh @@ -1 +1 @@ -#include "../api/ResolverSchema.hh" /* inclink generated by yamaker */ +#include "../api/ResolverSchema.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/ResolvingReader.hh b/contrib/libs/apache/avro/avro/ResolvingReader.hh index 3652448bc0..04e4912747 100644 --- a/contrib/libs/apache/avro/avro/ResolvingReader.hh +++ b/contrib/libs/apache/avro/avro/ResolvingReader.hh @@ -1 +1 @@ -#include "../api/ResolvingReader.hh" /* inclink generated by yamaker */ +#include "../api/ResolvingReader.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Schema.hh b/contrib/libs/apache/avro/avro/Schema.hh index 0a2c3371c0..515b9d5855 100644 --- a/contrib/libs/apache/avro/avro/Schema.hh +++ b/contrib/libs/apache/avro/avro/Schema.hh @@ -1 +1 @@ -#include "../api/Schema.hh" /* inclink generated by yamaker */ +#include "../api/Schema.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/SchemaResolution.hh b/contrib/libs/apache/avro/avro/SchemaResolution.hh index e4fe3e9cdb..3aef7ff60e 100644 --- a/contrib/libs/apache/avro/avro/SchemaResolution.hh +++ b/contrib/libs/apache/avro/avro/SchemaResolution.hh @@ -1 +1 @@ -#include "../api/SchemaResolution.hh" /* inclink generated by yamaker */ +#include "../api/SchemaResolution.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Serializer.hh b/contrib/libs/apache/avro/avro/Serializer.hh index 04327c2e58..538b57dfa2 100644 --- a/contrib/libs/apache/avro/avro/Serializer.hh +++ b/contrib/libs/apache/avro/avro/Serializer.hh @@ -1 +1 @@ -#include "../api/Serializer.hh" /* inclink generated by yamaker */ +#include "../api/Serializer.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Specific.hh b/contrib/libs/apache/avro/avro/Specific.hh index 759a4d768e..4f8595c03f 100644 --- a/contrib/libs/apache/avro/avro/Specific.hh +++ b/contrib/libs/apache/avro/avro/Specific.hh @@ -1 +1 @@ -#include "../api/Specific.hh" /* inclink generated by yamaker */ +#include "../api/Specific.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Stream.hh b/contrib/libs/apache/avro/avro/Stream.hh index e575e0c80f..e056f5ae29 100644 --- a/contrib/libs/apache/avro/avro/Stream.hh +++ b/contrib/libs/apache/avro/avro/Stream.hh @@ -1 +1 @@ -#include "../api/Stream.hh" /* inclink generated by yamaker */ +#include "../api/Stream.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Types.hh b/contrib/libs/apache/avro/avro/Types.hh index bcc9a982ac..ad7ee126f4 100644 --- a/contrib/libs/apache/avro/avro/Types.hh +++ b/contrib/libs/apache/avro/avro/Types.hh @@ -1 +1 @@ -#include "../api/Types.hh" /* inclink generated by yamaker */ +#include "../api/Types.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/ValidSchema.hh b/contrib/libs/apache/avro/avro/ValidSchema.hh index 2e0adf09dd..a0845d77b1 100644 --- a/contrib/libs/apache/avro/avro/ValidSchema.hh +++ b/contrib/libs/apache/avro/avro/ValidSchema.hh @@ -1 +1 @@ -#include "../api/ValidSchema.hh" /* inclink generated by yamaker */ +#include "../api/ValidSchema.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Validator.hh b/contrib/libs/apache/avro/avro/Validator.hh index f0bb367357..bca184c46d 100644 --- a/contrib/libs/apache/avro/avro/Validator.hh +++ b/contrib/libs/apache/avro/avro/Validator.hh @@ -1 +1 @@ -#include "../api/Validator.hh" /* inclink generated by yamaker */ +#include "../api/Validator.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Writer.hh b/contrib/libs/apache/avro/avro/Writer.hh index af44f03bf1..f0ebac09dd 100644 --- a/contrib/libs/apache/avro/avro/Writer.hh +++ b/contrib/libs/apache/avro/avro/Writer.hh @@ -1 +1 @@ -#include "../api/Writer.hh" /* inclink generated by yamaker */ +#include "../api/Writer.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/avro/Zigzag.hh b/contrib/libs/apache/avro/avro/Zigzag.hh index 8b2235f012..db263917ca 100644 --- a/contrib/libs/apache/avro/avro/Zigzag.hh +++ b/contrib/libs/apache/avro/avro/Zigzag.hh @@ -1 +1 @@ -#include "../api/Zigzag.hh" /* inclink generated by yamaker */ +#include "../api/Zigzag.hh" /* inclink generated by yamaker */ diff --git a/contrib/libs/apache/avro/impl/BinaryDecoder.cc b/contrib/libs/apache/avro/impl/BinaryDecoder.cc index a6e6055b7f..71cbf9f107 100644 --- a/contrib/libs/apache/avro/impl/BinaryDecoder.cc +++ b/contrib/libs/apache/avro/impl/BinaryDecoder.cc @@ -1,252 +1,252 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define __STDC_LIMIT_MACROS - -#include <memory> -#include "Decoder.hh" -#include "Zigzag.hh" -#include "Exception.hh" - -namespace avro { - -using std::make_shared; - -class BinaryDecoder : public Decoder { - StreamReader in_; - const uint8_t* next_; - const uint8_t* end_; - - void init(InputStream& ib); - void decodeNull(); - bool decodeBool(); - int32_t decodeInt(); - int64_t decodeLong(); - float decodeFloat(); - double decodeDouble(); - void decodeString(std::string& value); - void skipString(); - void decodeBytes(std::vector<uint8_t>& value); - void skipBytes(); - void decodeFixed(size_t n, std::vector<uint8_t>& value); - void skipFixed(size_t n); - size_t decodeEnum(); - size_t arrayStart(); - size_t arrayNext(); - size_t skipArray(); - size_t mapStart(); - size_t mapNext(); - size_t skipMap(); - size_t decodeUnionIndex(); - - int64_t doDecodeLong(); - size_t doDecodeItemCount(); - size_t doDecodeLength(); - void drain(); - void more(); -}; - -DecoderPtr binaryDecoder() -{ - return make_shared<BinaryDecoder>(); -} - -void BinaryDecoder::init(InputStream& is) -{ - in_.reset(is); -} - -void BinaryDecoder::decodeNull() -{ -} - -bool BinaryDecoder::decodeBool() -{ - uint8_t v = in_.read(); - if (v == 0) { - return false; - } else if (v == 1) { - return true; - } - throw Exception("Invalid value for bool"); -} - -int32_t BinaryDecoder::decodeInt() -{ - int64_t val = doDecodeLong(); - if (val < INT32_MIN || val > INT32_MAX) { - throw Exception( - boost::format("Value out of range for Avro int: %1%") % val); - } - return static_cast<int32_t>(val); -} - -int64_t BinaryDecoder::decodeLong() -{ - return doDecodeLong(); -} - -float BinaryDecoder::decodeFloat() -{ - float result; - in_.readBytes(reinterpret_cast<uint8_t *>(&result), sizeof(float)); - return result; -} - -double BinaryDecoder::decodeDouble() -{ - double result; - in_.readBytes(reinterpret_cast<uint8_t *>(&result), sizeof(double)); - return result; -} - -size_t BinaryDecoder::doDecodeLength() -{ - ssize_t len = decodeInt(); - if (len < 0) { - throw Exception( - boost::format("Cannot have negative length: %1%") % len); - } - return len; -} - -void BinaryDecoder::drain() -{ - in_.drain(false); -} - -void BinaryDecoder::decodeString(std::string& value) -{ - size_t len = doDecodeLength(); - value.resize(len); - if (len > 0) { - in_.readBytes(const_cast<uint8_t*>( - reinterpret_cast<const uint8_t*>(value.c_str())), len); - } -} - -void BinaryDecoder::skipString() -{ - size_t len = doDecodeLength(); - in_.skipBytes(len); -} - -void BinaryDecoder::decodeBytes(std::vector<uint8_t>& value) -{ - size_t len = doDecodeLength(); - value.resize(len); - if (len > 0) { - in_.readBytes(value.data(), len); - } -} - -void BinaryDecoder::skipBytes() -{ - size_t len = doDecodeLength(); - in_.skipBytes(len); -} - -void BinaryDecoder::decodeFixed(size_t n, std::vector<uint8_t>& value) -{ - value.resize(n); - if (n > 0) { - in_.readBytes(value.data(), n); - } -} - -void BinaryDecoder::skipFixed(size_t n) -{ - in_.skipBytes(n); -} - -size_t BinaryDecoder::decodeEnum() -{ - return static_cast<size_t>(doDecodeLong()); -} - -size_t BinaryDecoder::arrayStart() -{ - return doDecodeItemCount(); -} - -size_t BinaryDecoder::doDecodeItemCount() -{ - int64_t result = doDecodeLong(); - if (result < 0) { - doDecodeLong(); - return static_cast<size_t>(-result); - } - return static_cast<size_t>(result); -} - -size_t BinaryDecoder::arrayNext() -{ - return static_cast<size_t>(doDecodeLong()); -} - -size_t BinaryDecoder::skipArray() -{ - for (; ;) { - int64_t r = doDecodeLong(); - if (r < 0) { - size_t n = static_cast<size_t>(doDecodeLong()); - in_.skipBytes(n); - } else { - return static_cast<size_t>(r); - } - } -} - -size_t BinaryDecoder::mapStart() -{ - return doDecodeItemCount(); -} - -size_t BinaryDecoder::mapNext() -{ - return doDecodeItemCount(); -} - -size_t BinaryDecoder::skipMap() -{ - return skipArray(); -} - -size_t BinaryDecoder::decodeUnionIndex() -{ - return static_cast<size_t>(doDecodeLong()); -} - -int64_t BinaryDecoder::doDecodeLong() { - uint64_t encoded = 0; - int shift = 0; - uint8_t u; - do { - if (shift >= 64) { - throw Exception("Invalid Avro varint"); - } - u = in_.read(); - encoded |= static_cast<uint64_t>(u & 0x7f) << shift; - shift += 7; - } while (u & 0x80); - - return decodeZigzag64(encoded); -} - -} // namespace avro - +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define __STDC_LIMIT_MACROS + +#include <memory> +#include "Decoder.hh" +#include "Zigzag.hh" +#include "Exception.hh" + +namespace avro { + +using std::make_shared; + +class BinaryDecoder : public Decoder { + StreamReader in_; + const uint8_t* next_; + const uint8_t* end_; + + void init(InputStream& ib); + void decodeNull(); + bool decodeBool(); + int32_t decodeInt(); + int64_t decodeLong(); + float decodeFloat(); + double decodeDouble(); + void decodeString(std::string& value); + void skipString(); + void decodeBytes(std::vector<uint8_t>& value); + void skipBytes(); + void decodeFixed(size_t n, std::vector<uint8_t>& value); + void skipFixed(size_t n); + size_t decodeEnum(); + size_t arrayStart(); + size_t arrayNext(); + size_t skipArray(); + size_t mapStart(); + size_t mapNext(); + size_t skipMap(); + size_t decodeUnionIndex(); + + int64_t doDecodeLong(); + size_t doDecodeItemCount(); + size_t doDecodeLength(); + void drain(); + void more(); +}; + +DecoderPtr binaryDecoder() +{ + return make_shared<BinaryDecoder>(); +} + +void BinaryDecoder::init(InputStream& is) +{ + in_.reset(is); +} + +void BinaryDecoder::decodeNull() +{ +} + +bool BinaryDecoder::decodeBool() +{ + uint8_t v = in_.read(); + if (v == 0) { + return false; + } else if (v == 1) { + return true; + } + throw Exception("Invalid value for bool"); +} + +int32_t BinaryDecoder::decodeInt() +{ + int64_t val = doDecodeLong(); + if (val < INT32_MIN || val > INT32_MAX) { + throw Exception( + boost::format("Value out of range for Avro int: %1%") % val); + } + return static_cast<int32_t>(val); +} + +int64_t BinaryDecoder::decodeLong() +{ + return doDecodeLong(); +} + +float BinaryDecoder::decodeFloat() +{ + float result; + in_.readBytes(reinterpret_cast<uint8_t *>(&result), sizeof(float)); + return result; +} + +double BinaryDecoder::decodeDouble() +{ + double result; + in_.readBytes(reinterpret_cast<uint8_t *>(&result), sizeof(double)); + return result; +} + +size_t BinaryDecoder::doDecodeLength() +{ + ssize_t len = decodeInt(); + if (len < 0) { + throw Exception( + boost::format("Cannot have negative length: %1%") % len); + } + return len; +} + +void BinaryDecoder::drain() +{ + in_.drain(false); +} + +void BinaryDecoder::decodeString(std::string& value) +{ + size_t len = doDecodeLength(); + value.resize(len); + if (len > 0) { + in_.readBytes(const_cast<uint8_t*>( + reinterpret_cast<const uint8_t*>(value.c_str())), len); + } +} + +void BinaryDecoder::skipString() +{ + size_t len = doDecodeLength(); + in_.skipBytes(len); +} + +void BinaryDecoder::decodeBytes(std::vector<uint8_t>& value) +{ + size_t len = doDecodeLength(); + value.resize(len); + if (len > 0) { + in_.readBytes(value.data(), len); + } +} + +void BinaryDecoder::skipBytes() +{ + size_t len = doDecodeLength(); + in_.skipBytes(len); +} + +void BinaryDecoder::decodeFixed(size_t n, std::vector<uint8_t>& value) +{ + value.resize(n); + if (n > 0) { + in_.readBytes(value.data(), n); + } +} + +void BinaryDecoder::skipFixed(size_t n) +{ + in_.skipBytes(n); +} + +size_t BinaryDecoder::decodeEnum() +{ + return static_cast<size_t>(doDecodeLong()); +} + +size_t BinaryDecoder::arrayStart() +{ + return doDecodeItemCount(); +} + +size_t BinaryDecoder::doDecodeItemCount() +{ + int64_t result = doDecodeLong(); + if (result < 0) { + doDecodeLong(); + return static_cast<size_t>(-result); + } + return static_cast<size_t>(result); +} + +size_t BinaryDecoder::arrayNext() +{ + return static_cast<size_t>(doDecodeLong()); +} + +size_t BinaryDecoder::skipArray() +{ + for (; ;) { + int64_t r = doDecodeLong(); + if (r < 0) { + size_t n = static_cast<size_t>(doDecodeLong()); + in_.skipBytes(n); + } else { + return static_cast<size_t>(r); + } + } +} + +size_t BinaryDecoder::mapStart() +{ + return doDecodeItemCount(); +} + +size_t BinaryDecoder::mapNext() +{ + return doDecodeItemCount(); +} + +size_t BinaryDecoder::skipMap() +{ + return skipArray(); +} + +size_t BinaryDecoder::decodeUnionIndex() +{ + return static_cast<size_t>(doDecodeLong()); +} + +int64_t BinaryDecoder::doDecodeLong() { + uint64_t encoded = 0; + int shift = 0; + uint8_t u; + do { + if (shift >= 64) { + throw Exception("Invalid Avro varint"); + } + u = in_.read(); + encoded |= static_cast<uint64_t>(u & 0x7f) << shift; + shift += 7; + } while (u & 0x80); + + return decodeZigzag64(encoded); +} + +} // namespace avro + diff --git a/contrib/libs/apache/avro/impl/BinaryEncoder.cc b/contrib/libs/apache/avro/impl/BinaryEncoder.cc index 1fa83fe8ff..5ceb872f8c 100644 --- a/contrib/libs/apache/avro/impl/BinaryEncoder.cc +++ b/contrib/libs/apache/avro/impl/BinaryEncoder.cc @@ -1,168 +1,168 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "Encoder.hh" -#include "Zigzag.hh" -#include <array> - -namespace avro { - -using std::make_shared; - -class BinaryEncoder : public Encoder { - StreamWriter out_; - - void init(OutputStream& os); - void flush(); - int64_t byteCount() const; - void encodeNull(); - void encodeBool(bool b); - void encodeInt(int32_t i); - void encodeLong(int64_t l); - void encodeFloat(float f); - void encodeDouble(double d); - void encodeString(const std::string& s); - void encodeBytes(const uint8_t *bytes, size_t len); - void encodeFixed(const uint8_t *bytes, size_t len); - void encodeEnum(size_t e); - void arrayStart(); - void arrayEnd(); - void mapStart(); - void mapEnd(); - void setItemCount(size_t count); - void startItem(); - void encodeUnionIndex(size_t e); - - void doEncodeLong(int64_t l); -}; - -EncoderPtr binaryEncoder() -{ - return make_shared<BinaryEncoder>(); -} - -void BinaryEncoder::init(OutputStream& os) -{ - out_.reset(os); -} - -void BinaryEncoder::flush() -{ - out_.flush(); -} - -void BinaryEncoder::encodeNull() -{ -} - -void BinaryEncoder::encodeBool(bool b) -{ - out_.write(b ? 1 : 0); -} - -void BinaryEncoder::encodeInt(int32_t i) -{ - doEncodeLong(i); -} - -void BinaryEncoder::encodeLong(int64_t l) -{ - doEncodeLong(l); -} - -void BinaryEncoder::encodeFloat(float f) -{ - const uint8_t* p = reinterpret_cast<const uint8_t*>(&f); - out_.writeBytes(p, sizeof(float)); -} - -void BinaryEncoder::encodeDouble(double d) -{ - const uint8_t* p = reinterpret_cast<const uint8_t*>(&d); - out_.writeBytes(p, sizeof(double)); -} - -void BinaryEncoder::encodeString(const std::string& s) -{ - doEncodeLong(s.size()); - out_.writeBytes(reinterpret_cast<const uint8_t*>(s.c_str()), s.size()); -} - -void BinaryEncoder::encodeBytes(const uint8_t *bytes, size_t len) -{ - doEncodeLong(len); - out_.writeBytes(bytes, len); -} - -void BinaryEncoder::encodeFixed(const uint8_t *bytes, size_t len) -{ - out_.writeBytes(bytes, len); -} - -void BinaryEncoder::encodeEnum(size_t e) -{ - doEncodeLong(e); -} - -void BinaryEncoder::arrayStart() -{ -} - -void BinaryEncoder::arrayEnd() -{ - doEncodeLong(0); -} - -void BinaryEncoder::mapStart() -{ -} - -void BinaryEncoder::mapEnd() -{ - doEncodeLong(0); -} - -void BinaryEncoder::setItemCount(size_t count) -{ - if (count == 0) { - throw Exception("Count cannot be zero"); - } - doEncodeLong(count); -} - -void BinaryEncoder::startItem() -{ -} - -void BinaryEncoder::encodeUnionIndex(size_t e) -{ - doEncodeLong(e); -} - -int64_t BinaryEncoder::byteCount() const { - return out_.byteCount(); -} - - -void BinaryEncoder::doEncodeLong(int64_t l) -{ - std::array<uint8_t, 10> bytes; - size_t size = encodeInt64(l, bytes); - out_.writeBytes(bytes.data(), size); -} -} // namespace avro +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Encoder.hh" +#include "Zigzag.hh" +#include <array> + +namespace avro { + +using std::make_shared; + +class BinaryEncoder : public Encoder { + StreamWriter out_; + + void init(OutputStream& os); + void flush(); + int64_t byteCount() const; + void encodeNull(); + void encodeBool(bool b); + void encodeInt(int32_t i); + void encodeLong(int64_t l); + void encodeFloat(float f); + void encodeDouble(double d); + void encodeString(const std::string& s); + void encodeBytes(const uint8_t *bytes, size_t len); + void encodeFixed(const uint8_t *bytes, size_t len); + void encodeEnum(size_t e); + void arrayStart(); + void arrayEnd(); + void mapStart(); + void mapEnd(); + void setItemCount(size_t count); + void startItem(); + void encodeUnionIndex(size_t e); + + void doEncodeLong(int64_t l); +}; + +EncoderPtr binaryEncoder() +{ + return make_shared<BinaryEncoder>(); +} + +void BinaryEncoder::init(OutputStream& os) +{ + out_.reset(os); +} + +void BinaryEncoder::flush() +{ + out_.flush(); +} + +void BinaryEncoder::encodeNull() +{ +} + +void BinaryEncoder::encodeBool(bool b) +{ + out_.write(b ? 1 : 0); +} + +void BinaryEncoder::encodeInt(int32_t i) +{ + doEncodeLong(i); +} + +void BinaryEncoder::encodeLong(int64_t l) +{ + doEncodeLong(l); +} + +void BinaryEncoder::encodeFloat(float f) +{ + const uint8_t* p = reinterpret_cast<const uint8_t*>(&f); + out_.writeBytes(p, sizeof(float)); +} + +void BinaryEncoder::encodeDouble(double d) +{ + const uint8_t* p = reinterpret_cast<const uint8_t*>(&d); + out_.writeBytes(p, sizeof(double)); +} + +void BinaryEncoder::encodeString(const std::string& s) +{ + doEncodeLong(s.size()); + out_.writeBytes(reinterpret_cast<const uint8_t*>(s.c_str()), s.size()); +} + +void BinaryEncoder::encodeBytes(const uint8_t *bytes, size_t len) +{ + doEncodeLong(len); + out_.writeBytes(bytes, len); +} + +void BinaryEncoder::encodeFixed(const uint8_t *bytes, size_t len) +{ + out_.writeBytes(bytes, len); +} + +void BinaryEncoder::encodeEnum(size_t e) +{ + doEncodeLong(e); +} + +void BinaryEncoder::arrayStart() +{ +} + +void BinaryEncoder::arrayEnd() +{ + doEncodeLong(0); +} + +void BinaryEncoder::mapStart() +{ +} + +void BinaryEncoder::mapEnd() +{ + doEncodeLong(0); +} + +void BinaryEncoder::setItemCount(size_t count) +{ + if (count == 0) { + throw Exception("Count cannot be zero"); + } + doEncodeLong(count); +} + +void BinaryEncoder::startItem() +{ +} + +void BinaryEncoder::encodeUnionIndex(size_t e) +{ + doEncodeLong(e); +} + +int64_t BinaryEncoder::byteCount() const { + return out_.byteCount(); +} + + +void BinaryEncoder::doEncodeLong(int64_t l) +{ + std::array<uint8_t, 10> bytes; + size_t size = encodeInt64(l, bytes); + out_.writeBytes(bytes.data(), size); +} +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/Compiler.cc b/contrib/libs/apache/avro/impl/Compiler.cc index 8b1de49a1d..6453db8f17 100644 --- a/contrib/libs/apache/avro/impl/Compiler.cc +++ b/contrib/libs/apache/avro/impl/Compiler.cc @@ -1,591 +1,591 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include <boost/algorithm/string/replace.hpp> -#include <sstream> - -#include "Compiler.hh" -#include "Types.hh" -#include "Schema.hh" -#include "ValidSchema.hh" -#include "Stream.hh" - -#include "json/JsonDom.hh" - -using std::string; -using std::map; -using std::vector; -using std::pair; -using std::make_pair; - -namespace avro { -using json::Entity; -using json::Object; -using json::Array; -using json::EntityType; - -typedef map<Name, NodePtr> SymbolTable; - - -// #define DEBUG_VERBOSE - -static NodePtr makePrimitive(const string& t) -{ - if (t == "null") { - return NodePtr(new NodePrimitive(AVRO_NULL)); - } else if (t == "boolean") { - return NodePtr(new NodePrimitive(AVRO_BOOL)); - } else if (t == "int") { - return NodePtr(new NodePrimitive(AVRO_INT)); - } else if (t == "long") { - return NodePtr(new NodePrimitive(AVRO_LONG)); - } else if (t == "float") { - return NodePtr(new NodePrimitive(AVRO_FLOAT)); - } else if (t == "double") { - return NodePtr(new NodePrimitive(AVRO_DOUBLE)); - } else if (t == "string") { - return NodePtr(new NodePrimitive(AVRO_STRING)); - } else if (t == "bytes") { - return NodePtr(new NodePrimitive(AVRO_BYTES)); - } else { - return NodePtr(); - } -} - -static NodePtr makeNode(const json::Entity& e, SymbolTable& st, const string &ns); - -template <typename T> -concepts::SingleAttribute<T> asSingleAttribute(const T& t) -{ - concepts::SingleAttribute<T> n; - n.add(t); - return n; -} - -static bool isFullName(const string &s) -{ - return s.find('.') != string::npos; -} - -static Name getName(const string &name, const string &ns) -{ - return (isFullName(name)) ? Name(name) : Name(name, ns); -} - -static NodePtr makeNode(const string &t, SymbolTable &st, const string &ns) -{ - NodePtr result = makePrimitive(t); - if (result) { - return result; - } - Name n = getName(t, ns); - - SymbolTable::const_iterator it = st.find(n); - if (it != st.end()) { - return NodePtr(new NodeSymbolic(asSingleAttribute(n), it->second)); - } - throw Exception(boost::format("Unknown type: %1%") % n.fullname()); -} - -/** Returns "true" if the field is in the container */ -// e.g.: can be false for non-mandatory fields -bool containsField(const Object& m, const string& fieldName) { - Object::const_iterator it = m.find(fieldName); - return (it != m.end()); -} - -const json::Object::const_iterator findField(const Entity& e, - const Object& m, const string& fieldName) -{ - Object::const_iterator it = m.find(fieldName); - if (it == m.end()) { - throw Exception(boost::format("Missing Json field \"%1%\": %2%") % - fieldName % e.toString()); - } else { - return it; - } -} - -template <typename T> void ensureType(const Entity &e, const string &name) -{ - if (e.type() != json::type_traits<T>::type()) { - throw Exception(boost::format("Json field \"%1%\" is not a %2%: %3%") % - name % json::type_traits<T>::name() % e.toString()); - } -} - -string getStringField(const Entity &e, const Object &m, - const string &fieldName) -{ - Object::const_iterator it = findField(e, m, fieldName); - ensureType<string>(it->second, fieldName); - return it->second.stringValue(); -} - -const Array& getArrayField(const Entity& e, const Object& m, - const string& fieldName) -{ - Object::const_iterator it = findField(e, m, fieldName); - ensureType<Array >(it->second, fieldName); - return it->second.arrayValue(); -} - -const int64_t getLongField(const Entity& e, const Object& m, - const string& fieldName) -{ - Object::const_iterator it = findField(e, m, fieldName); - ensureType<int64_t>(it->second, fieldName); - return it->second.longValue(); -} - -// Unescape double quotes (") for de-serialization. This method complements the -// method NodeImpl::escape() which is used for serialization. -static void unescape(string& s) { - boost::replace_all(s, "\\\"", "\""); -} - -const string getDocField(const Entity& e, const Object& m) -{ - string doc = getStringField(e, m, "doc"); - unescape(doc); - return doc; -} - -struct Field { - const string name; - const NodePtr schema; - const GenericDatum defaultValue; - Field(const string& n, const NodePtr& v, GenericDatum dv) : - name(n), schema(v), defaultValue(dv) { } -}; - -static void assertType(const Entity& e, EntityType et) -{ - if (e.type() != et) { - throw Exception(boost::format("Unexpected type for default value: " - "Expected %1%, but found %2% in line %3%") % - json::typeToString(et) % json::typeToString(e.type()) % - e.line()); - } -} - -static vector<uint8_t> toBin(const string& s) -{ - vector<uint8_t> result(s.size()); - if (s.size() > 0) { - std::copy(s.c_str(), s.c_str() + s.size(), result.data()); - } - return result; -} - -static GenericDatum makeGenericDatum(NodePtr n, - const Entity& e, const SymbolTable& st) -{ - Type t = n->type(); - EntityType dt = e.type(); - - if (t == AVRO_SYMBOLIC) { - n = st.find(n->name())->second; - t = n->type(); - } - switch (t) { - case AVRO_STRING: - assertType(e, json::etString); - return GenericDatum(e.stringValue()); - case AVRO_BYTES: - assertType(e, json::etString); - return GenericDatum(toBin(e.bytesValue())); - case AVRO_INT: - assertType(e, json::etLong); - return GenericDatum(static_cast<int32_t>(e.longValue())); - case AVRO_LONG: - assertType(e, json::etLong); - return GenericDatum(e.longValue()); - case AVRO_FLOAT: - if (dt == json::etLong) { - return GenericDatum(static_cast<float>(e.longValue())); - } - assertType(e, json::etDouble); - return GenericDatum(static_cast<float>(e.doubleValue())); - case AVRO_DOUBLE: - if (dt == json::etLong) { - return GenericDatum(static_cast<double>(e.longValue())); - } - assertType(e, json::etDouble); - return GenericDatum(e.doubleValue()); - case AVRO_BOOL: - assertType(e, json::etBool); - return GenericDatum(e.boolValue()); - case AVRO_NULL: - assertType(e, json::etNull); - return GenericDatum(); - case AVRO_RECORD: - { - assertType(e, json::etObject); - GenericRecord result(n); - const map<string, Entity>& v = e.objectValue(); - for (size_t i = 0; i < n->leaves(); ++i) { - map<string, Entity>::const_iterator it = v.find(n->nameAt(i)); - if (it == v.end()) { - throw Exception(boost::format( - "No value found in default for %1%") % n->nameAt(i)); - } - result.setFieldAt(i, - makeGenericDatum(n->leafAt(i), it->second, st)); - } - return GenericDatum(n, result); - } - case AVRO_ENUM: - assertType(e, json::etString); - return GenericDatum(n, GenericEnum(n, e.stringValue())); - case AVRO_ARRAY: - { - assertType(e, json::etArray); - GenericArray result(n); - const vector<Entity>& elements = e.arrayValue(); - for (vector<Entity>::const_iterator it = elements.begin(); - it != elements.end(); ++it) { - result.value().push_back(makeGenericDatum(n->leafAt(0), *it, st)); - } - return GenericDatum(n, result); - } - case AVRO_MAP: - { - assertType(e, json::etObject); - GenericMap result(n); - const map<string, Entity>& v = e.objectValue(); - for (map<string, Entity>::const_iterator it = v.begin(); - it != v.end(); ++it) { - result.value().push_back(make_pair(it->first, - makeGenericDatum(n->leafAt(1), it->second, st))); - } - return GenericDatum(n, result); - } - case AVRO_UNION: - { - GenericUnion result(n); - result.selectBranch(0); - result.datum() = makeGenericDatum(n->leafAt(0), e, st); - return GenericDatum(n, result); - } - case AVRO_FIXED: - assertType(e, json::etString); - return GenericDatum(n, GenericFixed(n, toBin(e.bytesValue()))); - default: - throw Exception(boost::format("Unknown type: %1%") % t); - } - return GenericDatum(); -} - - -static Field makeField(const Entity& e, SymbolTable& st, const string& ns) -{ - const Object& m = e.objectValue(); - const string& n = getStringField(e, m, "name"); - Object::const_iterator it = findField(e, m, "type"); - map<string, Entity>::const_iterator it2 = m.find("default"); - NodePtr node = makeNode(it->second, st, ns); - if (containsField(m, "doc")) { - node->setDoc(getDocField(e, m)); - } - GenericDatum d = (it2 == m.end()) ? GenericDatum() : - makeGenericDatum(node, it2->second, st); - return Field(n, node, d); -} - -// Extended makeRecordNode (with doc). -static NodePtr makeRecordNode(const Entity& e, const Name& name, - const string* doc, const Object& m, - SymbolTable& st, const string& ns) { - const Array& v = getArrayField(e, m, "fields"); - concepts::MultiAttribute<string> fieldNames; - concepts::MultiAttribute<NodePtr> fieldValues; - vector<GenericDatum> defaultValues; - - for (Array::const_iterator it = v.begin(); it != v.end(); ++it) { - Field f = makeField(*it, st, ns); - fieldNames.add(f.name); - fieldValues.add(f.schema); - defaultValues.push_back(f.defaultValue); - } - NodeRecord* node; - if (doc == NULL) { - node = new NodeRecord(asSingleAttribute(name), fieldValues, fieldNames, - defaultValues); - } else { - node = new NodeRecord(asSingleAttribute(name), asSingleAttribute(*doc), - fieldValues, fieldNames, defaultValues); - } - return NodePtr(node); -} - -static LogicalType makeLogicalType(const Entity& e, const Object& m) { - if (!containsField(m, "logicalType")) { - return LogicalType(LogicalType::NONE); - } - - const std::string& typeField = getStringField(e, m, "logicalType"); - - if (typeField == "decimal") { - LogicalType decimalType(LogicalType::DECIMAL); - try { - decimalType.setPrecision(getLongField(e, m, "precision")); - if (containsField(m, "scale")) { - decimalType.setScale(getLongField(e, m, "scale")); - } - } catch (Exception& ex) { - // If any part of the logical type is malformed, per the standard we - // must ignore the whole attribute. - return LogicalType(LogicalType::NONE); - } - return decimalType; - } - - LogicalType::Type t = LogicalType::NONE; - if (typeField == "date") - t = LogicalType::DATE; - else if (typeField == "time-millis") - t = LogicalType::TIME_MILLIS; - else if (typeField == "time-micros") - t = LogicalType::TIME_MICROS; - else if (typeField == "timestamp-millis") - t = LogicalType::TIMESTAMP_MILLIS; - else if (typeField == "timestamp-micros") - t = LogicalType::TIMESTAMP_MICROS; - else if (typeField == "duration") - t = LogicalType::DURATION; - else if (typeField == "uuid") - t = LogicalType::UUID; - return LogicalType(t); -} - -static NodePtr makeEnumNode(const Entity& e, - const Name& name, const Object& m) -{ - const Array& v = getArrayField(e, m, "symbols"); - concepts::MultiAttribute<string> symbols; - for (Array::const_iterator it = v.begin(); it != v.end(); ++it) { - if (it->type() != json::etString) { - throw Exception(boost::format("Enum symbol not a string: %1%") % - it->toString()); - } - symbols.add(it->stringValue()); - } - NodePtr node = NodePtr(new NodeEnum(asSingleAttribute(name), symbols)); - if (containsField(m, "doc")) { - node->setDoc(getDocField(e, m)); - } - return node; -} - -static NodePtr makeFixedNode(const Entity& e, - const Name& name, const Object& m) -{ - int v = static_cast<int>(getLongField(e, m, "size")); - if (v <= 0) { - throw Exception(boost::format("Size for fixed is not positive: %1%") % - e.toString()); - } - NodePtr node = - NodePtr(new NodeFixed(asSingleAttribute(name), asSingleAttribute(v))); - if (containsField(m, "doc")) { - node->setDoc(getDocField(e, m)); - } - return node; -} - -static NodePtr makeArrayNode(const Entity& e, const Object& m, - SymbolTable& st, const string& ns) -{ - Object::const_iterator it = findField(e, m, "items"); - NodePtr node = NodePtr(new NodeArray( - asSingleAttribute(makeNode(it->second, st, ns)))); - if (containsField(m, "doc")) { - node->setDoc(getDocField(e, m)); - } - return node; -} - -static NodePtr makeMapNode(const Entity& e, const Object& m, - SymbolTable& st, const string& ns) -{ - Object::const_iterator it = findField(e, m, "values"); - - NodePtr node = NodePtr(new NodeMap( - asSingleAttribute(makeNode(it->second, st, ns)))); - if (containsField(m, "doc")) { - node->setDoc(getDocField(e, m)); - } - return node; -} - -static Name getName(const Entity& e, const Object& m, const string& ns) -{ - const string& name = getStringField(e, m, "name"); - - if (isFullName(name)) { - return Name(name); - } else { - Object::const_iterator it = m.find("namespace"); - if (it != m.end()) { - if (it->second.type() != json::type_traits<string>::type()) { - throw Exception(boost::format( - "Json field \"%1%\" is not a %2%: %3%") % - "namespace" % json::type_traits<string>::name() % - it->second.toString()); - } - Name result = Name(name, it->second.stringValue()); - return result; - } - return Name(name, ns); - } -} - -static NodePtr makeNode(const Entity& e, const Object& m, - SymbolTable& st, const string& ns) -{ - const string& type = getStringField(e, m, "type"); - NodePtr result; - if (type == "record" || type == "error" || - type == "enum" || type == "fixed") { - Name nm = getName(e, m, ns); - if (type == "record" || type == "error") { - result = NodePtr(new NodeRecord()); - st[nm] = result; - // Get field doc - if (containsField(m, "doc")) { - string doc = getDocField(e, m); - - NodePtr r = makeRecordNode(e, nm, &doc, m, st, nm.ns()); - (std::dynamic_pointer_cast<NodeRecord>(r))->swap( - *std::dynamic_pointer_cast<NodeRecord>(result)); - } else { // No doc - NodePtr r = - makeRecordNode(e, nm, NULL, m, st, nm.ns()); - (std::dynamic_pointer_cast<NodeRecord>(r)) - ->swap(*std::dynamic_pointer_cast<NodeRecord>(result)); - } - } else { - result = (type == "enum") ? makeEnumNode(e, nm, m) : - makeFixedNode(e, nm, m); - st[nm] = result; - } - } else if (type == "array") { - result = makeArrayNode(e, m, st, ns); - } else if (type == "map") { - result = makeMapNode(e, m, st, ns); - } else { - result = makePrimitive(type); - } - - if (result) { - try { - result->setLogicalType(makeLogicalType(e, m)); - } catch (Exception& ex) { - // Per the standard we must ignore the logical type attribute if it - // is malformed. - } - return result; - } - - throw Exception(boost::format("Unknown type definition: %1%") - % e.toString()); -} - -static NodePtr makeNode(const Entity& e, const Array& m, - SymbolTable& st, const string& ns) -{ - concepts::MultiAttribute<NodePtr> mm; - for (Array::const_iterator it = m.begin(); it != m.end(); ++it) { - mm.add(makeNode(*it, st, ns)); - } - return NodePtr(new NodeUnion(mm)); -} - -static NodePtr makeNode(const json::Entity& e, SymbolTable& st, const string& ns) -{ - switch (e.type()) { - case json::etString: - return makeNode(e.stringValue(), st, ns); - case json::etObject: - return makeNode(e, e.objectValue(), st, ns); - case json::etArray: - return makeNode(e, e.arrayValue(), st, ns); - default: - throw Exception(boost::format("Invalid Avro type: %1%") % e.toString()); - } -} - -ValidSchema compileJsonSchemaFromStream(InputStream& is) -{ - json::Entity e = json::loadEntity(is); - SymbolTable st; - NodePtr n = makeNode(e, st, ""); - return ValidSchema(n); -} - -AVRO_DECL ValidSchema compileJsonSchemaFromFile(const char* filename) -{ - std::unique_ptr<InputStream> s = fileInputStream(filename); - return compileJsonSchemaFromStream(*s); -} - -AVRO_DECL ValidSchema compileJsonSchemaFromMemory(const uint8_t* input, size_t len) -{ - return compileJsonSchemaFromStream(*memoryInputStream(input, len)); -} - -AVRO_DECL ValidSchema compileJsonSchemaFromString(const char* input) -{ - return compileJsonSchemaFromMemory(reinterpret_cast<const uint8_t*>(input), - ::strlen(input)); -} - -AVRO_DECL ValidSchema compileJsonSchemaFromString(const string& input) -{ - return compileJsonSchemaFromMemory( - reinterpret_cast<const uint8_t*>(input.data()), input.size()); -} - -static ValidSchema compile(std::istream& is) -{ - std::unique_ptr<InputStream> in = istreamInputStream(is); - return compileJsonSchemaFromStream(*in); -} - -void compileJsonSchema(std::istream &is, ValidSchema &schema) -{ - if (!is.good()) { - throw Exception("Input stream is not good"); - } - - schema = compile(is); -} - -AVRO_DECL bool compileJsonSchema(std::istream &is, ValidSchema &schema, string &error) -{ - try { - compileJsonSchema(is, schema); - return true; - } catch (const Exception &e) { - error = e.what(); - return false; - } - -} - -} // namespace avro +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <boost/algorithm/string/replace.hpp> +#include <sstream> + +#include "Compiler.hh" +#include "Types.hh" +#include "Schema.hh" +#include "ValidSchema.hh" +#include "Stream.hh" + +#include "json/JsonDom.hh" + +using std::string; +using std::map; +using std::vector; +using std::pair; +using std::make_pair; + +namespace avro { +using json::Entity; +using json::Object; +using json::Array; +using json::EntityType; + +typedef map<Name, NodePtr> SymbolTable; + + +// #define DEBUG_VERBOSE + +static NodePtr makePrimitive(const string& t) +{ + if (t == "null") { + return NodePtr(new NodePrimitive(AVRO_NULL)); + } else if (t == "boolean") { + return NodePtr(new NodePrimitive(AVRO_BOOL)); + } else if (t == "int") { + return NodePtr(new NodePrimitive(AVRO_INT)); + } else if (t == "long") { + return NodePtr(new NodePrimitive(AVRO_LONG)); + } else if (t == "float") { + return NodePtr(new NodePrimitive(AVRO_FLOAT)); + } else if (t == "double") { + return NodePtr(new NodePrimitive(AVRO_DOUBLE)); + } else if (t == "string") { + return NodePtr(new NodePrimitive(AVRO_STRING)); + } else if (t == "bytes") { + return NodePtr(new NodePrimitive(AVRO_BYTES)); + } else { + return NodePtr(); + } +} + +static NodePtr makeNode(const json::Entity& e, SymbolTable& st, const string &ns); + +template <typename T> +concepts::SingleAttribute<T> asSingleAttribute(const T& t) +{ + concepts::SingleAttribute<T> n; + n.add(t); + return n; +} + +static bool isFullName(const string &s) +{ + return s.find('.') != string::npos; +} + +static Name getName(const string &name, const string &ns) +{ + return (isFullName(name)) ? Name(name) : Name(name, ns); +} + +static NodePtr makeNode(const string &t, SymbolTable &st, const string &ns) +{ + NodePtr result = makePrimitive(t); + if (result) { + return result; + } + Name n = getName(t, ns); + + SymbolTable::const_iterator it = st.find(n); + if (it != st.end()) { + return NodePtr(new NodeSymbolic(asSingleAttribute(n), it->second)); + } + throw Exception(boost::format("Unknown type: %1%") % n.fullname()); +} + +/** Returns "true" if the field is in the container */ +// e.g.: can be false for non-mandatory fields +bool containsField(const Object& m, const string& fieldName) { + Object::const_iterator it = m.find(fieldName); + return (it != m.end()); +} + +const json::Object::const_iterator findField(const Entity& e, + const Object& m, const string& fieldName) +{ + Object::const_iterator it = m.find(fieldName); + if (it == m.end()) { + throw Exception(boost::format("Missing Json field \"%1%\": %2%") % + fieldName % e.toString()); + } else { + return it; + } +} + +template <typename T> void ensureType(const Entity &e, const string &name) +{ + if (e.type() != json::type_traits<T>::type()) { + throw Exception(boost::format("Json field \"%1%\" is not a %2%: %3%") % + name % json::type_traits<T>::name() % e.toString()); + } +} + +string getStringField(const Entity &e, const Object &m, + const string &fieldName) +{ + Object::const_iterator it = findField(e, m, fieldName); + ensureType<string>(it->second, fieldName); + return it->second.stringValue(); +} + +const Array& getArrayField(const Entity& e, const Object& m, + const string& fieldName) +{ + Object::const_iterator it = findField(e, m, fieldName); + ensureType<Array >(it->second, fieldName); + return it->second.arrayValue(); +} + +const int64_t getLongField(const Entity& e, const Object& m, + const string& fieldName) +{ + Object::const_iterator it = findField(e, m, fieldName); + ensureType<int64_t>(it->second, fieldName); + return it->second.longValue(); +} + +// Unescape double quotes (") for de-serialization. This method complements the +// method NodeImpl::escape() which is used for serialization. +static void unescape(string& s) { + boost::replace_all(s, "\\\"", "\""); +} + +const string getDocField(const Entity& e, const Object& m) +{ + string doc = getStringField(e, m, "doc"); + unescape(doc); + return doc; +} + +struct Field { + const string name; + const NodePtr schema; + const GenericDatum defaultValue; + Field(const string& n, const NodePtr& v, GenericDatum dv) : + name(n), schema(v), defaultValue(dv) { } +}; + +static void assertType(const Entity& e, EntityType et) +{ + if (e.type() != et) { + throw Exception(boost::format("Unexpected type for default value: " + "Expected %1%, but found %2% in line %3%") % + json::typeToString(et) % json::typeToString(e.type()) % + e.line()); + } +} + +static vector<uint8_t> toBin(const string& s) +{ + vector<uint8_t> result(s.size()); + if (s.size() > 0) { + std::copy(s.c_str(), s.c_str() + s.size(), result.data()); + } + return result; +} + +static GenericDatum makeGenericDatum(NodePtr n, + const Entity& e, const SymbolTable& st) +{ + Type t = n->type(); + EntityType dt = e.type(); + + if (t == AVRO_SYMBOLIC) { + n = st.find(n->name())->second; + t = n->type(); + } + switch (t) { + case AVRO_STRING: + assertType(e, json::etString); + return GenericDatum(e.stringValue()); + case AVRO_BYTES: + assertType(e, json::etString); + return GenericDatum(toBin(e.bytesValue())); + case AVRO_INT: + assertType(e, json::etLong); + return GenericDatum(static_cast<int32_t>(e.longValue())); + case AVRO_LONG: + assertType(e, json::etLong); + return GenericDatum(e.longValue()); + case AVRO_FLOAT: + if (dt == json::etLong) { + return GenericDatum(static_cast<float>(e.longValue())); + } + assertType(e, json::etDouble); + return GenericDatum(static_cast<float>(e.doubleValue())); + case AVRO_DOUBLE: + if (dt == json::etLong) { + return GenericDatum(static_cast<double>(e.longValue())); + } + assertType(e, json::etDouble); + return GenericDatum(e.doubleValue()); + case AVRO_BOOL: + assertType(e, json::etBool); + return GenericDatum(e.boolValue()); + case AVRO_NULL: + assertType(e, json::etNull); + return GenericDatum(); + case AVRO_RECORD: + { + assertType(e, json::etObject); + GenericRecord result(n); + const map<string, Entity>& v = e.objectValue(); + for (size_t i = 0; i < n->leaves(); ++i) { + map<string, Entity>::const_iterator it = v.find(n->nameAt(i)); + if (it == v.end()) { + throw Exception(boost::format( + "No value found in default for %1%") % n->nameAt(i)); + } + result.setFieldAt(i, + makeGenericDatum(n->leafAt(i), it->second, st)); + } + return GenericDatum(n, result); + } + case AVRO_ENUM: + assertType(e, json::etString); + return GenericDatum(n, GenericEnum(n, e.stringValue())); + case AVRO_ARRAY: + { + assertType(e, json::etArray); + GenericArray result(n); + const vector<Entity>& elements = e.arrayValue(); + for (vector<Entity>::const_iterator it = elements.begin(); + it != elements.end(); ++it) { + result.value().push_back(makeGenericDatum(n->leafAt(0), *it, st)); + } + return GenericDatum(n, result); + } + case AVRO_MAP: + { + assertType(e, json::etObject); + GenericMap result(n); + const map<string, Entity>& v = e.objectValue(); + for (map<string, Entity>::const_iterator it = v.begin(); + it != v.end(); ++it) { + result.value().push_back(make_pair(it->first, + makeGenericDatum(n->leafAt(1), it->second, st))); + } + return GenericDatum(n, result); + } + case AVRO_UNION: + { + GenericUnion result(n); + result.selectBranch(0); + result.datum() = makeGenericDatum(n->leafAt(0), e, st); + return GenericDatum(n, result); + } + case AVRO_FIXED: + assertType(e, json::etString); + return GenericDatum(n, GenericFixed(n, toBin(e.bytesValue()))); + default: + throw Exception(boost::format("Unknown type: %1%") % t); + } + return GenericDatum(); +} + + +static Field makeField(const Entity& e, SymbolTable& st, const string& ns) +{ + const Object& m = e.objectValue(); + const string& n = getStringField(e, m, "name"); + Object::const_iterator it = findField(e, m, "type"); + map<string, Entity>::const_iterator it2 = m.find("default"); + NodePtr node = makeNode(it->second, st, ns); + if (containsField(m, "doc")) { + node->setDoc(getDocField(e, m)); + } + GenericDatum d = (it2 == m.end()) ? GenericDatum() : + makeGenericDatum(node, it2->second, st); + return Field(n, node, d); +} + +// Extended makeRecordNode (with doc). +static NodePtr makeRecordNode(const Entity& e, const Name& name, + const string* doc, const Object& m, + SymbolTable& st, const string& ns) { + const Array& v = getArrayField(e, m, "fields"); + concepts::MultiAttribute<string> fieldNames; + concepts::MultiAttribute<NodePtr> fieldValues; + vector<GenericDatum> defaultValues; + + for (Array::const_iterator it = v.begin(); it != v.end(); ++it) { + Field f = makeField(*it, st, ns); + fieldNames.add(f.name); + fieldValues.add(f.schema); + defaultValues.push_back(f.defaultValue); + } + NodeRecord* node; + if (doc == NULL) { + node = new NodeRecord(asSingleAttribute(name), fieldValues, fieldNames, + defaultValues); + } else { + node = new NodeRecord(asSingleAttribute(name), asSingleAttribute(*doc), + fieldValues, fieldNames, defaultValues); + } + return NodePtr(node); +} + +static LogicalType makeLogicalType(const Entity& e, const Object& m) { + if (!containsField(m, "logicalType")) { + return LogicalType(LogicalType::NONE); + } + + const std::string& typeField = getStringField(e, m, "logicalType"); + + if (typeField == "decimal") { + LogicalType decimalType(LogicalType::DECIMAL); + try { + decimalType.setPrecision(getLongField(e, m, "precision")); + if (containsField(m, "scale")) { + decimalType.setScale(getLongField(e, m, "scale")); + } + } catch (Exception& ex) { + // If any part of the logical type is malformed, per the standard we + // must ignore the whole attribute. + return LogicalType(LogicalType::NONE); + } + return decimalType; + } + + LogicalType::Type t = LogicalType::NONE; + if (typeField == "date") + t = LogicalType::DATE; + else if (typeField == "time-millis") + t = LogicalType::TIME_MILLIS; + else if (typeField == "time-micros") + t = LogicalType::TIME_MICROS; + else if (typeField == "timestamp-millis") + t = LogicalType::TIMESTAMP_MILLIS; + else if (typeField == "timestamp-micros") + t = LogicalType::TIMESTAMP_MICROS; + else if (typeField == "duration") + t = LogicalType::DURATION; + else if (typeField == "uuid") + t = LogicalType::UUID; + return LogicalType(t); +} + +static NodePtr makeEnumNode(const Entity& e, + const Name& name, const Object& m) +{ + const Array& v = getArrayField(e, m, "symbols"); + concepts::MultiAttribute<string> symbols; + for (Array::const_iterator it = v.begin(); it != v.end(); ++it) { + if (it->type() != json::etString) { + throw Exception(boost::format("Enum symbol not a string: %1%") % + it->toString()); + } + symbols.add(it->stringValue()); + } + NodePtr node = NodePtr(new NodeEnum(asSingleAttribute(name), symbols)); + if (containsField(m, "doc")) { + node->setDoc(getDocField(e, m)); + } + return node; +} + +static NodePtr makeFixedNode(const Entity& e, + const Name& name, const Object& m) +{ + int v = static_cast<int>(getLongField(e, m, "size")); + if (v <= 0) { + throw Exception(boost::format("Size for fixed is not positive: %1%") % + e.toString()); + } + NodePtr node = + NodePtr(new NodeFixed(asSingleAttribute(name), asSingleAttribute(v))); + if (containsField(m, "doc")) { + node->setDoc(getDocField(e, m)); + } + return node; +} + +static NodePtr makeArrayNode(const Entity& e, const Object& m, + SymbolTable& st, const string& ns) +{ + Object::const_iterator it = findField(e, m, "items"); + NodePtr node = NodePtr(new NodeArray( + asSingleAttribute(makeNode(it->second, st, ns)))); + if (containsField(m, "doc")) { + node->setDoc(getDocField(e, m)); + } + return node; +} + +static NodePtr makeMapNode(const Entity& e, const Object& m, + SymbolTable& st, const string& ns) +{ + Object::const_iterator it = findField(e, m, "values"); + + NodePtr node = NodePtr(new NodeMap( + asSingleAttribute(makeNode(it->second, st, ns)))); + if (containsField(m, "doc")) { + node->setDoc(getDocField(e, m)); + } + return node; +} + +static Name getName(const Entity& e, const Object& m, const string& ns) +{ + const string& name = getStringField(e, m, "name"); + + if (isFullName(name)) { + return Name(name); + } else { + Object::const_iterator it = m.find("namespace"); + if (it != m.end()) { + if (it->second.type() != json::type_traits<string>::type()) { + throw Exception(boost::format( + "Json field \"%1%\" is not a %2%: %3%") % + "namespace" % json::type_traits<string>::name() % + it->second.toString()); + } + Name result = Name(name, it->second.stringValue()); + return result; + } + return Name(name, ns); + } +} + +static NodePtr makeNode(const Entity& e, const Object& m, + SymbolTable& st, const string& ns) +{ + const string& type = getStringField(e, m, "type"); + NodePtr result; + if (type == "record" || type == "error" || + type == "enum" || type == "fixed") { + Name nm = getName(e, m, ns); + if (type == "record" || type == "error") { + result = NodePtr(new NodeRecord()); + st[nm] = result; + // Get field doc + if (containsField(m, "doc")) { + string doc = getDocField(e, m); + + NodePtr r = makeRecordNode(e, nm, &doc, m, st, nm.ns()); + (std::dynamic_pointer_cast<NodeRecord>(r))->swap( + *std::dynamic_pointer_cast<NodeRecord>(result)); + } else { // No doc + NodePtr r = + makeRecordNode(e, nm, NULL, m, st, nm.ns()); + (std::dynamic_pointer_cast<NodeRecord>(r)) + ->swap(*std::dynamic_pointer_cast<NodeRecord>(result)); + } + } else { + result = (type == "enum") ? makeEnumNode(e, nm, m) : + makeFixedNode(e, nm, m); + st[nm] = result; + } + } else if (type == "array") { + result = makeArrayNode(e, m, st, ns); + } else if (type == "map") { + result = makeMapNode(e, m, st, ns); + } else { + result = makePrimitive(type); + } + + if (result) { + try { + result->setLogicalType(makeLogicalType(e, m)); + } catch (Exception& ex) { + // Per the standard we must ignore the logical type attribute if it + // is malformed. + } + return result; + } + + throw Exception(boost::format("Unknown type definition: %1%") + % e.toString()); +} + +static NodePtr makeNode(const Entity& e, const Array& m, + SymbolTable& st, const string& ns) +{ + concepts::MultiAttribute<NodePtr> mm; + for (Array::const_iterator it = m.begin(); it != m.end(); ++it) { + mm.add(makeNode(*it, st, ns)); + } + return NodePtr(new NodeUnion(mm)); +} + +static NodePtr makeNode(const json::Entity& e, SymbolTable& st, const string& ns) +{ + switch (e.type()) { + case json::etString: + return makeNode(e.stringValue(), st, ns); + case json::etObject: + return makeNode(e, e.objectValue(), st, ns); + case json::etArray: + return makeNode(e, e.arrayValue(), st, ns); + default: + throw Exception(boost::format("Invalid Avro type: %1%") % e.toString()); + } +} + +ValidSchema compileJsonSchemaFromStream(InputStream& is) +{ + json::Entity e = json::loadEntity(is); + SymbolTable st; + NodePtr n = makeNode(e, st, ""); + return ValidSchema(n); +} + +AVRO_DECL ValidSchema compileJsonSchemaFromFile(const char* filename) +{ + std::unique_ptr<InputStream> s = fileInputStream(filename); + return compileJsonSchemaFromStream(*s); +} + +AVRO_DECL ValidSchema compileJsonSchemaFromMemory(const uint8_t* input, size_t len) +{ + return compileJsonSchemaFromStream(*memoryInputStream(input, len)); +} + +AVRO_DECL ValidSchema compileJsonSchemaFromString(const char* input) +{ + return compileJsonSchemaFromMemory(reinterpret_cast<const uint8_t*>(input), + ::strlen(input)); +} + +AVRO_DECL ValidSchema compileJsonSchemaFromString(const string& input) +{ + return compileJsonSchemaFromMemory( + reinterpret_cast<const uint8_t*>(input.data()), input.size()); +} + +static ValidSchema compile(std::istream& is) +{ + std::unique_ptr<InputStream> in = istreamInputStream(is); + return compileJsonSchemaFromStream(*in); +} + +void compileJsonSchema(std::istream &is, ValidSchema &schema) +{ + if (!is.good()) { + throw Exception("Input stream is not good"); + } + + schema = compile(is); +} + +AVRO_DECL bool compileJsonSchema(std::istream &is, ValidSchema &schema, string &error) +{ + try { + compileJsonSchema(is, schema); + return true; + } catch (const Exception &e) { + error = e.what(); + return false; + } + +} + +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/DataFile.cc b/contrib/libs/apache/avro/impl/DataFile.cc index 8b92440d04..e20e605827 100644 --- a/contrib/libs/apache/avro/impl/DataFile.cc +++ b/contrib/libs/apache/avro/impl/DataFile.cc @@ -1,600 +1,600 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "DataFile.hh" -#include "Compiler.hh" -#include "Exception.hh" - -#include <sstream> - -#include <boost/random/mersenne_twister.hpp> -#include <boost/iostreams/device/file.hpp> -#include <boost/iostreams/filter/gzip.hpp> -#include <boost/iostreams/filter/zlib.hpp> -#include <boost/crc.hpp> // for boost::crc_32_type - -#ifdef SNAPPY_CODEC_AVAILABLE -#include <snappy.h> -#endif - -namespace avro { -using std::unique_ptr; -using std::ostringstream; -using std::istringstream; -using std::vector; -using std::copy; -using std::string; - -using std::array; - -namespace { -const string AVRO_SCHEMA_KEY("avro.schema"); -const string AVRO_CODEC_KEY("avro.codec"); -const string AVRO_NULL_CODEC("null"); -const string AVRO_DEFLATE_CODEC("deflate"); - -#ifdef SNAPPY_CODEC_AVAILABLE -const string AVRO_SNAPPY_CODEC = "snappy"; -#endif - -const size_t minSyncInterval = 32; -const size_t maxSyncInterval = 1u << 30; - -boost::iostreams::zlib_params get_zlib_params() { - boost::iostreams::zlib_params ret; - ret.method = boost::iostreams::zlib::deflated; - ret.noheader = true; - return ret; -} -} - -DataFileWriterBase::DataFileWriterBase(const char* filename, const ValidSchema& schema, size_t syncInterval, - Codec codec) : - filename_(filename), - schema_(schema), - encoderPtr_(binaryEncoder()), - syncInterval_(syncInterval), - codec_(codec), - stream_(fileOutputStream(filename)), - buffer_(memoryOutputStream()), - sync_(makeSync()), - objectCount_(0), - lastSync_(0) -{ - init(schema, syncInterval, codec); -} - -DataFileWriterBase::DataFileWriterBase(std::unique_ptr<OutputStream> outputStream, - const ValidSchema& schema, size_t syncInterval, Codec codec) : - filename_(), - schema_(schema), - encoderPtr_(binaryEncoder()), - syncInterval_(syncInterval), - codec_(codec), - stream_(std::move(outputStream)), - buffer_(memoryOutputStream()), - sync_(makeSync()), - objectCount_(0), - lastSync_(0) -{ - init(schema, syncInterval, codec); -} - -void DataFileWriterBase::init(const ValidSchema &schema, size_t syncInterval, const Codec &codec) { - if (syncInterval < minSyncInterval || syncInterval > maxSyncInterval) { - throw Exception(boost::format("Invalid sync interval: %1%. " - "Should be between %2% and %3%") % syncInterval % - minSyncInterval % maxSyncInterval); - } - setMetadata(AVRO_CODEC_KEY, AVRO_NULL_CODEC); - - if (codec_ == NULL_CODEC) { - setMetadata(AVRO_CODEC_KEY, AVRO_NULL_CODEC); - } else if (codec_ == DEFLATE_CODEC) { - setMetadata(AVRO_CODEC_KEY, AVRO_DEFLATE_CODEC); -#ifdef SNAPPY_CODEC_AVAILABLE - } else if (codec_ == SNAPPY_CODEC) { - setMetadata(AVRO_CODEC_KEY, AVRO_SNAPPY_CODEC); -#endif - } else { - throw Exception(boost::format("Unknown codec: %1%") % codec); - } - setMetadata(AVRO_SCHEMA_KEY, schema.toJson(false)); - - writeHeader(); - encoderPtr_->init(*buffer_); - - lastSync_ = stream_->byteCount(); -} - - -DataFileWriterBase::~DataFileWriterBase() -{ - if (stream_.get()) { - close(); - } -} - -void DataFileWriterBase::close() -{ - flush(); - stream_.reset(); -} - -void DataFileWriterBase::sync() -{ - encoderPtr_->flush(); - - encoderPtr_->init(*stream_); - avro::encode(*encoderPtr_, objectCount_); - if (codec_ == NULL_CODEC) { - int64_t byteCount = buffer_->byteCount(); - avro::encode(*encoderPtr_, byteCount); - encoderPtr_->flush(); - std::unique_ptr<InputStream> in = memoryInputStream(*buffer_); - copy(*in, *stream_); - } else if (codec_ == DEFLATE_CODEC) { - std::vector<char> buf; - { - boost::iostreams::filtering_ostream os; - os.push(boost::iostreams::zlib_compressor(get_zlib_params())); - os.push(boost::iostreams::back_inserter(buf)); - const uint8_t* data; - size_t len; - - std::unique_ptr<InputStream> input = memoryInputStream(*buffer_); - while (input->next(&data, &len)) { - boost::iostreams::write(os, reinterpret_cast<const char*>(data), len); - } - } // make sure all is flushed - std::unique_ptr<InputStream> in = memoryInputStream( - reinterpret_cast<const uint8_t*>(buf.data()), buf.size()); - int64_t byteCount = buf.size(); - avro::encode(*encoderPtr_, byteCount); - encoderPtr_->flush(); - copy(*in, *stream_); -#ifdef SNAPPY_CODEC_AVAILABLE - } else if (codec_ == SNAPPY_CODEC) { - std::vector<char> temp; - std::string compressed; - boost::crc_32_type crc; - { - boost::iostreams::filtering_ostream os; - os.push(boost::iostreams::back_inserter(temp)); - const uint8_t* data; - size_t len; - - std::unique_ptr<InputStream> input = memoryInputStream(*buffer_); - while (input->next(&data, &len)) { - boost::iostreams::write(os, reinterpret_cast<const char*>(data), - len); - } - } // make sure all is flushed - - crc.process_bytes(reinterpret_cast<const char*>(temp.data()), - temp.size()); - // For Snappy, add the CRC32 checksum - int32_t checksum = crc(); - - // Now compress - size_t compressed_size = snappy::Compress( - reinterpret_cast<const char*>(temp.data()), temp.size(), - &compressed); - temp.clear(); - { - boost::iostreams::filtering_ostream os; - os.push(boost::iostreams::back_inserter(temp)); - boost::iostreams::write(os, compressed.c_str(), compressed_size); - } - temp.push_back((checksum >> 24) & 0xFF); - temp.push_back((checksum >> 16) & 0xFF); - temp.push_back((checksum >> 8) & 0xFF); - temp.push_back(checksum & 0xFF); - std::unique_ptr<InputStream> in = memoryInputStream( - reinterpret_cast<const uint8_t*>(temp.data()), temp.size()); - int64_t byteCount = temp.size(); - avro::encode(*encoderPtr_, byteCount); - encoderPtr_->flush(); - copy(*in, *stream_); -#endif - } - - encoderPtr_->init(*stream_); - avro::encode(*encoderPtr_, sync_); - encoderPtr_->flush(); - - lastSync_ = stream_->byteCount(); - - buffer_ = memoryOutputStream(); - encoderPtr_->init(*buffer_); - objectCount_ = 0; -} - -void DataFileWriterBase::syncIfNeeded() -{ - encoderPtr_->flush(); - if (buffer_->byteCount() >= syncInterval_) { - sync(); - } -} - -uint64_t DataFileWriterBase::getCurrentBlockStart() -{ - return lastSync_; -} - -void DataFileWriterBase::flush() -{ - sync(); -} - -boost::mt19937 random(static_cast<uint32_t>(time(0))); - -DataFileSync DataFileWriterBase::makeSync() -{ - DataFileSync sync; - for (size_t i = 0; i < sync.size(); ++i) { - sync[i] = random(); - } - return sync; -} - -typedef array<uint8_t, 4> Magic; -static Magic magic = { { 'O', 'b', 'j', '\x01' } }; - -void DataFileWriterBase::writeHeader() -{ - encoderPtr_->init(*stream_); - avro::encode(*encoderPtr_, magic); - avro::encode(*encoderPtr_, metadata_); - avro::encode(*encoderPtr_, sync_); - encoderPtr_->flush(); -} - -void DataFileWriterBase::setMetadata(const string& key, const string& value) -{ - vector<uint8_t> v(value.size()); - copy(value.begin(), value.end(), v.begin()); - metadata_[key] = v; -} - -DataFileReaderBase::DataFileReaderBase(const char* filename) : - filename_(filename), stream_(fileSeekableInputStream(filename)), - decoder_(binaryDecoder()), objectCount_(0), eof_(false), blockStart_(-1), - blockEnd_(-1) -{ - readHeader(); -} - -DataFileReaderBase::DataFileReaderBase(std::unique_ptr<InputStream> inputStream) : - filename_(""), stream_(std::move(inputStream)), - decoder_(binaryDecoder()), objectCount_(0), eof_(false) -{ - readHeader(); -} - -void DataFileReaderBase::init() -{ - readerSchema_ = dataSchema_; - dataDecoder_ = binaryDecoder(); - readDataBlock(); -} - -void DataFileReaderBase::init(const ValidSchema& readerSchema) -{ - readerSchema_ = readerSchema; - dataDecoder_ = (readerSchema_.toJson(true) != dataSchema_.toJson(true)) ? - resolvingDecoder(dataSchema_, readerSchema_, binaryDecoder()) : - binaryDecoder(); - readDataBlock(); -} - -static void drain(InputStream& in) -{ - const uint8_t *p = 0; - size_t n = 0; - while (in.next(&p, &n)); -} - -char hex(unsigned int x) -{ - return x + (x < 10 ? '0' : ('a' - 10)); -} - -std::ostream& operator << (std::ostream& os, const DataFileSync& s) -{ - for (size_t i = 0; i < s.size(); ++i) { - os << hex(s[i] / 16) << hex(s[i] % 16) << ' '; - } - os << std::endl; - return os; -} - - -bool DataFileReaderBase::hasMore() -{ - for (; ;) { - if (eof_) { - return false; - } else if (objectCount_ != 0) { - return true; - } - - dataDecoder_->init(*dataStream_); - drain(*dataStream_); - DataFileSync s; - decoder_->init(*stream_); - avro::decode(*decoder_, s); - if (s != sync_) { - throw Exception("Sync mismatch"); - } - readDataBlock(); - } -} - -class BoundedInputStream : public InputStream { - InputStream& in_; - size_t limit_; - - bool next(const uint8_t** data, size_t* len) { - if (limit_ != 0 && in_.next(data, len)) { - if (*len > limit_) { - in_.backup(*len - limit_); - *len = limit_; - } - limit_ -= *len; - return true; - } - return false; - } - - void backup(size_t len) { - in_.backup(len); - limit_ += len; - } - - void skip(size_t len) { - if (len > limit_) { - len = limit_; - } - in_.skip(len); - limit_ -= len; - } - - size_t byteCount() const { - return in_.byteCount(); - } - -public: - BoundedInputStream(InputStream& in, size_t limit) : - in_(in), limit_(limit) { } -}; - -unique_ptr<InputStream> boundedInputStream(InputStream& in, size_t limit) -{ - return unique_ptr<InputStream>(new BoundedInputStream(in, limit)); -} - -void DataFileReaderBase::readDataBlock() -{ - decoder_->init(*stream_); - blockStart_ = stream_->byteCount(); - const uint8_t* p = 0; - size_t n = 0; - if (! stream_->next(&p, &n)) { - eof_ = true; - return; - } - stream_->backup(n); - avro::decode(*decoder_, objectCount_); - int64_t byteCount; - avro::decode(*decoder_, byteCount); - decoder_->init(*stream_); - blockEnd_ = stream_->byteCount() + byteCount; - - unique_ptr<InputStream> st = boundedInputStream(*stream_, static_cast<size_t>(byteCount)); - if (codec_ == NULL_CODEC) { - dataDecoder_->init(*st); - dataStream_ = std::move(st); -#ifdef SNAPPY_CODEC_AVAILABLE - } else if (codec_ == SNAPPY_CODEC) { - boost::crc_32_type crc; - uint32_t checksum = 0; - compressed_.clear(); - uncompressed.clear(); - const uint8_t* data; - size_t len; - while (st->next(&data, &len)) { - compressed_.insert(compressed_.end(), data, data + len); - } - len = compressed_.size(); - int b1 = compressed_[len - 4] & 0xFF; - int b2 = compressed_[len - 3] & 0xFF; - int b3 = compressed_[len - 2] & 0xFF; - int b4 = compressed_[len - 1] & 0xFF; - - checksum = (b1 << 24) + (b2 << 16) + (b3 << 8) + (b4); - if (!snappy::Uncompress(reinterpret_cast<const char*>(compressed_.data()), - len - 4, &uncompressed)) { - throw Exception( - "Snappy Compression reported an error when decompressing"); - } - crc.process_bytes(uncompressed.c_str(), uncompressed.size()); - uint32_t c = crc(); - if (checksum != c) { - throw Exception(boost::format("Checksum did not match for Snappy compression: Expected: %1%, computed: %2%") % checksum % c); - } - os_.reset(new boost::iostreams::filtering_istream()); - os_->push( - boost::iostreams::basic_array_source<char>(uncompressed.c_str(), - uncompressed.size())); - std::unique_ptr<InputStream> in = istreamInputStream(*os_); - - dataDecoder_->init(*in); - dataStream_ = std::move(in); -#endif - } else { - compressed_.clear(); - const uint8_t* data; - size_t len; - while (st->next(&data, &len)) { - compressed_.insert(compressed_.end(), data, data + len); - } - // boost::iostreams::write(os, reinterpret_cast<const char*>(data), len); - os_.reset(new boost::iostreams::filtering_istream()); - os_->push(boost::iostreams::zlib_decompressor(get_zlib_params())); - os_->push(boost::iostreams::basic_array_source<char>( - compressed_.data(), compressed_.size())); - - std::unique_ptr<InputStream> in = nonSeekableIstreamInputStream(*os_); - dataDecoder_->init(*in); - dataStream_ = std::move(in); - } -} - -void DataFileReaderBase::close() -{ -} - -static string toString(const vector<uint8_t>& v) -{ - string result; - result.resize(v.size()); - copy(v.begin(), v.end(), result.begin()); - return result; -} - -static ValidSchema makeSchema(const vector<uint8_t>& v) -{ - istringstream iss(toString(v)); - ValidSchema vs; - compileJsonSchema(iss, vs); - return ValidSchema(vs); -} - -void DataFileReaderBase::readHeader() -{ - decoder_->init(*stream_); - Magic m; - avro::decode(*decoder_, m); - if (magic != m) { - throw Exception("Invalid data file. Magic does not match: " - + filename_); - } - avro::decode(*decoder_, metadata_); - Metadata::const_iterator it = metadata_.find(AVRO_SCHEMA_KEY); - if (it == metadata_.end()) { - throw Exception("No schema in metadata"); - } - - dataSchema_ = makeSchema(it->second); - if (! readerSchema_.root()) { - readerSchema_ = dataSchema(); - } - - it = metadata_.find(AVRO_CODEC_KEY); - if (it != metadata_.end() && toString(it->second) == AVRO_DEFLATE_CODEC) { - codec_ = DEFLATE_CODEC; -#ifdef SNAPPY_CODEC_AVAILABLE - } else if (it != metadata_.end() - && toString(it->second) == AVRO_SNAPPY_CODEC) { - codec_ = SNAPPY_CODEC; -#endif - } else { - codec_ = NULL_CODEC; - if (it != metadata_.end() && toString(it->second) != AVRO_NULL_CODEC) { - throw Exception("Unknown codec in data file: " + toString(it->second)); - } - } - - avro::decode(*decoder_, sync_); - decoder_->init(*stream_); - blockStart_ = stream_->byteCount(); -} - -void DataFileReaderBase::doSeek(int64_t position) -{ - if (SeekableInputStream *ss = dynamic_cast<SeekableInputStream *>(stream_.get())) { - if (!eof_) { - dataDecoder_->init(*dataStream_); - drain(*dataStream_); - } - decoder_->init(*stream_); - ss->seek(position); - eof_ = false; - } else { - throw Exception("seek not supported on non-SeekableInputStream"); - } -} - -void DataFileReaderBase::seek(int64_t position) -{ - doSeek(position); - readDataBlock(); -} - -void DataFileReaderBase::sync(int64_t position) -{ - doSeek(position); - DataFileSync sync_buffer; - const uint8_t *p = 0; - size_t n = 0; - size_t i = 0; - while (i < SyncSize) { - if (n == 0 && !stream_->next(&p, &n)) { - eof_ = true; - return; - } - int len = - std::min(static_cast<size_t>(SyncSize - i), n); - memcpy(&sync_buffer[i], p, len); - p += len; - n -= len; - i += len; - } - for (;;) { - size_t j = 0; - for (; j < SyncSize; ++j) { - if (sync_[j] != sync_buffer[(i + j) % SyncSize]) { - break; - } - } - if (j == SyncSize) { - // Found the sync marker! - break; - } - if (n == 0 && !stream_->next(&p, &n)) { - eof_ = true; - return; - } - sync_buffer[i++ % SyncSize] = *p++; - --n; - } - stream_->backup(n); - readDataBlock(); -} - -bool DataFileReaderBase::pastSync(int64_t position) { - return !hasMore() || blockStart_ >= position + SyncSize; -} - -int64_t DataFileReaderBase::previousSync() { - return blockStart_; -} - -} // namespace avro +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "DataFile.hh" +#include "Compiler.hh" +#include "Exception.hh" + +#include <sstream> + +#include <boost/random/mersenne_twister.hpp> +#include <boost/iostreams/device/file.hpp> +#include <boost/iostreams/filter/gzip.hpp> +#include <boost/iostreams/filter/zlib.hpp> +#include <boost/crc.hpp> // for boost::crc_32_type + +#ifdef SNAPPY_CODEC_AVAILABLE +#include <snappy.h> +#endif + +namespace avro { +using std::unique_ptr; +using std::ostringstream; +using std::istringstream; +using std::vector; +using std::copy; +using std::string; + +using std::array; + +namespace { +const string AVRO_SCHEMA_KEY("avro.schema"); +const string AVRO_CODEC_KEY("avro.codec"); +const string AVRO_NULL_CODEC("null"); +const string AVRO_DEFLATE_CODEC("deflate"); + +#ifdef SNAPPY_CODEC_AVAILABLE +const string AVRO_SNAPPY_CODEC = "snappy"; +#endif + +const size_t minSyncInterval = 32; +const size_t maxSyncInterval = 1u << 30; + +boost::iostreams::zlib_params get_zlib_params() { + boost::iostreams::zlib_params ret; + ret.method = boost::iostreams::zlib::deflated; + ret.noheader = true; + return ret; +} +} + +DataFileWriterBase::DataFileWriterBase(const char* filename, const ValidSchema& schema, size_t syncInterval, + Codec codec) : + filename_(filename), + schema_(schema), + encoderPtr_(binaryEncoder()), + syncInterval_(syncInterval), + codec_(codec), + stream_(fileOutputStream(filename)), + buffer_(memoryOutputStream()), + sync_(makeSync()), + objectCount_(0), + lastSync_(0) +{ + init(schema, syncInterval, codec); +} + +DataFileWriterBase::DataFileWriterBase(std::unique_ptr<OutputStream> outputStream, + const ValidSchema& schema, size_t syncInterval, Codec codec) : + filename_(), + schema_(schema), + encoderPtr_(binaryEncoder()), + syncInterval_(syncInterval), + codec_(codec), + stream_(std::move(outputStream)), + buffer_(memoryOutputStream()), + sync_(makeSync()), + objectCount_(0), + lastSync_(0) +{ + init(schema, syncInterval, codec); +} + +void DataFileWriterBase::init(const ValidSchema &schema, size_t syncInterval, const Codec &codec) { + if (syncInterval < minSyncInterval || syncInterval > maxSyncInterval) { + throw Exception(boost::format("Invalid sync interval: %1%. " + "Should be between %2% and %3%") % syncInterval % + minSyncInterval % maxSyncInterval); + } + setMetadata(AVRO_CODEC_KEY, AVRO_NULL_CODEC); + + if (codec_ == NULL_CODEC) { + setMetadata(AVRO_CODEC_KEY, AVRO_NULL_CODEC); + } else if (codec_ == DEFLATE_CODEC) { + setMetadata(AVRO_CODEC_KEY, AVRO_DEFLATE_CODEC); +#ifdef SNAPPY_CODEC_AVAILABLE + } else if (codec_ == SNAPPY_CODEC) { + setMetadata(AVRO_CODEC_KEY, AVRO_SNAPPY_CODEC); +#endif + } else { + throw Exception(boost::format("Unknown codec: %1%") % codec); + } + setMetadata(AVRO_SCHEMA_KEY, schema.toJson(false)); + + writeHeader(); + encoderPtr_->init(*buffer_); + + lastSync_ = stream_->byteCount(); +} + + +DataFileWriterBase::~DataFileWriterBase() +{ + if (stream_.get()) { + close(); + } +} + +void DataFileWriterBase::close() +{ + flush(); + stream_.reset(); +} + +void DataFileWriterBase::sync() +{ + encoderPtr_->flush(); + + encoderPtr_->init(*stream_); + avro::encode(*encoderPtr_, objectCount_); + if (codec_ == NULL_CODEC) { + int64_t byteCount = buffer_->byteCount(); + avro::encode(*encoderPtr_, byteCount); + encoderPtr_->flush(); + std::unique_ptr<InputStream> in = memoryInputStream(*buffer_); + copy(*in, *stream_); + } else if (codec_ == DEFLATE_CODEC) { + std::vector<char> buf; + { + boost::iostreams::filtering_ostream os; + os.push(boost::iostreams::zlib_compressor(get_zlib_params())); + os.push(boost::iostreams::back_inserter(buf)); + const uint8_t* data; + size_t len; + + std::unique_ptr<InputStream> input = memoryInputStream(*buffer_); + while (input->next(&data, &len)) { + boost::iostreams::write(os, reinterpret_cast<const char*>(data), len); + } + } // make sure all is flushed + std::unique_ptr<InputStream> in = memoryInputStream( + reinterpret_cast<const uint8_t*>(buf.data()), buf.size()); + int64_t byteCount = buf.size(); + avro::encode(*encoderPtr_, byteCount); + encoderPtr_->flush(); + copy(*in, *stream_); +#ifdef SNAPPY_CODEC_AVAILABLE + } else if (codec_ == SNAPPY_CODEC) { + std::vector<char> temp; + std::string compressed; + boost::crc_32_type crc; + { + boost::iostreams::filtering_ostream os; + os.push(boost::iostreams::back_inserter(temp)); + const uint8_t* data; + size_t len; + + std::unique_ptr<InputStream> input = memoryInputStream(*buffer_); + while (input->next(&data, &len)) { + boost::iostreams::write(os, reinterpret_cast<const char*>(data), + len); + } + } // make sure all is flushed + + crc.process_bytes(reinterpret_cast<const char*>(temp.data()), + temp.size()); + // For Snappy, add the CRC32 checksum + int32_t checksum = crc(); + + // Now compress + size_t compressed_size = snappy::Compress( + reinterpret_cast<const char*>(temp.data()), temp.size(), + &compressed); + temp.clear(); + { + boost::iostreams::filtering_ostream os; + os.push(boost::iostreams::back_inserter(temp)); + boost::iostreams::write(os, compressed.c_str(), compressed_size); + } + temp.push_back((checksum >> 24) & 0xFF); + temp.push_back((checksum >> 16) & 0xFF); + temp.push_back((checksum >> 8) & 0xFF); + temp.push_back(checksum & 0xFF); + std::unique_ptr<InputStream> in = memoryInputStream( + reinterpret_cast<const uint8_t*>(temp.data()), temp.size()); + int64_t byteCount = temp.size(); + avro::encode(*encoderPtr_, byteCount); + encoderPtr_->flush(); + copy(*in, *stream_); +#endif + } + + encoderPtr_->init(*stream_); + avro::encode(*encoderPtr_, sync_); + encoderPtr_->flush(); + + lastSync_ = stream_->byteCount(); + + buffer_ = memoryOutputStream(); + encoderPtr_->init(*buffer_); + objectCount_ = 0; +} + +void DataFileWriterBase::syncIfNeeded() +{ + encoderPtr_->flush(); + if (buffer_->byteCount() >= syncInterval_) { + sync(); + } +} + +uint64_t DataFileWriterBase::getCurrentBlockStart() +{ + return lastSync_; +} + +void DataFileWriterBase::flush() +{ + sync(); +} + +boost::mt19937 random(static_cast<uint32_t>(time(0))); + +DataFileSync DataFileWriterBase::makeSync() +{ + DataFileSync sync; + for (size_t i = 0; i < sync.size(); ++i) { + sync[i] = random(); + } + return sync; +} + +typedef array<uint8_t, 4> Magic; +static Magic magic = { { 'O', 'b', 'j', '\x01' } }; + +void DataFileWriterBase::writeHeader() +{ + encoderPtr_->init(*stream_); + avro::encode(*encoderPtr_, magic); + avro::encode(*encoderPtr_, metadata_); + avro::encode(*encoderPtr_, sync_); + encoderPtr_->flush(); +} + +void DataFileWriterBase::setMetadata(const string& key, const string& value) +{ + vector<uint8_t> v(value.size()); + copy(value.begin(), value.end(), v.begin()); + metadata_[key] = v; +} + +DataFileReaderBase::DataFileReaderBase(const char* filename) : + filename_(filename), stream_(fileSeekableInputStream(filename)), + decoder_(binaryDecoder()), objectCount_(0), eof_(false), blockStart_(-1), + blockEnd_(-1) +{ + readHeader(); +} + +DataFileReaderBase::DataFileReaderBase(std::unique_ptr<InputStream> inputStream) : + filename_(""), stream_(std::move(inputStream)), + decoder_(binaryDecoder()), objectCount_(0), eof_(false) +{ + readHeader(); +} + +void DataFileReaderBase::init() +{ + readerSchema_ = dataSchema_; + dataDecoder_ = binaryDecoder(); + readDataBlock(); +} + +void DataFileReaderBase::init(const ValidSchema& readerSchema) +{ + readerSchema_ = readerSchema; + dataDecoder_ = (readerSchema_.toJson(true) != dataSchema_.toJson(true)) ? + resolvingDecoder(dataSchema_, readerSchema_, binaryDecoder()) : + binaryDecoder(); + readDataBlock(); +} + +static void drain(InputStream& in) +{ + const uint8_t *p = 0; + size_t n = 0; + while (in.next(&p, &n)); +} + +char hex(unsigned int x) +{ + return x + (x < 10 ? '0' : ('a' - 10)); +} + +std::ostream& operator << (std::ostream& os, const DataFileSync& s) +{ + for (size_t i = 0; i < s.size(); ++i) { + os << hex(s[i] / 16) << hex(s[i] % 16) << ' '; + } + os << std::endl; + return os; +} + + +bool DataFileReaderBase::hasMore() +{ + for (; ;) { + if (eof_) { + return false; + } else if (objectCount_ != 0) { + return true; + } + + dataDecoder_->init(*dataStream_); + drain(*dataStream_); + DataFileSync s; + decoder_->init(*stream_); + avro::decode(*decoder_, s); + if (s != sync_) { + throw Exception("Sync mismatch"); + } + readDataBlock(); + } +} + +class BoundedInputStream : public InputStream { + InputStream& in_; + size_t limit_; + + bool next(const uint8_t** data, size_t* len) { + if (limit_ != 0 && in_.next(data, len)) { + if (*len > limit_) { + in_.backup(*len - limit_); + *len = limit_; + } + limit_ -= *len; + return true; + } + return false; + } + + void backup(size_t len) { + in_.backup(len); + limit_ += len; + } + + void skip(size_t len) { + if (len > limit_) { + len = limit_; + } + in_.skip(len); + limit_ -= len; + } + + size_t byteCount() const { + return in_.byteCount(); + } + +public: + BoundedInputStream(InputStream& in, size_t limit) : + in_(in), limit_(limit) { } +}; + +unique_ptr<InputStream> boundedInputStream(InputStream& in, size_t limit) +{ + return unique_ptr<InputStream>(new BoundedInputStream(in, limit)); +} + +void DataFileReaderBase::readDataBlock() +{ + decoder_->init(*stream_); + blockStart_ = stream_->byteCount(); + const uint8_t* p = 0; + size_t n = 0; + if (! stream_->next(&p, &n)) { + eof_ = true; + return; + } + stream_->backup(n); + avro::decode(*decoder_, objectCount_); + int64_t byteCount; + avro::decode(*decoder_, byteCount); + decoder_->init(*stream_); + blockEnd_ = stream_->byteCount() + byteCount; + + unique_ptr<InputStream> st = boundedInputStream(*stream_, static_cast<size_t>(byteCount)); + if (codec_ == NULL_CODEC) { + dataDecoder_->init(*st); + dataStream_ = std::move(st); +#ifdef SNAPPY_CODEC_AVAILABLE + } else if (codec_ == SNAPPY_CODEC) { + boost::crc_32_type crc; + uint32_t checksum = 0; + compressed_.clear(); + uncompressed.clear(); + const uint8_t* data; + size_t len; + while (st->next(&data, &len)) { + compressed_.insert(compressed_.end(), data, data + len); + } + len = compressed_.size(); + int b1 = compressed_[len - 4] & 0xFF; + int b2 = compressed_[len - 3] & 0xFF; + int b3 = compressed_[len - 2] & 0xFF; + int b4 = compressed_[len - 1] & 0xFF; + + checksum = (b1 << 24) + (b2 << 16) + (b3 << 8) + (b4); + if (!snappy::Uncompress(reinterpret_cast<const char*>(compressed_.data()), + len - 4, &uncompressed)) { + throw Exception( + "Snappy Compression reported an error when decompressing"); + } + crc.process_bytes(uncompressed.c_str(), uncompressed.size()); + uint32_t c = crc(); + if (checksum != c) { + throw Exception(boost::format("Checksum did not match for Snappy compression: Expected: %1%, computed: %2%") % checksum % c); + } + os_.reset(new boost::iostreams::filtering_istream()); + os_->push( + boost::iostreams::basic_array_source<char>(uncompressed.c_str(), + uncompressed.size())); + std::unique_ptr<InputStream> in = istreamInputStream(*os_); + + dataDecoder_->init(*in); + dataStream_ = std::move(in); +#endif + } else { + compressed_.clear(); + const uint8_t* data; + size_t len; + while (st->next(&data, &len)) { + compressed_.insert(compressed_.end(), data, data + len); + } + // boost::iostreams::write(os, reinterpret_cast<const char*>(data), len); + os_.reset(new boost::iostreams::filtering_istream()); + os_->push(boost::iostreams::zlib_decompressor(get_zlib_params())); + os_->push(boost::iostreams::basic_array_source<char>( + compressed_.data(), compressed_.size())); + + std::unique_ptr<InputStream> in = nonSeekableIstreamInputStream(*os_); + dataDecoder_->init(*in); + dataStream_ = std::move(in); + } +} + +void DataFileReaderBase::close() +{ +} + +static string toString(const vector<uint8_t>& v) +{ + string result; + result.resize(v.size()); + copy(v.begin(), v.end(), result.begin()); + return result; +} + +static ValidSchema makeSchema(const vector<uint8_t>& v) +{ + istringstream iss(toString(v)); + ValidSchema vs; + compileJsonSchema(iss, vs); + return ValidSchema(vs); +} + +void DataFileReaderBase::readHeader() +{ + decoder_->init(*stream_); + Magic m; + avro::decode(*decoder_, m); + if (magic != m) { + throw Exception("Invalid data file. Magic does not match: " + + filename_); + } + avro::decode(*decoder_, metadata_); + Metadata::const_iterator it = metadata_.find(AVRO_SCHEMA_KEY); + if (it == metadata_.end()) { + throw Exception("No schema in metadata"); + } + + dataSchema_ = makeSchema(it->second); + if (! readerSchema_.root()) { + readerSchema_ = dataSchema(); + } + + it = metadata_.find(AVRO_CODEC_KEY); + if (it != metadata_.end() && toString(it->second) == AVRO_DEFLATE_CODEC) { + codec_ = DEFLATE_CODEC; +#ifdef SNAPPY_CODEC_AVAILABLE + } else if (it != metadata_.end() + && toString(it->second) == AVRO_SNAPPY_CODEC) { + codec_ = SNAPPY_CODEC; +#endif + } else { + codec_ = NULL_CODEC; + if (it != metadata_.end() && toString(it->second) != AVRO_NULL_CODEC) { + throw Exception("Unknown codec in data file: " + toString(it->second)); + } + } + + avro::decode(*decoder_, sync_); + decoder_->init(*stream_); + blockStart_ = stream_->byteCount(); +} + +void DataFileReaderBase::doSeek(int64_t position) +{ + if (SeekableInputStream *ss = dynamic_cast<SeekableInputStream *>(stream_.get())) { + if (!eof_) { + dataDecoder_->init(*dataStream_); + drain(*dataStream_); + } + decoder_->init(*stream_); + ss->seek(position); + eof_ = false; + } else { + throw Exception("seek not supported on non-SeekableInputStream"); + } +} + +void DataFileReaderBase::seek(int64_t position) +{ + doSeek(position); + readDataBlock(); +} + +void DataFileReaderBase::sync(int64_t position) +{ + doSeek(position); + DataFileSync sync_buffer; + const uint8_t *p = 0; + size_t n = 0; + size_t i = 0; + while (i < SyncSize) { + if (n == 0 && !stream_->next(&p, &n)) { + eof_ = true; + return; + } + int len = + std::min(static_cast<size_t>(SyncSize - i), n); + memcpy(&sync_buffer[i], p, len); + p += len; + n -= len; + i += len; + } + for (;;) { + size_t j = 0; + for (; j < SyncSize; ++j) { + if (sync_[j] != sync_buffer[(i + j) % SyncSize]) { + break; + } + } + if (j == SyncSize) { + // Found the sync marker! + break; + } + if (n == 0 && !stream_->next(&p, &n)) { + eof_ = true; + return; + } + sync_buffer[i++ % SyncSize] = *p++; + --n; + } + stream_->backup(n); + readDataBlock(); +} + +bool DataFileReaderBase::pastSync(int64_t position) { + return !hasMore() || blockStart_ >= position + SyncSize; +} + +int64_t DataFileReaderBase::previousSync() { + return blockStart_; +} + +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/FileStream.cc b/contrib/libs/apache/avro/impl/FileStream.cc index 03013a1224..ed601b4c6f 100644 --- a/contrib/libs/apache/avro/impl/FileStream.cc +++ b/contrib/libs/apache/avro/impl/FileStream.cc @@ -1,397 +1,397 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <fstream> -#include "Stream.hh" -#ifndef _WIN32 -#include "unistd.h" -#include "fcntl.h" -#include "errno.h" - -#ifndef O_BINARY -#define O_BINARY 0 -#endif -#else -#include "Windows.h" - -#ifdef min -#undef min -#endif -#endif - -using std::unique_ptr; -using std::istream; -using std::ostream; - -namespace avro { -namespace { -struct BufferCopyIn { - virtual ~BufferCopyIn() { } - virtual void seek(size_t len) = 0; - virtual bool read(uint8_t* b, size_t toRead, size_t& actual) = 0; - -}; - -struct FileBufferCopyIn : public BufferCopyIn { -#ifdef _WIN32 - HANDLE h_; - FileBufferCopyIn(const char* filename) : - h_(::CreateFileA(filename, GENERIC_READ, 0, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL)) { - if (h_ == INVALID_HANDLE_VALUE) { - throw Exception(boost::format("Cannot open file: %1%") % ::GetLastError()); - } - } - - ~FileBufferCopyIn() { - ::CloseHandle(h_); - } - - void seek(size_t len) { - if (::SetFilePointer(h_, len, NULL, FILE_CURRENT) == INVALID_SET_FILE_POINTER && ::GetLastError() != NO_ERROR) { - throw Exception(boost::format("Cannot skip file: %1%") % ::GetLastError()); - } - } - - bool read(uint8_t* b, size_t toRead, size_t& actual) { - DWORD dw = 0; - if (! ::ReadFile(h_, b, toRead, &dw, NULL)) { - throw Exception(boost::format("Cannot read file: %1%") % ::GetLastError()); - } - actual = static_cast<size_t>(dw); - return actual != 0; - } -#else - const int fd_; - - FileBufferCopyIn(const char* filename) : - fd_(open(filename, O_RDONLY | O_BINARY)) { - if (fd_ < 0) { - throw Exception(boost::format("Cannot open file: %1%") % - ::strerror(errno)); - } - } - - ~FileBufferCopyIn() { - ::close(fd_); - } - - void seek(size_t len) { - off_t r = ::lseek(fd_, len, SEEK_CUR); - if (r == static_cast<off_t>(-1)) { - throw Exception(boost::format("Cannot skip file: %1%") % - strerror(errno)); - } - } - - bool read(uint8_t* b, size_t toRead, size_t& actual) { - int n = ::read(fd_, b, toRead); - if (n > 0) { - actual = n; - return true; - } - return false; - } -#endif - -}; - -struct IStreamBufferCopyIn : public BufferCopyIn { - istream& is_; - - IStreamBufferCopyIn(istream& is) : is_(is) { - } - - void seek(size_t len) { - if (! is_.seekg(len, std::ios_base::cur)) { - throw Exception("Cannot skip stream"); - } - } - - bool read(uint8_t* b, size_t toRead, size_t& actual) { - is_.read(reinterpret_cast<char*>(b), toRead); - if (is_.bad()) { - return false; - } - actual = static_cast<size_t>(is_.gcount()); - return (! is_.eof() || actual != 0); - } - -}; - -struct NonSeekableIStreamBufferCopyIn : public IStreamBufferCopyIn { - NonSeekableIStreamBufferCopyIn(istream& is) : IStreamBufferCopyIn(is) { } - - void seek(size_t len) { - const size_t bufSize = 4096; - uint8_t buf[bufSize]; - while (len > 0) { - size_t n = std::min(len, bufSize); - is_.read(reinterpret_cast<char*>(buf), n); - if (is_.bad()) { - throw Exception("Cannot skip stream"); - } - size_t actual = static_cast<size_t>(is_.gcount()); - if (is_.eof() && actual == 0) { - throw Exception("Cannot skip stream"); - } - len -= n; - } - } -}; - -} - -class BufferCopyInInputStream : public SeekableInputStream { - const size_t bufferSize_; - uint8_t* const buffer_; - unique_ptr<BufferCopyIn> in_; - size_t byteCount_; - uint8_t* next_; - size_t available_; - - bool next(const uint8_t** data, size_t *size) { - if (available_ == 0 && ! fill()) { - return false; - } - *data = next_; - *size = available_; - next_ += available_; - byteCount_ += available_; - available_ = 0; - return true; - } - - void backup(size_t len) { - next_ -= len; - available_ += len; - byteCount_ -= len; - } - - void skip(size_t len) { - while (len > 0) { - if (available_ == 0) { - in_->seek(len); - byteCount_ += len; - return; - } - size_t n = std::min(available_, len); - available_ -= n; - next_ += n; - len -= n; - byteCount_ += n; - } - } - - size_t byteCount() const { return byteCount_; } - - bool fill() { - size_t n = 0; - if (in_->read(buffer_, bufferSize_, n)) { - next_ = buffer_; - available_ = n; - return true; - } - return false; - } - - void seek(int64_t position) { - // BufferCopyIn::seek is relative to byteCount_, whereas position is - // absolute. - in_->seek(position - byteCount_ - available_); - byteCount_ = position; - available_ = 0; - } - -public: - BufferCopyInInputStream(unique_ptr<BufferCopyIn> in, size_t bufferSize) : - bufferSize_(bufferSize), - buffer_(new uint8_t[bufferSize]), - in_(std::move(in)), - byteCount_(0), - next_(buffer_), - available_(0) { } - - ~BufferCopyInInputStream() { - delete[] buffer_; - } -}; - -namespace { -struct BufferCopyOut { - virtual ~BufferCopyOut() { } - virtual void write(const uint8_t* b, size_t len) = 0; -}; - -struct FileBufferCopyOut : public BufferCopyOut { -#ifdef _WIN32 - HANDLE h_; - FileBufferCopyOut(const char* filename) : - h_(::CreateFileA(filename, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL)) { - if (h_ == INVALID_HANDLE_VALUE) { - throw Exception(boost::format("Cannot open file: %1%") % ::GetLastError()); - } - } - - ~FileBufferCopyOut() { - ::CloseHandle(h_); - } - - void write(const uint8_t* b, size_t len) { - while (len > 0) { - DWORD dw = 0; - if (! ::WriteFile(h_, b, len, &dw, NULL)) { - throw Exception(boost::format("Cannot read file: %1%") % ::GetLastError()); - } - b += dw; - len -= dw; - } - } -#else - const int fd_; - - FileBufferCopyOut(const char* filename) : - fd_(::open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644)) { - - if (fd_ < 0) { - throw Exception(boost::format("Cannot open file: %1%") % - ::strerror(errno)); - } - } - - ~FileBufferCopyOut() { - ::close(fd_); - } - - void write(const uint8_t* b, size_t len) { - if (::write(fd_, b, len) < 0) { - throw Exception(boost::format("Cannot write file: %1%") % - ::strerror(errno)); - } - } -#endif - -}; - -struct OStreamBufferCopyOut : public BufferCopyOut { - ostream& os_; - - OStreamBufferCopyOut(ostream& os) : os_(os) { - } - - void write(const uint8_t* b, size_t len) { - os_.write(reinterpret_cast<const char*>(b), len); - } - -}; - -} - -class BufferCopyOutputStream : public OutputStream { - size_t bufferSize_; - uint8_t* const buffer_; - unique_ptr<BufferCopyOut> out_; - uint8_t* next_; - size_t available_; - size_t byteCount_; - - // Invaiant: byteCount_ == byteswritten + bufferSize_ - available_; - bool next(uint8_t** data, size_t* len) { - if (available_ == 0) { - flush(); - } - *data = next_; - *len = available_; - next_ += available_; - byteCount_ += available_; - available_ = 0; - return true; - } - - void backup(size_t len) { - available_ += len; - next_ -= len; - byteCount_ -= len; - } - - uint64_t byteCount() const { - return byteCount_; - } - - void flush() { - out_->write(buffer_, bufferSize_ - available_); - next_ = buffer_; - available_ = bufferSize_; - } - -public: - BufferCopyOutputStream(unique_ptr<BufferCopyOut> out, size_t bufferSize) : - bufferSize_(bufferSize), - buffer_(new uint8_t[bufferSize]), - out_(std::move(out)), - next_(buffer_), - available_(bufferSize_), byteCount_(0) { } - - ~BufferCopyOutputStream() { - delete[] buffer_; - } -}; - -unique_ptr<InputStream> fileInputStream(const char* filename, - size_t bufferSize) -{ - unique_ptr<BufferCopyIn> in(new FileBufferCopyIn(filename)); - return unique_ptr<InputStream>( new BufferCopyInInputStream(std::move(in), bufferSize)); -} - -unique_ptr<SeekableInputStream> fileSeekableInputStream(const char* filename, - size_t bufferSize) -{ - unique_ptr<BufferCopyIn> in(new FileBufferCopyIn(filename)); - return unique_ptr<SeekableInputStream>( new BufferCopyInInputStream(std::move(in), - bufferSize)); -} - -unique_ptr<InputStream> istreamInputStream(istream& is, size_t bufferSize) -{ - unique_ptr<BufferCopyIn> in(new IStreamBufferCopyIn(is)); - return unique_ptr<InputStream>( new BufferCopyInInputStream(std::move(in), bufferSize)); -} - -unique_ptr<InputStream> nonSeekableIstreamInputStream( - istream& is, size_t bufferSize) -{ - unique_ptr<BufferCopyIn> in(new NonSeekableIStreamBufferCopyIn(is)); - return unique_ptr<InputStream>( new BufferCopyInInputStream(std::move(in), bufferSize)); -} - -unique_ptr<OutputStream> fileOutputStream(const char* filename, - size_t bufferSize) -{ - unique_ptr<BufferCopyOut> out(new FileBufferCopyOut(filename)); - return unique_ptr<OutputStream>(new BufferCopyOutputStream(std::move(out), bufferSize)); -} - -unique_ptr<OutputStream> ostreamOutputStream(ostream& os, - size_t bufferSize) -{ - unique_ptr<BufferCopyOut> out(new OStreamBufferCopyOut(os)); - return unique_ptr<OutputStream>(new BufferCopyOutputStream(std::move(out), bufferSize)); -} - - -} // namespace avro +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <fstream> +#include "Stream.hh" +#ifndef _WIN32 +#include "unistd.h" +#include "fcntl.h" +#include "errno.h" + +#ifndef O_BINARY +#define O_BINARY 0 +#endif +#else +#include "Windows.h" + +#ifdef min +#undef min +#endif +#endif + +using std::unique_ptr; +using std::istream; +using std::ostream; + +namespace avro { +namespace { +struct BufferCopyIn { + virtual ~BufferCopyIn() { } + virtual void seek(size_t len) = 0; + virtual bool read(uint8_t* b, size_t toRead, size_t& actual) = 0; + +}; + +struct FileBufferCopyIn : public BufferCopyIn { +#ifdef _WIN32 + HANDLE h_; + FileBufferCopyIn(const char* filename) : + h_(::CreateFileA(filename, GENERIC_READ, 0, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL)) { + if (h_ == INVALID_HANDLE_VALUE) { + throw Exception(boost::format("Cannot open file: %1%") % ::GetLastError()); + } + } + + ~FileBufferCopyIn() { + ::CloseHandle(h_); + } + + void seek(size_t len) { + if (::SetFilePointer(h_, len, NULL, FILE_CURRENT) == INVALID_SET_FILE_POINTER && ::GetLastError() != NO_ERROR) { + throw Exception(boost::format("Cannot skip file: %1%") % ::GetLastError()); + } + } + + bool read(uint8_t* b, size_t toRead, size_t& actual) { + DWORD dw = 0; + if (! ::ReadFile(h_, b, toRead, &dw, NULL)) { + throw Exception(boost::format("Cannot read file: %1%") % ::GetLastError()); + } + actual = static_cast<size_t>(dw); + return actual != 0; + } +#else + const int fd_; + + FileBufferCopyIn(const char* filename) : + fd_(open(filename, O_RDONLY | O_BINARY)) { + if (fd_ < 0) { + throw Exception(boost::format("Cannot open file: %1%") % + ::strerror(errno)); + } + } + + ~FileBufferCopyIn() { + ::close(fd_); + } + + void seek(size_t len) { + off_t r = ::lseek(fd_, len, SEEK_CUR); + if (r == static_cast<off_t>(-1)) { + throw Exception(boost::format("Cannot skip file: %1%") % + strerror(errno)); + } + } + + bool read(uint8_t* b, size_t toRead, size_t& actual) { + int n = ::read(fd_, b, toRead); + if (n > 0) { + actual = n; + return true; + } + return false; + } +#endif + +}; + +struct IStreamBufferCopyIn : public BufferCopyIn { + istream& is_; + + IStreamBufferCopyIn(istream& is) : is_(is) { + } + + void seek(size_t len) { + if (! is_.seekg(len, std::ios_base::cur)) { + throw Exception("Cannot skip stream"); + } + } + + bool read(uint8_t* b, size_t toRead, size_t& actual) { + is_.read(reinterpret_cast<char*>(b), toRead); + if (is_.bad()) { + return false; + } + actual = static_cast<size_t>(is_.gcount()); + return (! is_.eof() || actual != 0); + } + +}; + +struct NonSeekableIStreamBufferCopyIn : public IStreamBufferCopyIn { + NonSeekableIStreamBufferCopyIn(istream& is) : IStreamBufferCopyIn(is) { } + + void seek(size_t len) { + const size_t bufSize = 4096; + uint8_t buf[bufSize]; + while (len > 0) { + size_t n = std::min(len, bufSize); + is_.read(reinterpret_cast<char*>(buf), n); + if (is_.bad()) { + throw Exception("Cannot skip stream"); + } + size_t actual = static_cast<size_t>(is_.gcount()); + if (is_.eof() && actual == 0) { + throw Exception("Cannot skip stream"); + } + len -= n; + } + } +}; + +} + +class BufferCopyInInputStream : public SeekableInputStream { + const size_t bufferSize_; + uint8_t* const buffer_; + unique_ptr<BufferCopyIn> in_; + size_t byteCount_; + uint8_t* next_; + size_t available_; + + bool next(const uint8_t** data, size_t *size) { + if (available_ == 0 && ! fill()) { + return false; + } + *data = next_; + *size = available_; + next_ += available_; + byteCount_ += available_; + available_ = 0; + return true; + } + + void backup(size_t len) { + next_ -= len; + available_ += len; + byteCount_ -= len; + } + + void skip(size_t len) { + while (len > 0) { + if (available_ == 0) { + in_->seek(len); + byteCount_ += len; + return; + } + size_t n = std::min(available_, len); + available_ -= n; + next_ += n; + len -= n; + byteCount_ += n; + } + } + + size_t byteCount() const { return byteCount_; } + + bool fill() { + size_t n = 0; + if (in_->read(buffer_, bufferSize_, n)) { + next_ = buffer_; + available_ = n; + return true; + } + return false; + } + + void seek(int64_t position) { + // BufferCopyIn::seek is relative to byteCount_, whereas position is + // absolute. + in_->seek(position - byteCount_ - available_); + byteCount_ = position; + available_ = 0; + } + +public: + BufferCopyInInputStream(unique_ptr<BufferCopyIn> in, size_t bufferSize) : + bufferSize_(bufferSize), + buffer_(new uint8_t[bufferSize]), + in_(std::move(in)), + byteCount_(0), + next_(buffer_), + available_(0) { } + + ~BufferCopyInInputStream() { + delete[] buffer_; + } +}; + +namespace { +struct BufferCopyOut { + virtual ~BufferCopyOut() { } + virtual void write(const uint8_t* b, size_t len) = 0; +}; + +struct FileBufferCopyOut : public BufferCopyOut { +#ifdef _WIN32 + HANDLE h_; + FileBufferCopyOut(const char* filename) : + h_(::CreateFileA(filename, GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL)) { + if (h_ == INVALID_HANDLE_VALUE) { + throw Exception(boost::format("Cannot open file: %1%") % ::GetLastError()); + } + } + + ~FileBufferCopyOut() { + ::CloseHandle(h_); + } + + void write(const uint8_t* b, size_t len) { + while (len > 0) { + DWORD dw = 0; + if (! ::WriteFile(h_, b, len, &dw, NULL)) { + throw Exception(boost::format("Cannot read file: %1%") % ::GetLastError()); + } + b += dw; + len -= dw; + } + } +#else + const int fd_; + + FileBufferCopyOut(const char* filename) : + fd_(::open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644)) { + + if (fd_ < 0) { + throw Exception(boost::format("Cannot open file: %1%") % + ::strerror(errno)); + } + } + + ~FileBufferCopyOut() { + ::close(fd_); + } + + void write(const uint8_t* b, size_t len) { + if (::write(fd_, b, len) < 0) { + throw Exception(boost::format("Cannot write file: %1%") % + ::strerror(errno)); + } + } +#endif + +}; + +struct OStreamBufferCopyOut : public BufferCopyOut { + ostream& os_; + + OStreamBufferCopyOut(ostream& os) : os_(os) { + } + + void write(const uint8_t* b, size_t len) { + os_.write(reinterpret_cast<const char*>(b), len); + } + +}; + +} + +class BufferCopyOutputStream : public OutputStream { + size_t bufferSize_; + uint8_t* const buffer_; + unique_ptr<BufferCopyOut> out_; + uint8_t* next_; + size_t available_; + size_t byteCount_; + + // Invaiant: byteCount_ == byteswritten + bufferSize_ - available_; + bool next(uint8_t** data, size_t* len) { + if (available_ == 0) { + flush(); + } + *data = next_; + *len = available_; + next_ += available_; + byteCount_ += available_; + available_ = 0; + return true; + } + + void backup(size_t len) { + available_ += len; + next_ -= len; + byteCount_ -= len; + } + + uint64_t byteCount() const { + return byteCount_; + } + + void flush() { + out_->write(buffer_, bufferSize_ - available_); + next_ = buffer_; + available_ = bufferSize_; + } + +public: + BufferCopyOutputStream(unique_ptr<BufferCopyOut> out, size_t bufferSize) : + bufferSize_(bufferSize), + buffer_(new uint8_t[bufferSize]), + out_(std::move(out)), + next_(buffer_), + available_(bufferSize_), byteCount_(0) { } + + ~BufferCopyOutputStream() { + delete[] buffer_; + } +}; + +unique_ptr<InputStream> fileInputStream(const char* filename, + size_t bufferSize) +{ + unique_ptr<BufferCopyIn> in(new FileBufferCopyIn(filename)); + return unique_ptr<InputStream>( new BufferCopyInInputStream(std::move(in), bufferSize)); +} + +unique_ptr<SeekableInputStream> fileSeekableInputStream(const char* filename, + size_t bufferSize) +{ + unique_ptr<BufferCopyIn> in(new FileBufferCopyIn(filename)); + return unique_ptr<SeekableInputStream>( new BufferCopyInInputStream(std::move(in), + bufferSize)); +} + +unique_ptr<InputStream> istreamInputStream(istream& is, size_t bufferSize) +{ + unique_ptr<BufferCopyIn> in(new IStreamBufferCopyIn(is)); + return unique_ptr<InputStream>( new BufferCopyInInputStream(std::move(in), bufferSize)); +} + +unique_ptr<InputStream> nonSeekableIstreamInputStream( + istream& is, size_t bufferSize) +{ + unique_ptr<BufferCopyIn> in(new NonSeekableIStreamBufferCopyIn(is)); + return unique_ptr<InputStream>( new BufferCopyInInputStream(std::move(in), bufferSize)); +} + +unique_ptr<OutputStream> fileOutputStream(const char* filename, + size_t bufferSize) +{ + unique_ptr<BufferCopyOut> out(new FileBufferCopyOut(filename)); + return unique_ptr<OutputStream>(new BufferCopyOutputStream(std::move(out), bufferSize)); +} + +unique_ptr<OutputStream> ostreamOutputStream(ostream& os, + size_t bufferSize) +{ + unique_ptr<BufferCopyOut> out(new OStreamBufferCopyOut(os)); + return unique_ptr<OutputStream>(new BufferCopyOutputStream(std::move(out), bufferSize)); +} + + +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/Generic.cc b/contrib/libs/apache/avro/impl/Generic.cc index 5420e528c8..8efb7e9ac4 100644 --- a/contrib/libs/apache/avro/impl/Generic.cc +++ b/contrib/libs/apache/avro/impl/Generic.cc @@ -1,260 +1,260 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "Generic.hh" -#include <sstream> - -namespace avro { - -using std::string; -using std::vector; -using std::ostringstream; - -typedef vector<uint8_t> bytes; - -void GenericContainer::assertType(const NodePtr& schema, Type type) { - if (schema->type() != type) { - throw Exception(boost::format("Schema type %1 expected %2") % - toString(schema->type()) % toString(type)); - } -} - -GenericReader::GenericReader(const ValidSchema& s, const DecoderPtr& decoder) : - schema_(s), isResolving_(dynamic_cast<ResolvingDecoder*>(&(*decoder)) != 0), - decoder_(decoder) -{ -} - -GenericReader::GenericReader(const ValidSchema& writerSchema, - const ValidSchema& readerSchema, const DecoderPtr& decoder) : - schema_(readerSchema), - isResolving_(true), - decoder_(resolvingDecoder(writerSchema, readerSchema, decoder)) -{ -} - -void GenericReader::read(GenericDatum& datum) const -{ - datum = GenericDatum(schema_.root()); - read(datum, *decoder_, isResolving_); -} - -void GenericReader::read(GenericDatum& datum, Decoder& d, bool isResolving) -{ - if (datum.isUnion()) { - datum.selectBranch(d.decodeUnionIndex()); - } - switch (datum.type()) { - case AVRO_NULL: - d.decodeNull(); - break; - case AVRO_BOOL: - datum.value<bool>() = d.decodeBool(); - break; - case AVRO_INT: - datum.value<int32_t>() = d.decodeInt(); - break; - case AVRO_LONG: - datum.value<int64_t>() = d.decodeLong(); - break; - case AVRO_FLOAT: - datum.value<float>() = d.decodeFloat(); - break; - case AVRO_DOUBLE: - datum.value<double>() = d.decodeDouble(); - break; - case AVRO_STRING: - d.decodeString(datum.value<string>()); - break; - case AVRO_BYTES: - d.decodeBytes(datum.value<bytes>()); - break; - case AVRO_FIXED: - { - GenericFixed& f = datum.value<GenericFixed>(); - d.decodeFixed(f.schema()->fixedSize(), f.value()); - } - break; - case AVRO_RECORD: - { - GenericRecord& r = datum.value<GenericRecord>(); - size_t c = r.schema()->leaves(); - if (isResolving) { - std::vector<size_t> fo = - static_cast<ResolvingDecoder&>(d).fieldOrder(); - for (size_t i = 0; i < c; ++i) { - read(r.fieldAt(fo[i]), d, isResolving); - } - } else { - for (size_t i = 0; i < c; ++i) { - read(r.fieldAt(i), d, isResolving); - } - } - } - break; - case AVRO_ENUM: - datum.value<GenericEnum>().set(d.decodeEnum()); - break; - case AVRO_ARRAY: - { - GenericArray& v = datum.value<GenericArray>(); - vector<GenericDatum>& r = v.value(); - const NodePtr& nn = v.schema()->leafAt(0); - r.resize(0); - size_t start = 0; - for (size_t m = d.arrayStart(); m != 0; m = d.arrayNext()) { - r.resize(r.size() + m); - for (; start < r.size(); ++start) { - r[start] = GenericDatum(nn); - read(r[start], d, isResolving); - } - } - } - break; - case AVRO_MAP: - { - GenericMap& v = datum.value<GenericMap>(); - GenericMap::Value& r = v.value(); - const NodePtr& nn = v.schema()->leafAt(1); - r.resize(0); - size_t start = 0; - for (size_t m = d.mapStart(); m != 0; m = d.mapNext()) { - r.resize(r.size() + m); - for (; start < r.size(); ++start) { - d.decodeString(r[start].first); - r[start].second = GenericDatum(nn); - read(r[start].second, d, isResolving); - } - } - } - break; - default: - throw Exception(boost::format("Unknown schema type %1%") % - toString(datum.type())); - } -} - -void GenericReader::read(Decoder& d, GenericDatum& g, const ValidSchema& s) -{ - g = GenericDatum(s); - read(d, g); -} - -void GenericReader::read(Decoder& d, GenericDatum& g) -{ - read(g, d, dynamic_cast<ResolvingDecoder*>(&d) != 0); -} - -GenericWriter::GenericWriter(const ValidSchema& s, const EncoderPtr& encoder) : - schema_(s), encoder_(encoder) -{ -} - -void GenericWriter::write(const GenericDatum& datum) const -{ - write(datum, *encoder_); -} - -void GenericWriter::write(const GenericDatum& datum, Encoder& e) -{ - if (datum.isUnion()) { - e.encodeUnionIndex(datum.unionBranch()); - } - switch (datum.type()) { - case AVRO_NULL: - e.encodeNull(); - break; - case AVRO_BOOL: - e.encodeBool(datum.value<bool>()); - break; - case AVRO_INT: - e.encodeInt(datum.value<int32_t>()); - break; - case AVRO_LONG: - e.encodeLong(datum.value<int64_t>()); - break; - case AVRO_FLOAT: - e.encodeFloat(datum.value<float>()); - break; - case AVRO_DOUBLE: - e.encodeDouble(datum.value<double>()); - break; - case AVRO_STRING: - e.encodeString(datum.value<string>()); - break; - case AVRO_BYTES: - e.encodeBytes(datum.value<bytes>()); - break; - case AVRO_FIXED: - e.encodeFixed(datum.value<GenericFixed>().value()); - break; - case AVRO_RECORD: - { - const GenericRecord& r = datum.value<GenericRecord>(); - size_t c = r.schema()->leaves(); - for (size_t i = 0; i < c; ++i) { - write(r.fieldAt(i), e); - } - } - break; - case AVRO_ENUM: - e.encodeEnum(datum.value<GenericEnum>().value()); - break; - case AVRO_ARRAY: - { - const GenericArray::Value& r = datum.value<GenericArray>().value(); - e.arrayStart(); - if (! r.empty()) { - e.setItemCount(r.size()); - for (GenericArray::Value::const_iterator it = r.begin(); - it != r.end(); ++it) { - e.startItem(); - write(*it, e); - } - } - e.arrayEnd(); - } - break; - case AVRO_MAP: - { - const GenericMap::Value& r = datum.value<GenericMap>().value(); - e.mapStart(); - if (! r.empty()) { - e.setItemCount(r.size()); - for (GenericMap::Value::const_iterator it = r.begin(); - it != r.end(); ++it) { - e.startItem(); - e.encodeString(it->first); - write(it->second, e); - } - } - e.mapEnd(); - } - break; - default: - throw Exception(boost::format("Unknown schema type %1%") % - toString(datum.type())); - } -} - -void GenericWriter::write(Encoder& e, const GenericDatum& g) -{ - write(g, e); -} - -} // namespace avro +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Generic.hh" +#include <sstream> + +namespace avro { + +using std::string; +using std::vector; +using std::ostringstream; + +typedef vector<uint8_t> bytes; + +void GenericContainer::assertType(const NodePtr& schema, Type type) { + if (schema->type() != type) { + throw Exception(boost::format("Schema type %1 expected %2") % + toString(schema->type()) % toString(type)); + } +} + +GenericReader::GenericReader(const ValidSchema& s, const DecoderPtr& decoder) : + schema_(s), isResolving_(dynamic_cast<ResolvingDecoder*>(&(*decoder)) != 0), + decoder_(decoder) +{ +} + +GenericReader::GenericReader(const ValidSchema& writerSchema, + const ValidSchema& readerSchema, const DecoderPtr& decoder) : + schema_(readerSchema), + isResolving_(true), + decoder_(resolvingDecoder(writerSchema, readerSchema, decoder)) +{ +} + +void GenericReader::read(GenericDatum& datum) const +{ + datum = GenericDatum(schema_.root()); + read(datum, *decoder_, isResolving_); +} + +void GenericReader::read(GenericDatum& datum, Decoder& d, bool isResolving) +{ + if (datum.isUnion()) { + datum.selectBranch(d.decodeUnionIndex()); + } + switch (datum.type()) { + case AVRO_NULL: + d.decodeNull(); + break; + case AVRO_BOOL: + datum.value<bool>() = d.decodeBool(); + break; + case AVRO_INT: + datum.value<int32_t>() = d.decodeInt(); + break; + case AVRO_LONG: + datum.value<int64_t>() = d.decodeLong(); + break; + case AVRO_FLOAT: + datum.value<float>() = d.decodeFloat(); + break; + case AVRO_DOUBLE: + datum.value<double>() = d.decodeDouble(); + break; + case AVRO_STRING: + d.decodeString(datum.value<string>()); + break; + case AVRO_BYTES: + d.decodeBytes(datum.value<bytes>()); + break; + case AVRO_FIXED: + { + GenericFixed& f = datum.value<GenericFixed>(); + d.decodeFixed(f.schema()->fixedSize(), f.value()); + } + break; + case AVRO_RECORD: + { + GenericRecord& r = datum.value<GenericRecord>(); + size_t c = r.schema()->leaves(); + if (isResolving) { + std::vector<size_t> fo = + static_cast<ResolvingDecoder&>(d).fieldOrder(); + for (size_t i = 0; i < c; ++i) { + read(r.fieldAt(fo[i]), d, isResolving); + } + } else { + for (size_t i = 0; i < c; ++i) { + read(r.fieldAt(i), d, isResolving); + } + } + } + break; + case AVRO_ENUM: + datum.value<GenericEnum>().set(d.decodeEnum()); + break; + case AVRO_ARRAY: + { + GenericArray& v = datum.value<GenericArray>(); + vector<GenericDatum>& r = v.value(); + const NodePtr& nn = v.schema()->leafAt(0); + r.resize(0); + size_t start = 0; + for (size_t m = d.arrayStart(); m != 0; m = d.arrayNext()) { + r.resize(r.size() + m); + for (; start < r.size(); ++start) { + r[start] = GenericDatum(nn); + read(r[start], d, isResolving); + } + } + } + break; + case AVRO_MAP: + { + GenericMap& v = datum.value<GenericMap>(); + GenericMap::Value& r = v.value(); + const NodePtr& nn = v.schema()->leafAt(1); + r.resize(0); + size_t start = 0; + for (size_t m = d.mapStart(); m != 0; m = d.mapNext()) { + r.resize(r.size() + m); + for (; start < r.size(); ++start) { + d.decodeString(r[start].first); + r[start].second = GenericDatum(nn); + read(r[start].second, d, isResolving); + } + } + } + break; + default: + throw Exception(boost::format("Unknown schema type %1%") % + toString(datum.type())); + } +} + +void GenericReader::read(Decoder& d, GenericDatum& g, const ValidSchema& s) +{ + g = GenericDatum(s); + read(d, g); +} + +void GenericReader::read(Decoder& d, GenericDatum& g) +{ + read(g, d, dynamic_cast<ResolvingDecoder*>(&d) != 0); +} + +GenericWriter::GenericWriter(const ValidSchema& s, const EncoderPtr& encoder) : + schema_(s), encoder_(encoder) +{ +} + +void GenericWriter::write(const GenericDatum& datum) const +{ + write(datum, *encoder_); +} + +void GenericWriter::write(const GenericDatum& datum, Encoder& e) +{ + if (datum.isUnion()) { + e.encodeUnionIndex(datum.unionBranch()); + } + switch (datum.type()) { + case AVRO_NULL: + e.encodeNull(); + break; + case AVRO_BOOL: + e.encodeBool(datum.value<bool>()); + break; + case AVRO_INT: + e.encodeInt(datum.value<int32_t>()); + break; + case AVRO_LONG: + e.encodeLong(datum.value<int64_t>()); + break; + case AVRO_FLOAT: + e.encodeFloat(datum.value<float>()); + break; + case AVRO_DOUBLE: + e.encodeDouble(datum.value<double>()); + break; + case AVRO_STRING: + e.encodeString(datum.value<string>()); + break; + case AVRO_BYTES: + e.encodeBytes(datum.value<bytes>()); + break; + case AVRO_FIXED: + e.encodeFixed(datum.value<GenericFixed>().value()); + break; + case AVRO_RECORD: + { + const GenericRecord& r = datum.value<GenericRecord>(); + size_t c = r.schema()->leaves(); + for (size_t i = 0; i < c; ++i) { + write(r.fieldAt(i), e); + } + } + break; + case AVRO_ENUM: + e.encodeEnum(datum.value<GenericEnum>().value()); + break; + case AVRO_ARRAY: + { + const GenericArray::Value& r = datum.value<GenericArray>().value(); + e.arrayStart(); + if (! r.empty()) { + e.setItemCount(r.size()); + for (GenericArray::Value::const_iterator it = r.begin(); + it != r.end(); ++it) { + e.startItem(); + write(*it, e); + } + } + e.arrayEnd(); + } + break; + case AVRO_MAP: + { + const GenericMap::Value& r = datum.value<GenericMap>().value(); + e.mapStart(); + if (! r.empty()) { + e.setItemCount(r.size()); + for (GenericMap::Value::const_iterator it = r.begin(); + it != r.end(); ++it) { + e.startItem(); + e.encodeString(it->first); + write(it->second, e); + } + } + e.mapEnd(); + } + break; + default: + throw Exception(boost::format("Unknown schema type %1%") % + toString(datum.type())); + } +} + +void GenericWriter::write(Encoder& e, const GenericDatum& g) +{ + write(g, e); +} + +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/GenericDatum.cc b/contrib/libs/apache/avro/impl/GenericDatum.cc index 855e1c4a55..cdf9006eef 100644 --- a/contrib/libs/apache/avro/impl/GenericDatum.cc +++ b/contrib/libs/apache/avro/impl/GenericDatum.cc @@ -1,105 +1,105 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "GenericDatum.hh" -#include "NodeImpl.hh" - -using std::string; -using std::vector; - -namespace avro { - -GenericDatum::GenericDatum(const ValidSchema& schema) : - type_(schema.root()->type()), - logicalType_(schema.root()->logicalType()) -{ - init(schema.root()); -} - -GenericDatum::GenericDatum(const NodePtr& schema) : - type_(schema->type()), - logicalType_(schema->logicalType()) -{ - init(schema); -} - -void GenericDatum::init(const NodePtr& schema) -{ - NodePtr sc = schema; - if (type_ == AVRO_SYMBOLIC) { - sc = resolveSymbol(schema); - type_ = sc->type(); - logicalType_ = sc->logicalType(); - } - switch (type_) { - case AVRO_NULL: - break; - case AVRO_BOOL: - value_ = bool(); - break; - case AVRO_INT: - value_ = int32_t(); - break; - case AVRO_LONG: - value_ = int64_t(); - break; - case AVRO_FLOAT: - value_ = float(); - break; - case AVRO_DOUBLE: - value_ = double(); - break; - case AVRO_STRING: - value_ = string(); - break; - case AVRO_BYTES: - value_ = vector<uint8_t>(); - break; - case AVRO_FIXED: - value_ = GenericFixed(sc); - break; - case AVRO_RECORD: - value_ = GenericRecord(sc); - break; - case AVRO_ENUM: - value_ = GenericEnum(sc); - break; - case AVRO_ARRAY: - value_ = GenericArray(sc); - break; - case AVRO_MAP: - value_ = GenericMap(sc); - break; - case AVRO_UNION: - value_ = GenericUnion(sc); - break; - default: - throw Exception(boost::format("Unknown schema type %1%") % - toString(type_)); - } -} - -GenericRecord::GenericRecord(const NodePtr& schema) : - GenericContainer(AVRO_RECORD, schema) { - fields_.resize(schema->leaves()); - for (size_t i = 0; i < schema->leaves(); ++i) { - fields_[i] = GenericDatum(schema->leafAt(i)); - } -} - -} // namespace avro +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "GenericDatum.hh" +#include "NodeImpl.hh" + +using std::string; +using std::vector; + +namespace avro { + +GenericDatum::GenericDatum(const ValidSchema& schema) : + type_(schema.root()->type()), + logicalType_(schema.root()->logicalType()) +{ + init(schema.root()); +} + +GenericDatum::GenericDatum(const NodePtr& schema) : + type_(schema->type()), + logicalType_(schema->logicalType()) +{ + init(schema); +} + +void GenericDatum::init(const NodePtr& schema) +{ + NodePtr sc = schema; + if (type_ == AVRO_SYMBOLIC) { + sc = resolveSymbol(schema); + type_ = sc->type(); + logicalType_ = sc->logicalType(); + } + switch (type_) { + case AVRO_NULL: + break; + case AVRO_BOOL: + value_ = bool(); + break; + case AVRO_INT: + value_ = int32_t(); + break; + case AVRO_LONG: + value_ = int64_t(); + break; + case AVRO_FLOAT: + value_ = float(); + break; + case AVRO_DOUBLE: + value_ = double(); + break; + case AVRO_STRING: + value_ = string(); + break; + case AVRO_BYTES: + value_ = vector<uint8_t>(); + break; + case AVRO_FIXED: + value_ = GenericFixed(sc); + break; + case AVRO_RECORD: + value_ = GenericRecord(sc); + break; + case AVRO_ENUM: + value_ = GenericEnum(sc); + break; + case AVRO_ARRAY: + value_ = GenericArray(sc); + break; + case AVRO_MAP: + value_ = GenericMap(sc); + break; + case AVRO_UNION: + value_ = GenericUnion(sc); + break; + default: + throw Exception(boost::format("Unknown schema type %1%") % + toString(type_)); + } +} + +GenericRecord::GenericRecord(const NodePtr& schema) : + GenericContainer(AVRO_RECORD, schema) { + fields_.resize(schema->leaves()); + for (size_t i = 0; i < schema->leaves(); ++i) { + fields_[i] = GenericDatum(schema->leafAt(i)); + } +} + +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/LogicalType.cc b/contrib/libs/apache/avro/impl/LogicalType.cc index 9c75590392..a0d9cc3b6f 100644 --- a/contrib/libs/apache/avro/impl/LogicalType.cc +++ b/contrib/libs/apache/avro/impl/LogicalType.cc @@ -1,84 +1,84 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "Exception.hh" -#include "LogicalType.hh" - -namespace avro { - -LogicalType::LogicalType(Type type) - : type_(type), precision_(0), scale_(0) {} - -LogicalType::Type LogicalType::type() const { - return type_; -} - -void LogicalType::setPrecision(int precision) { - if (type_ != DECIMAL) { - throw Exception("Only logical type DECIMAL can have precision"); - } - if (precision <= 0) { - throw Exception(boost::format("Precision cannot be: %1%") % precision); - } - precision_ = precision; -} - -void LogicalType::setScale(int scale) { - if (type_ != DECIMAL) { - throw Exception("Only logical type DECIMAL can have scale"); - } - if (scale < 0) { - throw Exception(boost::format("Scale cannot be: %1%") % scale); - } - scale_ = scale; -} - -void LogicalType::printJson(std::ostream& os) const { - switch (type_) { - case LogicalType::NONE: - break; - case LogicalType::DECIMAL: - os << "\"logicalType\": \"decimal\""; - os << ", \"precision\": " << precision_; - os << ", \"scale\": " << scale_; - break; - case DATE: - os << "\"logicalType\": \"date\""; - break; - case TIME_MILLIS: - os << "\"logicalType\": \"time-millis\""; - break; - case TIME_MICROS: - os << "\"logicalType\": \"time-micros\""; - break; - case TIMESTAMP_MILLIS: - os << "\"logicalType\": \"timestamp-millis\""; - break; - case TIMESTAMP_MICROS: - os << "\"logicalType\": \"timestamp-micros\""; - break; - case DURATION: - os << "\"logicalType\": \"duration\""; - break; - case UUID: - os << "\"logicalType\": \"uuid\""; - break; - } -} - -} // namespace avro +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Exception.hh" +#include "LogicalType.hh" + +namespace avro { + +LogicalType::LogicalType(Type type) + : type_(type), precision_(0), scale_(0) {} + +LogicalType::Type LogicalType::type() const { + return type_; +} + +void LogicalType::setPrecision(int precision) { + if (type_ != DECIMAL) { + throw Exception("Only logical type DECIMAL can have precision"); + } + if (precision <= 0) { + throw Exception(boost::format("Precision cannot be: %1%") % precision); + } + precision_ = precision; +} + +void LogicalType::setScale(int scale) { + if (type_ != DECIMAL) { + throw Exception("Only logical type DECIMAL can have scale"); + } + if (scale < 0) { + throw Exception(boost::format("Scale cannot be: %1%") % scale); + } + scale_ = scale; +} + +void LogicalType::printJson(std::ostream& os) const { + switch (type_) { + case LogicalType::NONE: + break; + case LogicalType::DECIMAL: + os << "\"logicalType\": \"decimal\""; + os << ", \"precision\": " << precision_; + os << ", \"scale\": " << scale_; + break; + case DATE: + os << "\"logicalType\": \"date\""; + break; + case TIME_MILLIS: + os << "\"logicalType\": \"time-millis\""; + break; + case TIME_MICROS: + os << "\"logicalType\": \"time-micros\""; + break; + case TIMESTAMP_MILLIS: + os << "\"logicalType\": \"timestamp-millis\""; + break; + case TIMESTAMP_MICROS: + os << "\"logicalType\": \"timestamp-micros\""; + break; + case DURATION: + os << "\"logicalType\": \"duration\""; + break; + case UUID: + os << "\"logicalType\": \"uuid\""; + break; + } +} + +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/Node.cc b/contrib/libs/apache/avro/impl/Node.cc index 87b3652b4d..bb510cc147 100644 --- a/contrib/libs/apache/avro/impl/Node.cc +++ b/contrib/libs/apache/avro/impl/Node.cc @@ -1,161 +1,161 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <cmath> - -#include "Node.hh" - -namespace avro { - -using std::string; - -Node::~Node() -{ } - -Name::Name(const std::string& name) -{ - fullname(name); -} - -const string Name::fullname() const -{ - return (ns_.empty()) ? simpleName_ : ns_ + "." + simpleName_; -} - -void Name::fullname(const string& name) -{ - string::size_type n = name.find_last_of('.'); - if (n == string::npos) { - simpleName_ = name; - ns_.clear(); - } else { - ns_ = name.substr(0, n); - simpleName_ = name.substr(n + 1); - } - check(); -} - -bool Name::operator < (const Name& n) const -{ - return (ns_ < n.ns_) ? true : - (n.ns_ < ns_) ? false : - (simpleName_ < n.simpleName_); -} - -static bool invalidChar1(char c) -{ - return !isalnum(c) && c != '_' && c != '.' && c != '$'; -} - -static bool invalidChar2(char c) -{ - return !isalnum(c) && c != '_'; -} - -void Name::check() const -{ - if (! ns_.empty() && (ns_[0] == '.' || ns_[ns_.size() - 1] == '.' || std::find_if(ns_.begin(), ns_.end(), invalidChar1) != ns_.end())) { - throw Exception("Invalid namespace: " + ns_); - } - if (simpleName_.empty() || std::find_if(simpleName_.begin(), simpleName_.end(), invalidChar2) != simpleName_.end()) { - throw Exception("Invalid name: " + simpleName_); - } -} - -bool Name::operator == (const Name& n) const -{ - return ns_ == n.ns_ && simpleName_ == n.simpleName_; -} - -void Node::setLogicalType(LogicalType logicalType) { - checkLock(); - - // Check that the logical type is applicable to the node type. - switch (logicalType.type()) { - case LogicalType::NONE: - break; - case LogicalType::DECIMAL: { - if (type_ != AVRO_BYTES && type_ != AVRO_FIXED) { - throw Exception("DECIMAL logical type can annotate " - "only BYTES or FIXED type"); - } - if (type_ == AVRO_FIXED) { - // Max precision that can be supported by the current size of - // the FIXED type. - long maxPrecision = floor(log10(2.0) * (8.0 * fixedSize() - 1)); - if (logicalType.precision() > maxPrecision) { - throw Exception( - boost::format( - "DECIMAL precision %1% is too large for the " - "FIXED type of size %2%, precision cannot be " - "larget than %3%") % logicalType.precision() % - fixedSize() % maxPrecision); - } - } - if (logicalType.scale() > logicalType.precision()) { - throw Exception("DECIMAL scale cannot exceed precision"); - } - break; - } - case LogicalType::DATE: - if (type_ != AVRO_INT) { - throw Exception("DATE logical type can only annotate INT type"); - } - break; - case LogicalType::TIME_MILLIS: - if (type_ != AVRO_INT) { - throw Exception("TIME-MILLIS logical type can only annotate " - "INT type"); - } - break; - case LogicalType::TIME_MICROS: - if (type_ != AVRO_LONG) { - throw Exception("TIME-MICROS logical type can only annotate " - "LONG type"); - } - break; - case LogicalType::TIMESTAMP_MILLIS: - if (type_ != AVRO_LONG) { - throw Exception("TIMESTAMP-MILLIS logical type can only annotate " - "LONG type"); - } - break; - case LogicalType::TIMESTAMP_MICROS: - if (type_ != AVRO_LONG) { - throw Exception("TIMESTAMP-MICROS logical type can only annotate " - "LONG type"); - } - break; - case LogicalType::DURATION: - if (type_ != AVRO_FIXED || fixedSize() != 12) { - throw Exception("DURATION logical type can only annotate " - "FIXED type of size 12"); - } - break; - case LogicalType::UUID: - if (type_ != AVRO_STRING) { - throw Exception("UUID logical type can only annotate " - "STRING type"); - } - break; - } - - logicalType_ = logicalType; -} - -} // namespace avro +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <cmath> + +#include "Node.hh" + +namespace avro { + +using std::string; + +Node::~Node() +{ } + +Name::Name(const std::string& name) +{ + fullname(name); +} + +const string Name::fullname() const +{ + return (ns_.empty()) ? simpleName_ : ns_ + "." + simpleName_; +} + +void Name::fullname(const string& name) +{ + string::size_type n = name.find_last_of('.'); + if (n == string::npos) { + simpleName_ = name; + ns_.clear(); + } else { + ns_ = name.substr(0, n); + simpleName_ = name.substr(n + 1); + } + check(); +} + +bool Name::operator < (const Name& n) const +{ + return (ns_ < n.ns_) ? true : + (n.ns_ < ns_) ? false : + (simpleName_ < n.simpleName_); +} + +static bool invalidChar1(char c) +{ + return !isalnum(c) && c != '_' && c != '.' && c != '$'; +} + +static bool invalidChar2(char c) +{ + return !isalnum(c) && c != '_'; +} + +void Name::check() const +{ + if (! ns_.empty() && (ns_[0] == '.' || ns_[ns_.size() - 1] == '.' || std::find_if(ns_.begin(), ns_.end(), invalidChar1) != ns_.end())) { + throw Exception("Invalid namespace: " + ns_); + } + if (simpleName_.empty() || std::find_if(simpleName_.begin(), simpleName_.end(), invalidChar2) != simpleName_.end()) { + throw Exception("Invalid name: " + simpleName_); + } +} + +bool Name::operator == (const Name& n) const +{ + return ns_ == n.ns_ && simpleName_ == n.simpleName_; +} + +void Node::setLogicalType(LogicalType logicalType) { + checkLock(); + + // Check that the logical type is applicable to the node type. + switch (logicalType.type()) { + case LogicalType::NONE: + break; + case LogicalType::DECIMAL: { + if (type_ != AVRO_BYTES && type_ != AVRO_FIXED) { + throw Exception("DECIMAL logical type can annotate " + "only BYTES or FIXED type"); + } + if (type_ == AVRO_FIXED) { + // Max precision that can be supported by the current size of + // the FIXED type. + long maxPrecision = floor(log10(2.0) * (8.0 * fixedSize() - 1)); + if (logicalType.precision() > maxPrecision) { + throw Exception( + boost::format( + "DECIMAL precision %1% is too large for the " + "FIXED type of size %2%, precision cannot be " + "larget than %3%") % logicalType.precision() % + fixedSize() % maxPrecision); + } + } + if (logicalType.scale() > logicalType.precision()) { + throw Exception("DECIMAL scale cannot exceed precision"); + } + break; + } + case LogicalType::DATE: + if (type_ != AVRO_INT) { + throw Exception("DATE logical type can only annotate INT type"); + } + break; + case LogicalType::TIME_MILLIS: + if (type_ != AVRO_INT) { + throw Exception("TIME-MILLIS logical type can only annotate " + "INT type"); + } + break; + case LogicalType::TIME_MICROS: + if (type_ != AVRO_LONG) { + throw Exception("TIME-MICROS logical type can only annotate " + "LONG type"); + } + break; + case LogicalType::TIMESTAMP_MILLIS: + if (type_ != AVRO_LONG) { + throw Exception("TIMESTAMP-MILLIS logical type can only annotate " + "LONG type"); + } + break; + case LogicalType::TIMESTAMP_MICROS: + if (type_ != AVRO_LONG) { + throw Exception("TIMESTAMP-MICROS logical type can only annotate " + "LONG type"); + } + break; + case LogicalType::DURATION: + if (type_ != AVRO_FIXED || fixedSize() != 12) { + throw Exception("DURATION logical type can only annotate " + "FIXED type of size 12"); + } + break; + case LogicalType::UUID: + if (type_ != AVRO_STRING) { + throw Exception("UUID logical type can only annotate " + "STRING type"); + } + break; + } + + logicalType_ = logicalType; +} + +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/NodeImpl.cc b/contrib/libs/apache/avro/impl/NodeImpl.cc index 124152c4ee..4a0acb92c0 100644 --- a/contrib/libs/apache/avro/impl/NodeImpl.cc +++ b/contrib/libs/apache/avro/impl/NodeImpl.cc @@ -1,547 +1,547 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include <sstream> -#include <iomanip> -#include <boost/algorithm/string/replace.hpp> -#include "NodeImpl.hh" - - -using std::string; -namespace avro { - -namespace { - -// Escape string for serialization. -string escape(const string &unescaped) { - string s; - s.reserve(unescaped.length()); - for (std::string::const_iterator it = unescaped.begin(); it != unescaped.end(); ++it) { - char c = *it; - switch (c) { - case '\\': - case '"': - case '/': - s += '\\'; - s += c; - break; - case '\b': - s += '\\'; - s += 'b'; - break; - case '\f': - s += '\f'; - break; - case '\n': - s += '\\'; - s += 'n'; - break; - case '\r': - s += '\\'; - s += 'r'; - break; - case '\t': - s += '\\'; - s += 't'; - break; - default: - if (!std::iscntrl(c, std::locale::classic())) { - s += c; - continue; - } - s += intToHex(static_cast<unsigned int>(c)); - break; - } - } - return s; -} - -// Wrap an indentation in a struct for ostream operator<< -struct indent { - indent(int depth) : - d(depth) - { } - int d; -}; - -/// ostream operator for indent -std::ostream& operator <<(std::ostream &os, indent x) -{ - static const string spaces(" "); - while (x.d--) { - os << spaces; - } - return os; -} - -} // anonymous namespace - -const int kByteStringSize = 6; - -SchemaResolution -NodePrimitive::resolve(const Node &reader) const -{ - if (type() == reader.type()) { - return RESOLVE_MATCH; - } - - switch ( type() ) { - - case AVRO_INT: - - if ( reader.type() == AVRO_LONG ) { - return RESOLVE_PROMOTABLE_TO_LONG; - } - - // fall-through intentional - - case AVRO_LONG: - - if (reader.type() == AVRO_FLOAT) { - return RESOLVE_PROMOTABLE_TO_FLOAT; - } - - // fall-through intentional - - case AVRO_FLOAT: - - if (reader.type() == AVRO_DOUBLE) { - return RESOLVE_PROMOTABLE_TO_DOUBLE; - } - - default: - break; - } - - return furtherResolution(reader); -} - -SchemaResolution -NodeRecord::resolve(const Node &reader) const -{ - if (reader.type() == AVRO_RECORD) { - if (name() == reader.name()) { - return RESOLVE_MATCH; - } - } - return furtherResolution(reader); -} - -SchemaResolution -NodeEnum::resolve(const Node &reader) const -{ - if (reader.type() == AVRO_ENUM) { - return (name() == reader.name()) ? RESOLVE_MATCH : RESOLVE_NO_MATCH; - } - return furtherResolution(reader); -} - -SchemaResolution -NodeArray::resolve(const Node &reader) const -{ - if (reader.type() == AVRO_ARRAY) { - const NodePtr &arrayType = leafAt(0); - return arrayType->resolve(*reader.leafAt(0)); - } - return furtherResolution(reader); -} - -SchemaResolution -NodeMap::resolve(const Node &reader) const -{ - if (reader.type() == AVRO_MAP) { - const NodePtr &mapType = leafAt(1); - return mapType->resolve(*reader.leafAt(1)); - } - return furtherResolution(reader); -} - -SchemaResolution -NodeUnion::resolve(const Node &reader) const -{ - - // If the writer is union, resolution only needs to occur when the selected - // type of the writer is known, so this function is not very helpful. - // - // In this case, this function returns if there is a possible match given - // any writer type, so just search type by type returning the best match - // found. - - SchemaResolution match = RESOLVE_NO_MATCH; - for (size_t i=0; i < leaves(); ++i) { - const NodePtr &node = leafAt(i); - SchemaResolution thisMatch = node->resolve(reader); - if (thisMatch == RESOLVE_MATCH) { - match = thisMatch; - break; - } - if (match == RESOLVE_NO_MATCH) { - match = thisMatch; - } - } - return match; -} - -SchemaResolution -NodeFixed::resolve(const Node &reader) const -{ - if (reader.type() == AVRO_FIXED) { - return ( - (reader.fixedSize() == fixedSize()) && - (reader.name() == name()) - ) ? - RESOLVE_MATCH : RESOLVE_NO_MATCH; - } - return furtherResolution(reader); -} - -SchemaResolution -NodeSymbolic::resolve(const Node &reader) const -{ - const NodePtr &node = leafAt(0); - return node->resolve(reader); -} - -void -NodePrimitive::printJson(std::ostream &os, int depth) const -{ - bool hasLogicalType = logicalType().type() != LogicalType::NONE; - - if (hasLogicalType) { - os << "{\n" << indent(depth) << "\"type\": "; - } - - os << '\"' << type() << '\"'; - - if (hasLogicalType) { - os << ",\n" << indent(depth); - logicalType().printJson(os); - os << "\n}"; - } - if (getDoc().size()) { - os << ",\n" << indent(depth) << "\"doc\": \"" - << escape(getDoc()) << "\""; - } -} - -void -NodeSymbolic::printJson(std::ostream &os, int depth) const -{ - os << '\"' << nameAttribute_.get() << '\"'; - if (getDoc().size()) { - os << ",\n" << indent(depth) << "\"doc\": \"" - << escape(getDoc()) << "\""; - } -} - -static void printName(std::ostream& os, const Name& n, int depth) -{ - if (!n.ns().empty()) { - os << indent(depth) << "\"namespace\": \"" << n.ns() << "\",\n"; - } - os << indent(depth) << "\"name\": \"" << n.simpleName() << "\",\n"; -} - -void -NodeRecord::printJson(std::ostream &os, int depth) const -{ - os << "{\n"; - os << indent(++depth) << "\"type\": \"record\",\n"; - printName(os, nameAttribute_.get(), depth); - if (getDoc().size()) { - os << indent(depth) << "\"doc\": \"" - << escape(getDoc()) << "\",\n"; - } - os << indent(depth) << "\"fields\": ["; - - size_t fields = leafAttributes_.size(); - ++depth; - // Serialize "default" field: - assert(defaultValues.empty() || (defaultValues.size() == fields)); - for (size_t i = 0; i < fields; ++i) { - if (i > 0) { - os << ','; - } - os << '\n' << indent(depth) << "{\n"; - os << indent(++depth) << "\"name\": \"" << leafNameAttributes_.get(i) << "\",\n"; - os << indent(depth) << "\"type\": "; - leafAttributes_.get(i)->printJson(os, depth); - - if (!defaultValues.empty()) { - if (!defaultValues[i].isUnion() && - defaultValues[i].type() == AVRO_NULL) { - // No "default" field. - } else { - os << ",\n" << indent(depth) << "\"default\": "; - leafAttributes_.get(i)->printDefaultToJson(defaultValues[i], os, - depth); - } - } - os << '\n'; - os << indent(--depth) << '}'; - } - os << '\n' << indent(--depth) << "]\n"; - os << indent(--depth) << '}'; -} - -void NodePrimitive::printDefaultToJson(const GenericDatum &g, std::ostream &os, - int depth) const { - assert(isPrimitive(g.type())); - - switch (g.type()) { - case AVRO_NULL: - os << "null"; - break; - case AVRO_BOOL: - os << (g.value<bool>() ? "true" : "false"); - break; - case AVRO_INT: - os << g.value<int32_t>(); - break; - case AVRO_LONG: - os << g.value<int64_t>(); - break; - case AVRO_FLOAT: - os << g.value<float>(); - break; - case AVRO_DOUBLE: - os << g.value<double>(); - break; - case AVRO_STRING: - os << "\"" << escape(g.value<string>()) << "\""; - break; - case AVRO_BYTES: { - // Convert to a string: - const std::vector<uint8_t> &vg = g.value<std::vector<uint8_t> >(); - string s; - s.resize(vg.size() * kByteStringSize); - for (unsigned int i = 0; i < vg.size(); i++) { - string hex_string = intToHex(static_cast<int>(vg[i])); - s.replace(i*kByteStringSize, kByteStringSize, hex_string); - } - os << "\"" << s << "\""; - } break; - default: - break; - } -} - -void NodeEnum::printDefaultToJson(const GenericDatum &g, std::ostream &os, - int depth) const { - assert(g.type() == AVRO_ENUM); - os << "\"" << g.value<GenericEnum>().symbol() << "\""; -} - -void NodeFixed::printDefaultToJson(const GenericDatum &g, std::ostream &os, - int depth) const { - assert(g.type() == AVRO_FIXED); - // ex: "\uOOff" - // Convert to a string - const std::vector<uint8_t> &vg = g.value<GenericFixed>().value(); - string s; - s.resize(vg.size() * kByteStringSize); - for (unsigned int i = 0; i < vg.size(); i++) { - string hex_string = intToHex(static_cast<int>(vg[i])); - s.replace(i*kByteStringSize, kByteStringSize, hex_string); - } - os << "\"" << s << "\""; -} - -void NodeUnion::printDefaultToJson(const GenericDatum &g, std::ostream &os, - int depth) const { - leafAt(0)->printDefaultToJson(g, os, depth); -} - -void NodeArray::printDefaultToJson(const GenericDatum &g, std::ostream &os, - int depth) const { - assert(g.type() == AVRO_ARRAY); - // ex: "default": [1] - if (g.value<GenericArray>().value().empty()) { - os << "[]"; - } else { - os << "[\n"; - depth++; - - // Serialize all values of the array with recursive calls: - for (unsigned int i = 0; i < g.value<GenericArray>().value().size(); i++) { - if (i > 0) { - os << ",\n"; - } - os << indent(depth); - leafAt(0)->printDefaultToJson(g.value<GenericArray>().value()[i], os, - depth); - } - os << "\n" << indent(--depth) << "]"; - } -} - -void NodeSymbolic::printDefaultToJson(const GenericDatum &g, std::ostream &os, - int depth) const { - getNode()->printDefaultToJson(g, os, depth); -} - -void NodeRecord::printDefaultToJson(const GenericDatum &g, std::ostream &os, - int depth) const { - assert(g.type() == AVRO_RECORD); - if (g.value<GenericRecord>().fieldCount() == 0) { - os << "{}"; - } else { - os << "{\n"; - - // Serialize all fields of the record with recursive calls: - for (unsigned int i = 0; i < g.value<GenericRecord>().fieldCount(); i++) { - if (i == 0) { - ++depth; - } else { // i > 0 - os << ",\n"; - } - - os << indent(depth) << "\""; - assert(i < leaves()); - os << leafNameAttributes_.get(i); - os << "\": "; - - // Recursive call on child node to be able to get the name attribute - // (In case of a record we need the name of the leaves (contained in - // 'this')) - leafAt(i)->printDefaultToJson(g.value<GenericRecord>().fieldAt(i), os, - depth); - } - os << "\n" << indent(--depth) << "}"; - } -} - -void NodeMap::printDefaultToJson(const GenericDatum &g, std::ostream &os, - int depth) const { - assert(g.type() == AVRO_MAP); - //{"a": 1} - if (g.value<GenericMap>().value().empty()) { - os << "{}"; - } else { - os << "{\n"; - - for (unsigned int i = 0; i < g.value<GenericMap>().value().size(); i++) { - if (i == 0) { - ++depth; - } else { - os << ",\n"; - } - os << indent(depth) << "\"" << g.value<GenericMap>().value()[i].first - << "\": "; - - leafAt(i)->printDefaultToJson(g.value<GenericMap>().value()[i].second, os, - depth); - } - os << "\n" << indent(--depth) << "}"; - } -} - -void -NodeEnum::printJson(std::ostream &os, int depth) const -{ - os << "{\n"; - os << indent(++depth) << "\"type\": \"enum\",\n"; - if (getDoc().size()) { - os << indent(depth) << "\"doc\": \"" - << escape(getDoc()) << "\",\n"; - } - printName(os, nameAttribute_.get(), depth); - os << indent(depth) << "\"symbols\": [\n"; - - int names = leafNameAttributes_.size(); - ++depth; - for (int i = 0; i < names; ++i) { - if (i > 0) { - os << ",\n"; - } - os << indent(depth) << '\"' << leafNameAttributes_.get(i) << '\"'; - } - os << '\n'; - os << indent(--depth) << "]\n"; - os << indent(--depth) << '}'; -} - -void -NodeArray::printJson(std::ostream &os, int depth) const -{ - os << "{\n"; - os << indent(depth+1) << "\"type\": \"array\",\n"; - if (getDoc().size()) { - os << indent(depth+1) << "\"doc\": \"" - << escape(getDoc()) << "\",\n"; - } - os << indent(depth+1) << "\"items\": "; - leafAttributes_.get()->printJson(os, depth+1); - os << '\n'; - os << indent(depth) << '}'; -} - -void -NodeMap::printJson(std::ostream &os, int depth) const -{ - os << "{\n"; - os << indent(depth+1) <<"\"type\": \"map\",\n"; - if (getDoc().size()) { - os << indent(depth+1) << "\"doc\": \"" - << escape(getDoc()) << "\",\n"; - } - os << indent(depth+1) << "\"values\": "; - leafAttributes_.get(1)->printJson(os, depth+1); - os << '\n'; - os << indent(depth) << '}'; -} - -void -NodeUnion::printJson(std::ostream &os, int depth) const -{ - os << "[\n"; - int fields = leafAttributes_.size(); - ++depth; - for (int i = 0; i < fields; ++i) { - if (i > 0) { - os << ",\n"; - } - os << indent(depth); - leafAttributes_.get(i)->printJson(os, depth); - } - os << '\n'; - os << indent(--depth) << ']'; -} - -void -NodeFixed::printJson(std::ostream &os, int depth) const -{ - os << "{\n"; - os << indent(++depth) << "\"type\": \"fixed\",\n"; - if (getDoc().size()) { - os << indent(depth) << "\"doc\": \"" - << escape(getDoc()) << "\",\n"; - } - printName(os, nameAttribute_.get(), depth); - os << indent(depth) << "\"size\": " << sizeAttribute_.get(); - - if (logicalType().type() != LogicalType::NONE) { - os << ",\n" << indent(depth); - logicalType().printJson(os); - } - - os << "\n" << indent(--depth) << '}'; -} - -} // namespace avro +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include <sstream> +#include <iomanip> +#include <boost/algorithm/string/replace.hpp> +#include "NodeImpl.hh" + + +using std::string; +namespace avro { + +namespace { + +// Escape string for serialization. +string escape(const string &unescaped) { + string s; + s.reserve(unescaped.length()); + for (std::string::const_iterator it = unescaped.begin(); it != unescaped.end(); ++it) { + char c = *it; + switch (c) { + case '\\': + case '"': + case '/': + s += '\\'; + s += c; + break; + case '\b': + s += '\\'; + s += 'b'; + break; + case '\f': + s += '\f'; + break; + case '\n': + s += '\\'; + s += 'n'; + break; + case '\r': + s += '\\'; + s += 'r'; + break; + case '\t': + s += '\\'; + s += 't'; + break; + default: + if (!std::iscntrl(c, std::locale::classic())) { + s += c; + continue; + } + s += intToHex(static_cast<unsigned int>(c)); + break; + } + } + return s; +} + +// Wrap an indentation in a struct for ostream operator<< +struct indent { + indent(int depth) : + d(depth) + { } + int d; +}; + +/// ostream operator for indent +std::ostream& operator <<(std::ostream &os, indent x) +{ + static const string spaces(" "); + while (x.d--) { + os << spaces; + } + return os; +} + +} // anonymous namespace + +const int kByteStringSize = 6; + +SchemaResolution +NodePrimitive::resolve(const Node &reader) const +{ + if (type() == reader.type()) { + return RESOLVE_MATCH; + } + + switch ( type() ) { + + case AVRO_INT: + + if ( reader.type() == AVRO_LONG ) { + return RESOLVE_PROMOTABLE_TO_LONG; + } + + // fall-through intentional + + case AVRO_LONG: + + if (reader.type() == AVRO_FLOAT) { + return RESOLVE_PROMOTABLE_TO_FLOAT; + } + + // fall-through intentional + + case AVRO_FLOAT: + + if (reader.type() == AVRO_DOUBLE) { + return RESOLVE_PROMOTABLE_TO_DOUBLE; + } + + default: + break; + } + + return furtherResolution(reader); +} + +SchemaResolution +NodeRecord::resolve(const Node &reader) const +{ + if (reader.type() == AVRO_RECORD) { + if (name() == reader.name()) { + return RESOLVE_MATCH; + } + } + return furtherResolution(reader); +} + +SchemaResolution +NodeEnum::resolve(const Node &reader) const +{ + if (reader.type() == AVRO_ENUM) { + return (name() == reader.name()) ? RESOLVE_MATCH : RESOLVE_NO_MATCH; + } + return furtherResolution(reader); +} + +SchemaResolution +NodeArray::resolve(const Node &reader) const +{ + if (reader.type() == AVRO_ARRAY) { + const NodePtr &arrayType = leafAt(0); + return arrayType->resolve(*reader.leafAt(0)); + } + return furtherResolution(reader); +} + +SchemaResolution +NodeMap::resolve(const Node &reader) const +{ + if (reader.type() == AVRO_MAP) { + const NodePtr &mapType = leafAt(1); + return mapType->resolve(*reader.leafAt(1)); + } + return furtherResolution(reader); +} + +SchemaResolution +NodeUnion::resolve(const Node &reader) const +{ + + // If the writer is union, resolution only needs to occur when the selected + // type of the writer is known, so this function is not very helpful. + // + // In this case, this function returns if there is a possible match given + // any writer type, so just search type by type returning the best match + // found. + + SchemaResolution match = RESOLVE_NO_MATCH; + for (size_t i=0; i < leaves(); ++i) { + const NodePtr &node = leafAt(i); + SchemaResolution thisMatch = node->resolve(reader); + if (thisMatch == RESOLVE_MATCH) { + match = thisMatch; + break; + } + if (match == RESOLVE_NO_MATCH) { + match = thisMatch; + } + } + return match; +} + +SchemaResolution +NodeFixed::resolve(const Node &reader) const +{ + if (reader.type() == AVRO_FIXED) { + return ( + (reader.fixedSize() == fixedSize()) && + (reader.name() == name()) + ) ? + RESOLVE_MATCH : RESOLVE_NO_MATCH; + } + return furtherResolution(reader); +} + +SchemaResolution +NodeSymbolic::resolve(const Node &reader) const +{ + const NodePtr &node = leafAt(0); + return node->resolve(reader); +} + +void +NodePrimitive::printJson(std::ostream &os, int depth) const +{ + bool hasLogicalType = logicalType().type() != LogicalType::NONE; + + if (hasLogicalType) { + os << "{\n" << indent(depth) << "\"type\": "; + } + + os << '\"' << type() << '\"'; + + if (hasLogicalType) { + os << ",\n" << indent(depth); + logicalType().printJson(os); + os << "\n}"; + } + if (getDoc().size()) { + os << ",\n" << indent(depth) << "\"doc\": \"" + << escape(getDoc()) << "\""; + } +} + +void +NodeSymbolic::printJson(std::ostream &os, int depth) const +{ + os << '\"' << nameAttribute_.get() << '\"'; + if (getDoc().size()) { + os << ",\n" << indent(depth) << "\"doc\": \"" + << escape(getDoc()) << "\""; + } +} + +static void printName(std::ostream& os, const Name& n, int depth) +{ + if (!n.ns().empty()) { + os << indent(depth) << "\"namespace\": \"" << n.ns() << "\",\n"; + } + os << indent(depth) << "\"name\": \"" << n.simpleName() << "\",\n"; +} + +void +NodeRecord::printJson(std::ostream &os, int depth) const +{ + os << "{\n"; + os << indent(++depth) << "\"type\": \"record\",\n"; + printName(os, nameAttribute_.get(), depth); + if (getDoc().size()) { + os << indent(depth) << "\"doc\": \"" + << escape(getDoc()) << "\",\n"; + } + os << indent(depth) << "\"fields\": ["; + + size_t fields = leafAttributes_.size(); + ++depth; + // Serialize "default" field: + assert(defaultValues.empty() || (defaultValues.size() == fields)); + for (size_t i = 0; i < fields; ++i) { + if (i > 0) { + os << ','; + } + os << '\n' << indent(depth) << "{\n"; + os << indent(++depth) << "\"name\": \"" << leafNameAttributes_.get(i) << "\",\n"; + os << indent(depth) << "\"type\": "; + leafAttributes_.get(i)->printJson(os, depth); + + if (!defaultValues.empty()) { + if (!defaultValues[i].isUnion() && + defaultValues[i].type() == AVRO_NULL) { + // No "default" field. + } else { + os << ",\n" << indent(depth) << "\"default\": "; + leafAttributes_.get(i)->printDefaultToJson(defaultValues[i], os, + depth); + } + } + os << '\n'; + os << indent(--depth) << '}'; + } + os << '\n' << indent(--depth) << "]\n"; + os << indent(--depth) << '}'; +} + +void NodePrimitive::printDefaultToJson(const GenericDatum &g, std::ostream &os, + int depth) const { + assert(isPrimitive(g.type())); + + switch (g.type()) { + case AVRO_NULL: + os << "null"; + break; + case AVRO_BOOL: + os << (g.value<bool>() ? "true" : "false"); + break; + case AVRO_INT: + os << g.value<int32_t>(); + break; + case AVRO_LONG: + os << g.value<int64_t>(); + break; + case AVRO_FLOAT: + os << g.value<float>(); + break; + case AVRO_DOUBLE: + os << g.value<double>(); + break; + case AVRO_STRING: + os << "\"" << escape(g.value<string>()) << "\""; + break; + case AVRO_BYTES: { + // Convert to a string: + const std::vector<uint8_t> &vg = g.value<std::vector<uint8_t> >(); + string s; + s.resize(vg.size() * kByteStringSize); + for (unsigned int i = 0; i < vg.size(); i++) { + string hex_string = intToHex(static_cast<int>(vg[i])); + s.replace(i*kByteStringSize, kByteStringSize, hex_string); + } + os << "\"" << s << "\""; + } break; + default: + break; + } +} + +void NodeEnum::printDefaultToJson(const GenericDatum &g, std::ostream &os, + int depth) const { + assert(g.type() == AVRO_ENUM); + os << "\"" << g.value<GenericEnum>().symbol() << "\""; +} + +void NodeFixed::printDefaultToJson(const GenericDatum &g, std::ostream &os, + int depth) const { + assert(g.type() == AVRO_FIXED); + // ex: "\uOOff" + // Convert to a string + const std::vector<uint8_t> &vg = g.value<GenericFixed>().value(); + string s; + s.resize(vg.size() * kByteStringSize); + for (unsigned int i = 0; i < vg.size(); i++) { + string hex_string = intToHex(static_cast<int>(vg[i])); + s.replace(i*kByteStringSize, kByteStringSize, hex_string); + } + os << "\"" << s << "\""; +} + +void NodeUnion::printDefaultToJson(const GenericDatum &g, std::ostream &os, + int depth) const { + leafAt(0)->printDefaultToJson(g, os, depth); +} + +void NodeArray::printDefaultToJson(const GenericDatum &g, std::ostream &os, + int depth) const { + assert(g.type() == AVRO_ARRAY); + // ex: "default": [1] + if (g.value<GenericArray>().value().empty()) { + os << "[]"; + } else { + os << "[\n"; + depth++; + + // Serialize all values of the array with recursive calls: + for (unsigned int i = 0; i < g.value<GenericArray>().value().size(); i++) { + if (i > 0) { + os << ",\n"; + } + os << indent(depth); + leafAt(0)->printDefaultToJson(g.value<GenericArray>().value()[i], os, + depth); + } + os << "\n" << indent(--depth) << "]"; + } +} + +void NodeSymbolic::printDefaultToJson(const GenericDatum &g, std::ostream &os, + int depth) const { + getNode()->printDefaultToJson(g, os, depth); +} + +void NodeRecord::printDefaultToJson(const GenericDatum &g, std::ostream &os, + int depth) const { + assert(g.type() == AVRO_RECORD); + if (g.value<GenericRecord>().fieldCount() == 0) { + os << "{}"; + } else { + os << "{\n"; + + // Serialize all fields of the record with recursive calls: + for (unsigned int i = 0; i < g.value<GenericRecord>().fieldCount(); i++) { + if (i == 0) { + ++depth; + } else { // i > 0 + os << ",\n"; + } + + os << indent(depth) << "\""; + assert(i < leaves()); + os << leafNameAttributes_.get(i); + os << "\": "; + + // Recursive call on child node to be able to get the name attribute + // (In case of a record we need the name of the leaves (contained in + // 'this')) + leafAt(i)->printDefaultToJson(g.value<GenericRecord>().fieldAt(i), os, + depth); + } + os << "\n" << indent(--depth) << "}"; + } +} + +void NodeMap::printDefaultToJson(const GenericDatum &g, std::ostream &os, + int depth) const { + assert(g.type() == AVRO_MAP); + //{"a": 1} + if (g.value<GenericMap>().value().empty()) { + os << "{}"; + } else { + os << "{\n"; + + for (unsigned int i = 0; i < g.value<GenericMap>().value().size(); i++) { + if (i == 0) { + ++depth; + } else { + os << ",\n"; + } + os << indent(depth) << "\"" << g.value<GenericMap>().value()[i].first + << "\": "; + + leafAt(i)->printDefaultToJson(g.value<GenericMap>().value()[i].second, os, + depth); + } + os << "\n" << indent(--depth) << "}"; + } +} + +void +NodeEnum::printJson(std::ostream &os, int depth) const +{ + os << "{\n"; + os << indent(++depth) << "\"type\": \"enum\",\n"; + if (getDoc().size()) { + os << indent(depth) << "\"doc\": \"" + << escape(getDoc()) << "\",\n"; + } + printName(os, nameAttribute_.get(), depth); + os << indent(depth) << "\"symbols\": [\n"; + + int names = leafNameAttributes_.size(); + ++depth; + for (int i = 0; i < names; ++i) { + if (i > 0) { + os << ",\n"; + } + os << indent(depth) << '\"' << leafNameAttributes_.get(i) << '\"'; + } + os << '\n'; + os << indent(--depth) << "]\n"; + os << indent(--depth) << '}'; +} + +void +NodeArray::printJson(std::ostream &os, int depth) const +{ + os << "{\n"; + os << indent(depth+1) << "\"type\": \"array\",\n"; + if (getDoc().size()) { + os << indent(depth+1) << "\"doc\": \"" + << escape(getDoc()) << "\",\n"; + } + os << indent(depth+1) << "\"items\": "; + leafAttributes_.get()->printJson(os, depth+1); + os << '\n'; + os << indent(depth) << '}'; +} + +void +NodeMap::printJson(std::ostream &os, int depth) const +{ + os << "{\n"; + os << indent(depth+1) <<"\"type\": \"map\",\n"; + if (getDoc().size()) { + os << indent(depth+1) << "\"doc\": \"" + << escape(getDoc()) << "\",\n"; + } + os << indent(depth+1) << "\"values\": "; + leafAttributes_.get(1)->printJson(os, depth+1); + os << '\n'; + os << indent(depth) << '}'; +} + +void +NodeUnion::printJson(std::ostream &os, int depth) const +{ + os << "[\n"; + int fields = leafAttributes_.size(); + ++depth; + for (int i = 0; i < fields; ++i) { + if (i > 0) { + os << ",\n"; + } + os << indent(depth); + leafAttributes_.get(i)->printJson(os, depth); + } + os << '\n'; + os << indent(--depth) << ']'; +} + +void +NodeFixed::printJson(std::ostream &os, int depth) const +{ + os << "{\n"; + os << indent(++depth) << "\"type\": \"fixed\",\n"; + if (getDoc().size()) { + os << indent(depth) << "\"doc\": \"" + << escape(getDoc()) << "\",\n"; + } + printName(os, nameAttribute_.get(), depth); + os << indent(depth) << "\"size\": " << sizeAttribute_.get(); + + if (logicalType().type() != LogicalType::NONE) { + os << ",\n" << indent(depth); + logicalType().printJson(os); + } + + os << "\n" << indent(--depth) << '}'; +} + +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/Resolver.cc b/contrib/libs/apache/avro/impl/Resolver.cc index 209f99d98e..43467c028d 100644 --- a/contrib/libs/apache/avro/impl/Resolver.cc +++ b/contrib/libs/apache/avro/impl/Resolver.cc @@ -1,872 +1,872 @@ - -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <memory> -#include "Resolver.hh" -#include "Layout.hh" -#include "NodeImpl.hh" -#include "ValidSchema.hh" -#include "Reader.hh" -#include "AvroTraits.hh" - -namespace avro { -using std::unique_ptr; - -class ResolverFactory; -typedef std::shared_ptr<Resolver> ResolverPtr; -typedef std::vector<std::unique_ptr<Resolver> > ResolverPtrVector; - -// #define DEBUG_VERBOSE - -#ifdef DEBUG_VERBOSE -#define DEBUG_OUT(str) std::cout << str << '\n' -#else -class NoOp {}; -template<typename T> NoOp& operator<<(NoOp &noOp, const T&) { - return noOp; -} -NoOp noop; -#define DEBUG_OUT(str) noop << str -#endif - -template<typename T> -class PrimitiveSkipper : public Resolver -{ - public: - - PrimitiveSkipper() : - Resolver() - {} - - virtual void parse(Reader &reader, uint8_t *address) const - { - T val; - reader.readValue(val); - DEBUG_OUT("Skipping " << val); - } -}; - -template<typename T> -class PrimitiveParser : public Resolver -{ - public: - - PrimitiveParser(const PrimitiveLayout &offset) : - Resolver(), - offset_(offset.offset()) - {} - - virtual void parse(Reader &reader, uint8_t *address) const - { - T* location = reinterpret_cast<T *> (address + offset_); - reader.readValue(*location); - DEBUG_OUT("Reading " << *location); - } - - private: - - size_t offset_; -}; - -template<typename WT, typename RT> -class PrimitivePromoter : public Resolver -{ - public: - - PrimitivePromoter(const PrimitiveLayout &offset) : - Resolver(), - offset_(offset.offset()) - {} - - virtual void parse(Reader &reader, uint8_t *address) const - { - parseIt<WT>(reader, address); - } - - private: - - void parseIt(Reader &reader, uint8_t *address, const std::true_type &) const - { - WT val; - reader.readValue(val); - RT *location = reinterpret_cast<RT *> (address + offset_); - *location = static_cast<RT>(val); - DEBUG_OUT("Promoting " << val); - } - - void parseIt(Reader &reader, uint8_t *address, const std::false_type &) const - { } - - template<typename T> - void parseIt(Reader &reader, uint8_t *address) const - { - parseIt(reader, address, is_promotable<T>()); - } - - size_t offset_; -}; - -template <> -class PrimitiveSkipper<std::vector<uint8_t> > : public Resolver -{ - public: - - PrimitiveSkipper() : - Resolver() - {} - - virtual void parse(Reader &reader, uint8_t *address) const - { - std::vector<uint8_t> val; - reader.readBytes(val); - DEBUG_OUT("Skipping bytes"); - } -}; - -template <> -class PrimitiveParser<std::vector<uint8_t> > : public Resolver -{ - public: - - PrimitiveParser(const PrimitiveLayout &offset) : - Resolver(), - offset_(offset.offset()) - {} - - virtual void parse(Reader &reader, uint8_t *address) const - { - std::vector<uint8_t> *location = reinterpret_cast<std::vector<uint8_t> *> (address + offset_); - reader.readBytes(*location); - DEBUG_OUT("Reading bytes"); - } - - private: - - size_t offset_; -}; - -class RecordSkipper : public Resolver -{ - public: - - RecordSkipper(ResolverFactory &factory, const NodePtr &writer); - - virtual void parse(Reader &reader, uint8_t *address) const - { - DEBUG_OUT("Skipping record"); - - reader.readRecord(); - size_t steps = resolvers_.size(); - for(size_t i = 0; i < steps; ++i) { - resolvers_[i]->parse(reader, address); - } - } - - protected: - - ResolverPtrVector resolvers_; - -}; - -class RecordParser : public Resolver -{ - public: - - virtual void parse(Reader &reader, uint8_t *address) const - { - DEBUG_OUT("Reading record"); - - reader.readRecord(); - size_t steps = resolvers_.size(); - for(size_t i = 0; i < steps; ++i) { - resolvers_[i]->parse(reader, address); - } - } - - RecordParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets); - - protected: - - ResolverPtrVector resolvers_; - -}; - - -class MapSkipper : public Resolver -{ - public: - - MapSkipper(ResolverFactory &factory, const NodePtr &writer); - - virtual void parse(Reader &reader, uint8_t *address) const - { - DEBUG_OUT("Skipping map"); - - std::string key; - int64_t size = 0; - do { - size = reader.readMapBlockSize(); - for(int64_t i = 0; i < size; ++i) { - reader.readValue(key); - resolver_->parse(reader, address); - } - } while (size != 0); - } - - protected: - - ResolverPtr resolver_; -}; - - -class MapParser : public Resolver -{ - public: - - typedef uint8_t *(*GenericMapSetter)(uint8_t *map, const std::string &key); - - MapParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets); - - virtual void parse(Reader &reader, uint8_t *address) const - { - DEBUG_OUT("Reading map"); - - uint8_t *mapAddress = address + offset_; - - std::string key; - GenericMapSetter* setter = reinterpret_cast<GenericMapSetter *> (address + setFuncOffset_); - - int64_t size = 0; - do { - size = reader.readMapBlockSize(); - for(int64_t i = 0; i < size; ++i) { - reader.readValue(key); - - // create a new map entry and get the address - uint8_t *location = (*setter)(mapAddress, key); - resolver_->parse(reader, location); - } - } while (size != 0); - } - - protected: - - ResolverPtr resolver_; - size_t offset_; - size_t setFuncOffset_; -}; - -class ArraySkipper : public Resolver -{ - public: - - ArraySkipper(ResolverFactory &factory, const NodePtr &writer); - - virtual void parse(Reader &reader, uint8_t *address) const - { - DEBUG_OUT("Skipping array"); - - int64_t size = 0; - do { - size = reader.readArrayBlockSize(); - for(int64_t i = 0; i < size; ++i) { - resolver_->parse(reader, address); - } - } while (size != 0); - } - - protected: - - ResolverPtr resolver_; -}; - -typedef uint8_t *(*GenericArraySetter)(uint8_t *array); - -class ArrayParser : public Resolver -{ - public: - - ArrayParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets); - - virtual void parse(Reader &reader, uint8_t *address) const - { - DEBUG_OUT("Reading array"); - - uint8_t *arrayAddress = address + offset_; - - GenericArraySetter* setter = reinterpret_cast<GenericArraySetter *> (address + setFuncOffset_); - - int64_t size = 0; - do { - size = reader.readArrayBlockSize(); - for(int64_t i = 0; i < size; ++i) { - // create a new map entry and get the address - uint8_t *location = (*setter)(arrayAddress); - resolver_->parse(reader, location); - } - } while (size != 0); - } - - protected: - - ArrayParser() : - Resolver() - {} - - ResolverPtr resolver_; - size_t offset_; - size_t setFuncOffset_; -}; - -class EnumSkipper : public Resolver -{ - public: - - EnumSkipper(ResolverFactory &factory, const NodePtr &writer) : - Resolver() - { } - - virtual void parse(Reader &reader, uint8_t *address) const - { - int64_t val = reader.readEnum(); - DEBUG_OUT("Skipping enum" << val); - } -}; - -class EnumParser : public Resolver -{ - public: - - enum EnumRepresentation { - VAL - }; - - EnumParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets) : - Resolver(), - offset_(offsets.at(0).offset()), - readerSize_(reader->names()) - { - const size_t writerSize = writer->names(); - - mapping_.reserve(writerSize); - - for(size_t i = 0; i < writerSize; ++i) { - const std::string &name = writer->nameAt(i); - size_t readerIndex = readerSize_; - reader->nameIndex(name, readerIndex); - mapping_.push_back(readerIndex); - } - } - - virtual void parse(Reader &reader, uint8_t *address) const - { - size_t val = static_cast<size_t>(reader.readEnum()); - assert(static_cast<size_t>(val) < mapping_.size()); - - if(mapping_[val] < readerSize_) { - EnumRepresentation* location = reinterpret_cast<EnumRepresentation *> (address + offset_); - *location = static_cast<EnumRepresentation>(mapping_[val]); - DEBUG_OUT("Setting enum" << *location); - } - } - -protected: - - size_t offset_; - size_t readerSize_; - std::vector<size_t> mapping_; - -}; - -class UnionSkipper : public Resolver -{ - public: - - UnionSkipper(ResolverFactory &factory, const NodePtr &writer); - - virtual void parse(Reader &reader, uint8_t *address) const - { - DEBUG_OUT("Skipping union"); - size_t choice = static_cast<size_t>(reader.readUnion()); - resolvers_[choice]->parse(reader, address); - } - - protected: - - ResolverPtrVector resolvers_; -}; - - -class UnionParser : public Resolver -{ - public: - - typedef uint8_t *(*GenericUnionSetter)(uint8_t *, int64_t); - - UnionParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets); - - virtual void parse(Reader &reader, uint8_t *address) const - { - DEBUG_OUT("Reading union"); - size_t writerChoice = static_cast<size_t>(reader.readUnion()); - int64_t *readerChoice = reinterpret_cast<int64_t *>(address + choiceOffset_); - - *readerChoice = choiceMapping_[writerChoice]; - GenericUnionSetter* setter = reinterpret_cast<GenericUnionSetter *> (address + setFuncOffset_); - uint8_t *value = reinterpret_cast<uint8_t *> (address + offset_); - uint8_t *location = (*setter)(value, *readerChoice); - - resolvers_[writerChoice]->parse(reader, location); - } - - protected: - - ResolverPtrVector resolvers_; - std::vector<int64_t> choiceMapping_; - size_t offset_; - size_t choiceOffset_; - size_t setFuncOffset_; -}; - -class UnionToNonUnionParser : public Resolver -{ - public: - - typedef uint8_t *(*GenericUnionSetter)(uint8_t *, int64_t); - - UnionToNonUnionParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const Layout &offsets); - - virtual void parse(Reader &reader, uint8_t *address) const - { - DEBUG_OUT("Reading union to non-union"); - size_t choice = static_cast<size_t>(reader.readUnion()); - resolvers_[choice]->parse(reader, address); - } - - protected: - - ResolverPtrVector resolvers_; -}; - -class NonUnionToUnionParser : public Resolver -{ - public: - - typedef uint8_t *(*GenericUnionSetter)(uint8_t *, int64_t); - - NonUnionToUnionParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets); - - virtual void parse(Reader &reader, uint8_t *address) const - { - DEBUG_OUT("Reading non-union to union"); - - int64_t *choice = reinterpret_cast<int64_t *>(address + choiceOffset_); - *choice = choice_; - GenericUnionSetter* setter = reinterpret_cast<GenericUnionSetter *> (address + setFuncOffset_); - uint8_t *value = reinterpret_cast<uint8_t *> (address + offset_); - uint8_t *location = (*setter)(value, choice_); - - resolver_->parse(reader, location); - } - - protected: - - ResolverPtr resolver_; - size_t choice_; - size_t offset_; - size_t choiceOffset_; - size_t setFuncOffset_; -}; - -class FixedSkipper : public Resolver -{ - public: - - FixedSkipper(ResolverFactory &factory, const NodePtr &writer) : - Resolver() - { - size_ = writer->fixedSize(); - } - - virtual void parse(Reader &reader, uint8_t *address) const - { - DEBUG_OUT("Skipping fixed"); - std::unique_ptr<uint8_t[]> val(new uint8_t[size_]); - reader.readFixed(&val[0], size_); - } - - protected: - - int size_; - -}; - -class FixedParser : public Resolver -{ - public: - - FixedParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets) : - Resolver() - { - size_ = writer->fixedSize(); - offset_ = offsets.at(0).offset(); - } - - virtual void parse(Reader &reader, uint8_t *address) const - { - DEBUG_OUT("Reading fixed"); - uint8_t *location = reinterpret_cast<uint8_t *> (address + offset_); - reader.readFixed(location, size_); - } - - protected: - - int size_; - size_t offset_; - -}; - - -class ResolverFactory : private boost::noncopyable { - - template<typename T> - unique_ptr<Resolver> - constructPrimitiveSkipper(const NodePtr &writer) - { - return unique_ptr<Resolver>(new PrimitiveSkipper<T>()); - } - - template<typename T> - unique_ptr<Resolver> - constructPrimitive(const NodePtr &writer, const NodePtr &reader, const Layout &offset) - { - unique_ptr<Resolver> instruction; - - SchemaResolution match = writer->resolve(*reader); - - if (match == RESOLVE_NO_MATCH) { - instruction = unique_ptr<Resolver>(new PrimitiveSkipper<T>()); - } - else if (reader->type() == AVRO_UNION) { - const CompoundLayout &compoundLayout = static_cast<const CompoundLayout &>(offset); - instruction = unique_ptr<Resolver>(new NonUnionToUnionParser(*this, writer, reader, compoundLayout)); - } - else if (match == RESOLVE_MATCH) { - const PrimitiveLayout &primitiveLayout = static_cast<const PrimitiveLayout &>(offset); - instruction = unique_ptr<Resolver>(new PrimitiveParser<T>(primitiveLayout)); - } - else if(match == RESOLVE_PROMOTABLE_TO_LONG) { - const PrimitiveLayout &primitiveLayout = static_cast<const PrimitiveLayout &>(offset); - instruction = unique_ptr<Resolver>(new PrimitivePromoter<T, int64_t>(primitiveLayout)); - } - else if(match == RESOLVE_PROMOTABLE_TO_FLOAT) { - const PrimitiveLayout &primitiveLayout = static_cast<const PrimitiveLayout &>(offset); - instruction = unique_ptr<Resolver>(new PrimitivePromoter<T, float>(primitiveLayout)); - } - else if(match == RESOLVE_PROMOTABLE_TO_DOUBLE) { - const PrimitiveLayout &primitiveLayout = static_cast<const PrimitiveLayout &>(offset); - instruction = unique_ptr<Resolver>(new PrimitivePromoter<T, double>(primitiveLayout)); - } - else { - assert(0); - } - return instruction; - } - - template<typename Skipper> - unique_ptr<Resolver> - constructCompoundSkipper(const NodePtr &writer) - { - return unique_ptr<Resolver>(new Skipper(*this, writer)); - } - - - template<typename Parser, typename Skipper> - unique_ptr<Resolver> - constructCompound(const NodePtr &writer, const NodePtr &reader, const Layout &offset) - { - unique_ptr<Resolver> instruction; - - SchemaResolution match = RESOLVE_NO_MATCH; - - match = writer->resolve(*reader); - - if (match == RESOLVE_NO_MATCH) { - instruction = unique_ptr<Resolver>(new Skipper(*this, writer)); - } - else if(writer->type() != AVRO_UNION && reader->type() == AVRO_UNION) { - const CompoundLayout &compoundLayout = dynamic_cast<const CompoundLayout &>(offset); - instruction = unique_ptr<Resolver>(new NonUnionToUnionParser(*this, writer, reader, compoundLayout)); - } - else if(writer->type() == AVRO_UNION && reader->type() != AVRO_UNION) { - instruction = unique_ptr<Resolver>(new UnionToNonUnionParser(*this, writer, reader, offset)); - } - else { - const CompoundLayout &compoundLayout = dynamic_cast<const CompoundLayout &>(offset); - instruction = unique_ptr<Resolver>(new Parser(*this, writer, reader, compoundLayout)); - } - - return instruction; - } - - public: - - unique_ptr<Resolver> - construct(const NodePtr &writer, const NodePtr &reader, const Layout &offset) - { - - typedef unique_ptr<Resolver> (ResolverFactory::*BuilderFunc)(const NodePtr &writer, const NodePtr &reader, const Layout &offset); - - NodePtr currentWriter = (writer->type() == AVRO_SYMBOLIC) ? - resolveSymbol(writer) : writer; - - NodePtr currentReader = (reader->type() == AVRO_SYMBOLIC) ? - resolveSymbol(reader) : reader; - - static const BuilderFunc funcs[] = { - &ResolverFactory::constructPrimitive<std::string>, - &ResolverFactory::constructPrimitive<std::vector<uint8_t> >, - &ResolverFactory::constructPrimitive<int32_t>, - &ResolverFactory::constructPrimitive<int64_t>, - &ResolverFactory::constructPrimitive<float>, - &ResolverFactory::constructPrimitive<double>, - &ResolverFactory::constructPrimitive<bool>, - &ResolverFactory::constructPrimitive<Null>, - &ResolverFactory::constructCompound<RecordParser, RecordSkipper>, - &ResolverFactory::constructCompound<EnumParser, EnumSkipper>, - &ResolverFactory::constructCompound<ArrayParser, ArraySkipper>, - &ResolverFactory::constructCompound<MapParser, MapSkipper>, - &ResolverFactory::constructCompound<UnionParser, UnionSkipper>, - &ResolverFactory::constructCompound<FixedParser, FixedSkipper> - }; - - static_assert((sizeof(funcs)/sizeof(BuilderFunc)) == (AVRO_NUM_TYPES), - "Invalid number of builder functions"); - - BuilderFunc func = funcs[currentWriter->type()]; - assert(func); - - return ((this)->*(func))(currentWriter, currentReader, offset); - } - - unique_ptr<Resolver> - skipper(const NodePtr &writer) - { - - typedef unique_ptr<Resolver> (ResolverFactory::*BuilderFunc)(const NodePtr &writer); - - NodePtr currentWriter = (writer->type() == AVRO_SYMBOLIC) ? - writer->leafAt(0) : writer; - - static const BuilderFunc funcs[] = { - &ResolverFactory::constructPrimitiveSkipper<std::string>, - &ResolverFactory::constructPrimitiveSkipper<std::vector<uint8_t> >, - &ResolverFactory::constructPrimitiveSkipper<int32_t>, - &ResolverFactory::constructPrimitiveSkipper<int64_t>, - &ResolverFactory::constructPrimitiveSkipper<float>, - &ResolverFactory::constructPrimitiveSkipper<double>, - &ResolverFactory::constructPrimitiveSkipper<bool>, - &ResolverFactory::constructPrimitiveSkipper<Null>, - &ResolverFactory::constructCompoundSkipper<RecordSkipper>, - &ResolverFactory::constructCompoundSkipper<EnumSkipper>, - &ResolverFactory::constructCompoundSkipper<ArraySkipper>, - &ResolverFactory::constructCompoundSkipper<MapSkipper>, - &ResolverFactory::constructCompoundSkipper<UnionSkipper>, - &ResolverFactory::constructCompoundSkipper<FixedSkipper> - }; - - static_assert((sizeof(funcs)/sizeof(BuilderFunc)) == (AVRO_NUM_TYPES), - "Invalid number of builder functions"); - - BuilderFunc func = funcs[currentWriter->type()]; - assert(func); - - return ((this)->*(func))(currentWriter); - } -}; - - -RecordSkipper::RecordSkipper(ResolverFactory &factory, const NodePtr &writer) : - Resolver() -{ - size_t leaves = writer->leaves(); - resolvers_.reserve(leaves); - for(size_t i = 0; i < leaves; ++i) { - const NodePtr &w = writer->leafAt(i); - resolvers_.push_back(factory.skipper(w)); - } -} - -RecordParser::RecordParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets) : - Resolver() -{ - size_t leaves = writer->leaves(); - resolvers_.reserve(leaves); - for(size_t i = 0; i < leaves; ++i) { - - const NodePtr &w = writer->leafAt(i); - - const std::string &name = writer->nameAt(i); - - size_t readerIndex = 0; - bool found = reader->nameIndex(name, readerIndex); - - if(found) { - const NodePtr &r = reader->leafAt(readerIndex); - resolvers_.push_back(factory.construct(w, r, offsets.at(readerIndex))); - } - else { - resolvers_.push_back(factory.skipper(w)); - } - } -} - -MapSkipper::MapSkipper(ResolverFactory &factory, const NodePtr &writer) : - Resolver(), - resolver_(factory.skipper(writer->leafAt(1))) -{ } - -MapParser::MapParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets) : - Resolver(), - resolver_(factory.construct(writer->leafAt(1), reader->leafAt(1), offsets.at(1))), - offset_(offsets.offset()), - setFuncOffset_( offsets.at(0).offset()) -{ } - -ArraySkipper::ArraySkipper(ResolverFactory &factory, const NodePtr &writer) : - Resolver(), - resolver_(factory.skipper(writer->leafAt(0))) -{ } - -ArrayParser::ArrayParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets) : - Resolver(), - resolver_(factory.construct(writer->leafAt(0), reader->leafAt(0), offsets.at(1))), - offset_(offsets.offset()), - setFuncOffset_(offsets.at(0).offset()) -{ } - -UnionSkipper::UnionSkipper(ResolverFactory &factory, const NodePtr &writer) : - Resolver() -{ - size_t leaves = writer->leaves(); - resolvers_.reserve(leaves); - for(size_t i = 0; i < leaves; ++i) { - const NodePtr &w = writer->leafAt(i); - resolvers_.push_back(factory.skipper(w)); - } -} - -namespace { - -// asumes the writer is NOT a union, and the reader IS a union - -SchemaResolution -checkUnionMatch(const NodePtr &writer, const NodePtr &reader, size_t &index) -{ - SchemaResolution bestMatch = RESOLVE_NO_MATCH; - - index = 0; - size_t leaves = reader->leaves(); - - for(size_t i=0; i < leaves; ++i) { - - const NodePtr &leaf = reader->leafAt(i); - SchemaResolution newMatch = writer->resolve(*leaf); - - if(newMatch == RESOLVE_MATCH) { - bestMatch = newMatch; - index = i; - break; - } - if(bestMatch == RESOLVE_NO_MATCH) { - bestMatch = newMatch; - index = i; - } - } - - return bestMatch; -} - -}; - -UnionParser::UnionParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets) : - Resolver(), - offset_(offsets.offset()), - choiceOffset_(offsets.at(0).offset()), - setFuncOffset_(offsets.at(1).offset()) -{ - - size_t leaves = writer->leaves(); - resolvers_.reserve(leaves); - choiceMapping_.reserve(leaves); - for(size_t i = 0; i < leaves; ++i) { - - // for each writer, we need a schema match for the reader - const NodePtr &w = writer->leafAt(i); - size_t index = 0; - - SchemaResolution match = checkUnionMatch(w, reader, index); - - if(match == RESOLVE_NO_MATCH) { - resolvers_.push_back(factory.skipper(w)); - // push back a non-sensical number - choiceMapping_.push_back(reader->leaves()); - } - else { - const NodePtr &r = reader->leafAt(index); - resolvers_.push_back(factory.construct(w, r, offsets.at(index+2))); - choiceMapping_.push_back(index); - } - } -} - -NonUnionToUnionParser::NonUnionToUnionParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets) : - Resolver(), - offset_(offsets.offset()), - choiceOffset_(offsets.at(0).offset()), - setFuncOffset_(offsets.at(1).offset()) -{ -#ifndef NDEBUG - SchemaResolution bestMatch = -#endif - checkUnionMatch(writer, reader, choice_); - assert(bestMatch != RESOLVE_NO_MATCH); - resolver_ = factory.construct(writer, reader->leafAt(choice_), offsets.at(choice_+2)); -} - -UnionToNonUnionParser::UnionToNonUnionParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const Layout &offsets) : - Resolver() -{ - size_t leaves = writer->leaves(); - resolvers_.reserve(leaves); - for(size_t i = 0; i < leaves; ++i) { - const NodePtr &w = writer->leafAt(i); - resolvers_.push_back(factory.construct(w, reader, offsets)); - } -} - -unique_ptr<Resolver> constructResolver(const ValidSchema &writerSchema, - const ValidSchema &readerSchema, - const Layout &readerLayout) -{ - ResolverFactory factory; - return factory.construct(writerSchema.root(), readerSchema.root(), readerLayout); -} - -} // namespace avro + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <memory> +#include "Resolver.hh" +#include "Layout.hh" +#include "NodeImpl.hh" +#include "ValidSchema.hh" +#include "Reader.hh" +#include "AvroTraits.hh" + +namespace avro { +using std::unique_ptr; + +class ResolverFactory; +typedef std::shared_ptr<Resolver> ResolverPtr; +typedef std::vector<std::unique_ptr<Resolver> > ResolverPtrVector; + +// #define DEBUG_VERBOSE + +#ifdef DEBUG_VERBOSE +#define DEBUG_OUT(str) std::cout << str << '\n' +#else +class NoOp {}; +template<typename T> NoOp& operator<<(NoOp &noOp, const T&) { + return noOp; +} +NoOp noop; +#define DEBUG_OUT(str) noop << str +#endif + +template<typename T> +class PrimitiveSkipper : public Resolver +{ + public: + + PrimitiveSkipper() : + Resolver() + {} + + virtual void parse(Reader &reader, uint8_t *address) const + { + T val; + reader.readValue(val); + DEBUG_OUT("Skipping " << val); + } +}; + +template<typename T> +class PrimitiveParser : public Resolver +{ + public: + + PrimitiveParser(const PrimitiveLayout &offset) : + Resolver(), + offset_(offset.offset()) + {} + + virtual void parse(Reader &reader, uint8_t *address) const + { + T* location = reinterpret_cast<T *> (address + offset_); + reader.readValue(*location); + DEBUG_OUT("Reading " << *location); + } + + private: + + size_t offset_; +}; + +template<typename WT, typename RT> +class PrimitivePromoter : public Resolver +{ + public: + + PrimitivePromoter(const PrimitiveLayout &offset) : + Resolver(), + offset_(offset.offset()) + {} + + virtual void parse(Reader &reader, uint8_t *address) const + { + parseIt<WT>(reader, address); + } + + private: + + void parseIt(Reader &reader, uint8_t *address, const std::true_type &) const + { + WT val; + reader.readValue(val); + RT *location = reinterpret_cast<RT *> (address + offset_); + *location = static_cast<RT>(val); + DEBUG_OUT("Promoting " << val); + } + + void parseIt(Reader &reader, uint8_t *address, const std::false_type &) const + { } + + template<typename T> + void parseIt(Reader &reader, uint8_t *address) const + { + parseIt(reader, address, is_promotable<T>()); + } + + size_t offset_; +}; + +template <> +class PrimitiveSkipper<std::vector<uint8_t> > : public Resolver +{ + public: + + PrimitiveSkipper() : + Resolver() + {} + + virtual void parse(Reader &reader, uint8_t *address) const + { + std::vector<uint8_t> val; + reader.readBytes(val); + DEBUG_OUT("Skipping bytes"); + } +}; + +template <> +class PrimitiveParser<std::vector<uint8_t> > : public Resolver +{ + public: + + PrimitiveParser(const PrimitiveLayout &offset) : + Resolver(), + offset_(offset.offset()) + {} + + virtual void parse(Reader &reader, uint8_t *address) const + { + std::vector<uint8_t> *location = reinterpret_cast<std::vector<uint8_t> *> (address + offset_); + reader.readBytes(*location); + DEBUG_OUT("Reading bytes"); + } + + private: + + size_t offset_; +}; + +class RecordSkipper : public Resolver +{ + public: + + RecordSkipper(ResolverFactory &factory, const NodePtr &writer); + + virtual void parse(Reader &reader, uint8_t *address) const + { + DEBUG_OUT("Skipping record"); + + reader.readRecord(); + size_t steps = resolvers_.size(); + for(size_t i = 0; i < steps; ++i) { + resolvers_[i]->parse(reader, address); + } + } + + protected: + + ResolverPtrVector resolvers_; + +}; + +class RecordParser : public Resolver +{ + public: + + virtual void parse(Reader &reader, uint8_t *address) const + { + DEBUG_OUT("Reading record"); + + reader.readRecord(); + size_t steps = resolvers_.size(); + for(size_t i = 0; i < steps; ++i) { + resolvers_[i]->parse(reader, address); + } + } + + RecordParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets); + + protected: + + ResolverPtrVector resolvers_; + +}; + + +class MapSkipper : public Resolver +{ + public: + + MapSkipper(ResolverFactory &factory, const NodePtr &writer); + + virtual void parse(Reader &reader, uint8_t *address) const + { + DEBUG_OUT("Skipping map"); + + std::string key; + int64_t size = 0; + do { + size = reader.readMapBlockSize(); + for(int64_t i = 0; i < size; ++i) { + reader.readValue(key); + resolver_->parse(reader, address); + } + } while (size != 0); + } + + protected: + + ResolverPtr resolver_; +}; + + +class MapParser : public Resolver +{ + public: + + typedef uint8_t *(*GenericMapSetter)(uint8_t *map, const std::string &key); + + MapParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets); + + virtual void parse(Reader &reader, uint8_t *address) const + { + DEBUG_OUT("Reading map"); + + uint8_t *mapAddress = address + offset_; + + std::string key; + GenericMapSetter* setter = reinterpret_cast<GenericMapSetter *> (address + setFuncOffset_); + + int64_t size = 0; + do { + size = reader.readMapBlockSize(); + for(int64_t i = 0; i < size; ++i) { + reader.readValue(key); + + // create a new map entry and get the address + uint8_t *location = (*setter)(mapAddress, key); + resolver_->parse(reader, location); + } + } while (size != 0); + } + + protected: + + ResolverPtr resolver_; + size_t offset_; + size_t setFuncOffset_; +}; + +class ArraySkipper : public Resolver +{ + public: + + ArraySkipper(ResolverFactory &factory, const NodePtr &writer); + + virtual void parse(Reader &reader, uint8_t *address) const + { + DEBUG_OUT("Skipping array"); + + int64_t size = 0; + do { + size = reader.readArrayBlockSize(); + for(int64_t i = 0; i < size; ++i) { + resolver_->parse(reader, address); + } + } while (size != 0); + } + + protected: + + ResolverPtr resolver_; +}; + +typedef uint8_t *(*GenericArraySetter)(uint8_t *array); + +class ArrayParser : public Resolver +{ + public: + + ArrayParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets); + + virtual void parse(Reader &reader, uint8_t *address) const + { + DEBUG_OUT("Reading array"); + + uint8_t *arrayAddress = address + offset_; + + GenericArraySetter* setter = reinterpret_cast<GenericArraySetter *> (address + setFuncOffset_); + + int64_t size = 0; + do { + size = reader.readArrayBlockSize(); + for(int64_t i = 0; i < size; ++i) { + // create a new map entry and get the address + uint8_t *location = (*setter)(arrayAddress); + resolver_->parse(reader, location); + } + } while (size != 0); + } + + protected: + + ArrayParser() : + Resolver() + {} + + ResolverPtr resolver_; + size_t offset_; + size_t setFuncOffset_; +}; + +class EnumSkipper : public Resolver +{ + public: + + EnumSkipper(ResolverFactory &factory, const NodePtr &writer) : + Resolver() + { } + + virtual void parse(Reader &reader, uint8_t *address) const + { + int64_t val = reader.readEnum(); + DEBUG_OUT("Skipping enum" << val); + } +}; + +class EnumParser : public Resolver +{ + public: + + enum EnumRepresentation { + VAL + }; + + EnumParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets) : + Resolver(), + offset_(offsets.at(0).offset()), + readerSize_(reader->names()) + { + const size_t writerSize = writer->names(); + + mapping_.reserve(writerSize); + + for(size_t i = 0; i < writerSize; ++i) { + const std::string &name = writer->nameAt(i); + size_t readerIndex = readerSize_; + reader->nameIndex(name, readerIndex); + mapping_.push_back(readerIndex); + } + } + + virtual void parse(Reader &reader, uint8_t *address) const + { + size_t val = static_cast<size_t>(reader.readEnum()); + assert(static_cast<size_t>(val) < mapping_.size()); + + if(mapping_[val] < readerSize_) { + EnumRepresentation* location = reinterpret_cast<EnumRepresentation *> (address + offset_); + *location = static_cast<EnumRepresentation>(mapping_[val]); + DEBUG_OUT("Setting enum" << *location); + } + } + +protected: + + size_t offset_; + size_t readerSize_; + std::vector<size_t> mapping_; + +}; + +class UnionSkipper : public Resolver +{ + public: + + UnionSkipper(ResolverFactory &factory, const NodePtr &writer); + + virtual void parse(Reader &reader, uint8_t *address) const + { + DEBUG_OUT("Skipping union"); + size_t choice = static_cast<size_t>(reader.readUnion()); + resolvers_[choice]->parse(reader, address); + } + + protected: + + ResolverPtrVector resolvers_; +}; + + +class UnionParser : public Resolver +{ + public: + + typedef uint8_t *(*GenericUnionSetter)(uint8_t *, int64_t); + + UnionParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets); + + virtual void parse(Reader &reader, uint8_t *address) const + { + DEBUG_OUT("Reading union"); + size_t writerChoice = static_cast<size_t>(reader.readUnion()); + int64_t *readerChoice = reinterpret_cast<int64_t *>(address + choiceOffset_); + + *readerChoice = choiceMapping_[writerChoice]; + GenericUnionSetter* setter = reinterpret_cast<GenericUnionSetter *> (address + setFuncOffset_); + uint8_t *value = reinterpret_cast<uint8_t *> (address + offset_); + uint8_t *location = (*setter)(value, *readerChoice); + + resolvers_[writerChoice]->parse(reader, location); + } + + protected: + + ResolverPtrVector resolvers_; + std::vector<int64_t> choiceMapping_; + size_t offset_; + size_t choiceOffset_; + size_t setFuncOffset_; +}; + +class UnionToNonUnionParser : public Resolver +{ + public: + + typedef uint8_t *(*GenericUnionSetter)(uint8_t *, int64_t); + + UnionToNonUnionParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const Layout &offsets); + + virtual void parse(Reader &reader, uint8_t *address) const + { + DEBUG_OUT("Reading union to non-union"); + size_t choice = static_cast<size_t>(reader.readUnion()); + resolvers_[choice]->parse(reader, address); + } + + protected: + + ResolverPtrVector resolvers_; +}; + +class NonUnionToUnionParser : public Resolver +{ + public: + + typedef uint8_t *(*GenericUnionSetter)(uint8_t *, int64_t); + + NonUnionToUnionParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets); + + virtual void parse(Reader &reader, uint8_t *address) const + { + DEBUG_OUT("Reading non-union to union"); + + int64_t *choice = reinterpret_cast<int64_t *>(address + choiceOffset_); + *choice = choice_; + GenericUnionSetter* setter = reinterpret_cast<GenericUnionSetter *> (address + setFuncOffset_); + uint8_t *value = reinterpret_cast<uint8_t *> (address + offset_); + uint8_t *location = (*setter)(value, choice_); + + resolver_->parse(reader, location); + } + + protected: + + ResolverPtr resolver_; + size_t choice_; + size_t offset_; + size_t choiceOffset_; + size_t setFuncOffset_; +}; + +class FixedSkipper : public Resolver +{ + public: + + FixedSkipper(ResolverFactory &factory, const NodePtr &writer) : + Resolver() + { + size_ = writer->fixedSize(); + } + + virtual void parse(Reader &reader, uint8_t *address) const + { + DEBUG_OUT("Skipping fixed"); + std::unique_ptr<uint8_t[]> val(new uint8_t[size_]); + reader.readFixed(&val[0], size_); + } + + protected: + + int size_; + +}; + +class FixedParser : public Resolver +{ + public: + + FixedParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets) : + Resolver() + { + size_ = writer->fixedSize(); + offset_ = offsets.at(0).offset(); + } + + virtual void parse(Reader &reader, uint8_t *address) const + { + DEBUG_OUT("Reading fixed"); + uint8_t *location = reinterpret_cast<uint8_t *> (address + offset_); + reader.readFixed(location, size_); + } + + protected: + + int size_; + size_t offset_; + +}; + + +class ResolverFactory : private boost::noncopyable { + + template<typename T> + unique_ptr<Resolver> + constructPrimitiveSkipper(const NodePtr &writer) + { + return unique_ptr<Resolver>(new PrimitiveSkipper<T>()); + } + + template<typename T> + unique_ptr<Resolver> + constructPrimitive(const NodePtr &writer, const NodePtr &reader, const Layout &offset) + { + unique_ptr<Resolver> instruction; + + SchemaResolution match = writer->resolve(*reader); + + if (match == RESOLVE_NO_MATCH) { + instruction = unique_ptr<Resolver>(new PrimitiveSkipper<T>()); + } + else if (reader->type() == AVRO_UNION) { + const CompoundLayout &compoundLayout = static_cast<const CompoundLayout &>(offset); + instruction = unique_ptr<Resolver>(new NonUnionToUnionParser(*this, writer, reader, compoundLayout)); + } + else if (match == RESOLVE_MATCH) { + const PrimitiveLayout &primitiveLayout = static_cast<const PrimitiveLayout &>(offset); + instruction = unique_ptr<Resolver>(new PrimitiveParser<T>(primitiveLayout)); + } + else if(match == RESOLVE_PROMOTABLE_TO_LONG) { + const PrimitiveLayout &primitiveLayout = static_cast<const PrimitiveLayout &>(offset); + instruction = unique_ptr<Resolver>(new PrimitivePromoter<T, int64_t>(primitiveLayout)); + } + else if(match == RESOLVE_PROMOTABLE_TO_FLOAT) { + const PrimitiveLayout &primitiveLayout = static_cast<const PrimitiveLayout &>(offset); + instruction = unique_ptr<Resolver>(new PrimitivePromoter<T, float>(primitiveLayout)); + } + else if(match == RESOLVE_PROMOTABLE_TO_DOUBLE) { + const PrimitiveLayout &primitiveLayout = static_cast<const PrimitiveLayout &>(offset); + instruction = unique_ptr<Resolver>(new PrimitivePromoter<T, double>(primitiveLayout)); + } + else { + assert(0); + } + return instruction; + } + + template<typename Skipper> + unique_ptr<Resolver> + constructCompoundSkipper(const NodePtr &writer) + { + return unique_ptr<Resolver>(new Skipper(*this, writer)); + } + + + template<typename Parser, typename Skipper> + unique_ptr<Resolver> + constructCompound(const NodePtr &writer, const NodePtr &reader, const Layout &offset) + { + unique_ptr<Resolver> instruction; + + SchemaResolution match = RESOLVE_NO_MATCH; + + match = writer->resolve(*reader); + + if (match == RESOLVE_NO_MATCH) { + instruction = unique_ptr<Resolver>(new Skipper(*this, writer)); + } + else if(writer->type() != AVRO_UNION && reader->type() == AVRO_UNION) { + const CompoundLayout &compoundLayout = dynamic_cast<const CompoundLayout &>(offset); + instruction = unique_ptr<Resolver>(new NonUnionToUnionParser(*this, writer, reader, compoundLayout)); + } + else if(writer->type() == AVRO_UNION && reader->type() != AVRO_UNION) { + instruction = unique_ptr<Resolver>(new UnionToNonUnionParser(*this, writer, reader, offset)); + } + else { + const CompoundLayout &compoundLayout = dynamic_cast<const CompoundLayout &>(offset); + instruction = unique_ptr<Resolver>(new Parser(*this, writer, reader, compoundLayout)); + } + + return instruction; + } + + public: + + unique_ptr<Resolver> + construct(const NodePtr &writer, const NodePtr &reader, const Layout &offset) + { + + typedef unique_ptr<Resolver> (ResolverFactory::*BuilderFunc)(const NodePtr &writer, const NodePtr &reader, const Layout &offset); + + NodePtr currentWriter = (writer->type() == AVRO_SYMBOLIC) ? + resolveSymbol(writer) : writer; + + NodePtr currentReader = (reader->type() == AVRO_SYMBOLIC) ? + resolveSymbol(reader) : reader; + + static const BuilderFunc funcs[] = { + &ResolverFactory::constructPrimitive<std::string>, + &ResolverFactory::constructPrimitive<std::vector<uint8_t> >, + &ResolverFactory::constructPrimitive<int32_t>, + &ResolverFactory::constructPrimitive<int64_t>, + &ResolverFactory::constructPrimitive<float>, + &ResolverFactory::constructPrimitive<double>, + &ResolverFactory::constructPrimitive<bool>, + &ResolverFactory::constructPrimitive<Null>, + &ResolverFactory::constructCompound<RecordParser, RecordSkipper>, + &ResolverFactory::constructCompound<EnumParser, EnumSkipper>, + &ResolverFactory::constructCompound<ArrayParser, ArraySkipper>, + &ResolverFactory::constructCompound<MapParser, MapSkipper>, + &ResolverFactory::constructCompound<UnionParser, UnionSkipper>, + &ResolverFactory::constructCompound<FixedParser, FixedSkipper> + }; + + static_assert((sizeof(funcs)/sizeof(BuilderFunc)) == (AVRO_NUM_TYPES), + "Invalid number of builder functions"); + + BuilderFunc func = funcs[currentWriter->type()]; + assert(func); + + return ((this)->*(func))(currentWriter, currentReader, offset); + } + + unique_ptr<Resolver> + skipper(const NodePtr &writer) + { + + typedef unique_ptr<Resolver> (ResolverFactory::*BuilderFunc)(const NodePtr &writer); + + NodePtr currentWriter = (writer->type() == AVRO_SYMBOLIC) ? + writer->leafAt(0) : writer; + + static const BuilderFunc funcs[] = { + &ResolverFactory::constructPrimitiveSkipper<std::string>, + &ResolverFactory::constructPrimitiveSkipper<std::vector<uint8_t> >, + &ResolverFactory::constructPrimitiveSkipper<int32_t>, + &ResolverFactory::constructPrimitiveSkipper<int64_t>, + &ResolverFactory::constructPrimitiveSkipper<float>, + &ResolverFactory::constructPrimitiveSkipper<double>, + &ResolverFactory::constructPrimitiveSkipper<bool>, + &ResolverFactory::constructPrimitiveSkipper<Null>, + &ResolverFactory::constructCompoundSkipper<RecordSkipper>, + &ResolverFactory::constructCompoundSkipper<EnumSkipper>, + &ResolverFactory::constructCompoundSkipper<ArraySkipper>, + &ResolverFactory::constructCompoundSkipper<MapSkipper>, + &ResolverFactory::constructCompoundSkipper<UnionSkipper>, + &ResolverFactory::constructCompoundSkipper<FixedSkipper> + }; + + static_assert((sizeof(funcs)/sizeof(BuilderFunc)) == (AVRO_NUM_TYPES), + "Invalid number of builder functions"); + + BuilderFunc func = funcs[currentWriter->type()]; + assert(func); + + return ((this)->*(func))(currentWriter); + } +}; + + +RecordSkipper::RecordSkipper(ResolverFactory &factory, const NodePtr &writer) : + Resolver() +{ + size_t leaves = writer->leaves(); + resolvers_.reserve(leaves); + for(size_t i = 0; i < leaves; ++i) { + const NodePtr &w = writer->leafAt(i); + resolvers_.push_back(factory.skipper(w)); + } +} + +RecordParser::RecordParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets) : + Resolver() +{ + size_t leaves = writer->leaves(); + resolvers_.reserve(leaves); + for(size_t i = 0; i < leaves; ++i) { + + const NodePtr &w = writer->leafAt(i); + + const std::string &name = writer->nameAt(i); + + size_t readerIndex = 0; + bool found = reader->nameIndex(name, readerIndex); + + if(found) { + const NodePtr &r = reader->leafAt(readerIndex); + resolvers_.push_back(factory.construct(w, r, offsets.at(readerIndex))); + } + else { + resolvers_.push_back(factory.skipper(w)); + } + } +} + +MapSkipper::MapSkipper(ResolverFactory &factory, const NodePtr &writer) : + Resolver(), + resolver_(factory.skipper(writer->leafAt(1))) +{ } + +MapParser::MapParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets) : + Resolver(), + resolver_(factory.construct(writer->leafAt(1), reader->leafAt(1), offsets.at(1))), + offset_(offsets.offset()), + setFuncOffset_( offsets.at(0).offset()) +{ } + +ArraySkipper::ArraySkipper(ResolverFactory &factory, const NodePtr &writer) : + Resolver(), + resolver_(factory.skipper(writer->leafAt(0))) +{ } + +ArrayParser::ArrayParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets) : + Resolver(), + resolver_(factory.construct(writer->leafAt(0), reader->leafAt(0), offsets.at(1))), + offset_(offsets.offset()), + setFuncOffset_(offsets.at(0).offset()) +{ } + +UnionSkipper::UnionSkipper(ResolverFactory &factory, const NodePtr &writer) : + Resolver() +{ + size_t leaves = writer->leaves(); + resolvers_.reserve(leaves); + for(size_t i = 0; i < leaves; ++i) { + const NodePtr &w = writer->leafAt(i); + resolvers_.push_back(factory.skipper(w)); + } +} + +namespace { + +// asumes the writer is NOT a union, and the reader IS a union + +SchemaResolution +checkUnionMatch(const NodePtr &writer, const NodePtr &reader, size_t &index) +{ + SchemaResolution bestMatch = RESOLVE_NO_MATCH; + + index = 0; + size_t leaves = reader->leaves(); + + for(size_t i=0; i < leaves; ++i) { + + const NodePtr &leaf = reader->leafAt(i); + SchemaResolution newMatch = writer->resolve(*leaf); + + if(newMatch == RESOLVE_MATCH) { + bestMatch = newMatch; + index = i; + break; + } + if(bestMatch == RESOLVE_NO_MATCH) { + bestMatch = newMatch; + index = i; + } + } + + return bestMatch; +} + +}; + +UnionParser::UnionParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets) : + Resolver(), + offset_(offsets.offset()), + choiceOffset_(offsets.at(0).offset()), + setFuncOffset_(offsets.at(1).offset()) +{ + + size_t leaves = writer->leaves(); + resolvers_.reserve(leaves); + choiceMapping_.reserve(leaves); + for(size_t i = 0; i < leaves; ++i) { + + // for each writer, we need a schema match for the reader + const NodePtr &w = writer->leafAt(i); + size_t index = 0; + + SchemaResolution match = checkUnionMatch(w, reader, index); + + if(match == RESOLVE_NO_MATCH) { + resolvers_.push_back(factory.skipper(w)); + // push back a non-sensical number + choiceMapping_.push_back(reader->leaves()); + } + else { + const NodePtr &r = reader->leafAt(index); + resolvers_.push_back(factory.construct(w, r, offsets.at(index+2))); + choiceMapping_.push_back(index); + } + } +} + +NonUnionToUnionParser::NonUnionToUnionParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const CompoundLayout &offsets) : + Resolver(), + offset_(offsets.offset()), + choiceOffset_(offsets.at(0).offset()), + setFuncOffset_(offsets.at(1).offset()) +{ +#ifndef NDEBUG + SchemaResolution bestMatch = +#endif + checkUnionMatch(writer, reader, choice_); + assert(bestMatch != RESOLVE_NO_MATCH); + resolver_ = factory.construct(writer, reader->leafAt(choice_), offsets.at(choice_+2)); +} + +UnionToNonUnionParser::UnionToNonUnionParser(ResolverFactory &factory, const NodePtr &writer, const NodePtr &reader, const Layout &offsets) : + Resolver() +{ + size_t leaves = writer->leaves(); + resolvers_.reserve(leaves); + for(size_t i = 0; i < leaves; ++i) { + const NodePtr &w = writer->leafAt(i); + resolvers_.push_back(factory.construct(w, reader, offsets)); + } +} + +unique_ptr<Resolver> constructResolver(const ValidSchema &writerSchema, + const ValidSchema &readerSchema, + const Layout &readerLayout) +{ + ResolverFactory factory; + return factory.construct(writerSchema.root(), readerSchema.root(), readerLayout); +} + +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/ResolverSchema.cc b/contrib/libs/apache/avro/impl/ResolverSchema.cc index 6ce8592559..f42946d692 100644 --- a/contrib/libs/apache/avro/impl/ResolverSchema.cc +++ b/contrib/libs/apache/avro/impl/ResolverSchema.cc @@ -1,39 +1,39 @@ - -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ResolverSchema.hh" -#include "Resolver.hh" -#include "ValidSchema.hh" - -namespace avro { - -ResolverSchema::ResolverSchema( - const ValidSchema &writerSchema, - const ValidSchema &readerSchema, - const Layout &readerLayout) : - resolver_(constructResolver(writerSchema, readerSchema, readerLayout)) -{ } - -void -ResolverSchema::parse(Reader &reader, uint8_t *address) -{ - resolver_->parse(reader, address); -} - -} // namespace avro + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ResolverSchema.hh" +#include "Resolver.hh" +#include "ValidSchema.hh" + +namespace avro { + +ResolverSchema::ResolverSchema( + const ValidSchema &writerSchema, + const ValidSchema &readerSchema, + const Layout &readerLayout) : + resolver_(constructResolver(writerSchema, readerSchema, readerLayout)) +{ } + +void +ResolverSchema::parse(Reader &reader, uint8_t *address) +{ + resolver_->parse(reader, address); +} + +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/Schema.cc b/contrib/libs/apache/avro/impl/Schema.cc index 077212895b..e6cfa45c2e 100644 --- a/contrib/libs/apache/avro/impl/Schema.cc +++ b/contrib/libs/apache/avro/impl/Schema.cc @@ -1,139 +1,139 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include "Schema.hh" - -namespace avro { - -Schema::Schema() -{ } - -Schema::~Schema() -{ } - -Schema::Schema(const NodePtr &node) : - node_(node) -{ } - -Schema::Schema(Node *node) : - node_(node) -{ } - -RecordSchema::RecordSchema(const std::string &name) : - Schema(new NodeRecord) -{ - node_->setName(name); -} - -void -RecordSchema::addField(const std::string &name, const Schema &fieldSchema) -{ - // add the name first. it will throw if the name is a duplicate, preventing - // the leaf from being added - node_->addName(name); - - node_->addLeaf(fieldSchema.root()); -} - -std::string RecordSchema::getDoc() const -{ - return node_->getDoc(); -} -void RecordSchema::setDoc(const std::string& doc) -{ - node_->setDoc(doc); -} - -EnumSchema::EnumSchema(const std::string &name) : - Schema(new NodeEnum) -{ - node_->setName(name); -} - -void -EnumSchema::addSymbol(const std::string &symbol) -{ - node_->addName(symbol); -} - -ArraySchema::ArraySchema(const Schema &itemsSchema) : - Schema(new NodeArray) -{ - node_->addLeaf(itemsSchema.root()); -} - -ArraySchema::ArraySchema(const ArraySchema &itemsSchema) : - Schema(new NodeArray) -{ - node_->addLeaf(itemsSchema.root()); -} - -MapSchema::MapSchema(const Schema &valuesSchema) : - Schema(new NodeMap) -{ - node_->addLeaf(valuesSchema.root()); -} - -MapSchema::MapSchema(const MapSchema &valuesSchema) : - Schema(new NodeMap) -{ - node_->addLeaf(valuesSchema.root()); -} - -UnionSchema::UnionSchema() : - Schema(new NodeUnion) -{ } - -void -UnionSchema::addType(const Schema &typeSchema) -{ - if(typeSchema.type() == AVRO_UNION) { - throw Exception("Cannot add unions to unions"); - } - - if(typeSchema.type() == AVRO_RECORD) { - // check for duplicate records - size_t types = node_->leaves(); - for(size_t i = 0; i < types; ++i) { - const NodePtr &leaf = node_->leafAt(i); - // TODO, more checks? - if(leaf->type() == AVRO_RECORD && leaf->name() == typeSchema.root()->name()) { - throw Exception("Records in unions cannot have duplicate names"); - } - } - } - - node_->addLeaf(typeSchema.root()); -} - -FixedSchema::FixedSchema(int size, const std::string &name) : - Schema(new NodeFixed) -{ - node_->setFixedSize(size); - node_->setName(name); -} - -SymbolicSchema::SymbolicSchema(const Name &name, const NodePtr& link) : - Schema(new NodeSymbolic(HasName(name), link)) -{ -} - - - -} // namespace avro +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include "Schema.hh" + +namespace avro { + +Schema::Schema() +{ } + +Schema::~Schema() +{ } + +Schema::Schema(const NodePtr &node) : + node_(node) +{ } + +Schema::Schema(Node *node) : + node_(node) +{ } + +RecordSchema::RecordSchema(const std::string &name) : + Schema(new NodeRecord) +{ + node_->setName(name); +} + +void +RecordSchema::addField(const std::string &name, const Schema &fieldSchema) +{ + // add the name first. it will throw if the name is a duplicate, preventing + // the leaf from being added + node_->addName(name); + + node_->addLeaf(fieldSchema.root()); +} + +std::string RecordSchema::getDoc() const +{ + return node_->getDoc(); +} +void RecordSchema::setDoc(const std::string& doc) +{ + node_->setDoc(doc); +} + +EnumSchema::EnumSchema(const std::string &name) : + Schema(new NodeEnum) +{ + node_->setName(name); +} + +void +EnumSchema::addSymbol(const std::string &symbol) +{ + node_->addName(symbol); +} + +ArraySchema::ArraySchema(const Schema &itemsSchema) : + Schema(new NodeArray) +{ + node_->addLeaf(itemsSchema.root()); +} + +ArraySchema::ArraySchema(const ArraySchema &itemsSchema) : + Schema(new NodeArray) +{ + node_->addLeaf(itemsSchema.root()); +} + +MapSchema::MapSchema(const Schema &valuesSchema) : + Schema(new NodeMap) +{ + node_->addLeaf(valuesSchema.root()); +} + +MapSchema::MapSchema(const MapSchema &valuesSchema) : + Schema(new NodeMap) +{ + node_->addLeaf(valuesSchema.root()); +} + +UnionSchema::UnionSchema() : + Schema(new NodeUnion) +{ } + +void +UnionSchema::addType(const Schema &typeSchema) +{ + if(typeSchema.type() == AVRO_UNION) { + throw Exception("Cannot add unions to unions"); + } + + if(typeSchema.type() == AVRO_RECORD) { + // check for duplicate records + size_t types = node_->leaves(); + for(size_t i = 0; i < types; ++i) { + const NodePtr &leaf = node_->leafAt(i); + // TODO, more checks? + if(leaf->type() == AVRO_RECORD && leaf->name() == typeSchema.root()->name()) { + throw Exception("Records in unions cannot have duplicate names"); + } + } + } + + node_->addLeaf(typeSchema.root()); +} + +FixedSchema::FixedSchema(int size, const std::string &name) : + Schema(new NodeFixed) +{ + node_->setFixedSize(size); + node_->setName(name); +} + +SymbolicSchema::SymbolicSchema(const Name &name, const NodePtr& link) : + Schema(new NodeSymbolic(HasName(name), link)) +{ +} + + + +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/Stream.cc b/contrib/libs/apache/avro/impl/Stream.cc index ab7849e5b6..7023f3f213 100644 --- a/contrib/libs/apache/avro/impl/Stream.cc +++ b/contrib/libs/apache/avro/impl/Stream.cc @@ -1,198 +1,198 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "Stream.hh" -#include <vector> - -namespace avro { - -using std::vector; - -class MemoryInputStream : public InputStream { - const std::vector<uint8_t*>& data_; - const size_t chunkSize_; - const size_t size_; - const size_t available_; - size_t cur_; - size_t curLen_; - - size_t maxLen() { - size_t n = (cur_ == (size_ - 1)) ? available_ : chunkSize_; - if (n == curLen_) { - if (cur_ == (size_ - 1)) { - return 0; - } - ++cur_; - n = (cur_ == (size_ - 1)) ? available_ : chunkSize_; - curLen_ = 0; - } - return n; - } - -public: - MemoryInputStream(const std::vector<uint8_t*>& b, - size_t chunkSize, size_t available) : - data_(b), chunkSize_(chunkSize), size_(b.size()), - available_(available), cur_(0), curLen_(0) { } - - bool next(const uint8_t** data, size_t* len) { - if (size_t n = maxLen()) { - *data = data_[cur_] + curLen_; - *len = n - curLen_; - curLen_ = n; - return true; - } - return false; - } - - void backup(size_t len) { - curLen_ -= len; - } - - void skip(size_t len) { - while (len > 0) { - if (size_t n = maxLen()) { - if ((curLen_ + len) < n) { - n = curLen_ + len; - } - len -= n - curLen_; - curLen_ = n; - } else { - break; - } - } - } - - size_t byteCount() const { - return cur_ * chunkSize_ + curLen_; - } -}; - -class MemoryInputStream2 : public InputStream { - const uint8_t* const data_; - const size_t size_; - size_t curLen_; -public: - MemoryInputStream2(const uint8_t *data, size_t len) - : data_(data), size_(len), curLen_(0) { } - - bool next(const uint8_t** data, size_t* len) { - if (curLen_ == size_) { - return false; - } - *data = &data_[curLen_]; - *len = size_ - curLen_; - curLen_ = size_; - return true; - } - - void backup(size_t len) { - curLen_ -= len; - } - - void skip(size_t len) { - if (len > (size_ - curLen_)) { - len = size_ - curLen_; - } - curLen_ += len; - } - - size_t byteCount() const { - return curLen_; - } -}; - -class MemoryOutputStream : public OutputStream { -public: - const size_t chunkSize_; - std::vector<uint8_t*> data_; - size_t available_; - size_t byteCount_; - - MemoryOutputStream(size_t chunkSize) : chunkSize_(chunkSize), - available_(0), byteCount_(0) { } - ~MemoryOutputStream() { - for (std::vector<uint8_t*>::const_iterator it = data_.begin(); - it != data_.end(); ++it) { - delete[] *it; - } - } - - bool next(uint8_t** data, size_t* len) { - if (available_ == 0) { - data_.push_back(new uint8_t[chunkSize_]); - available_ = chunkSize_; - } - *data = &data_.back()[chunkSize_ - available_]; - *len = available_; - byteCount_ += available_; - available_ = 0; - return true; - } - - void backup(size_t len) { - available_ += len; - byteCount_ -= len; - } - - uint64_t byteCount() const { - return byteCount_; - } - - void flush() { } -}; - -std::unique_ptr<OutputStream> memoryOutputStream(size_t chunkSize) -{ - return std::unique_ptr<OutputStream>(new MemoryOutputStream(chunkSize)); -} - -std::unique_ptr<InputStream> memoryInputStream(const uint8_t* data, size_t len) -{ - return std::unique_ptr<InputStream>(new MemoryInputStream2(data, len)); -} - -std::unique_ptr<InputStream> memoryInputStream(const OutputStream& source) -{ - const MemoryOutputStream& mos = - dynamic_cast<const MemoryOutputStream&>(source); - return (mos.data_.empty()) ? - std::unique_ptr<InputStream>(new MemoryInputStream2(0, 0)) : - std::unique_ptr<InputStream>(new MemoryInputStream(mos.data_, - mos.chunkSize_, - (mos.chunkSize_ - mos.available_))); -} - -std::shared_ptr<std::vector<uint8_t> > snapshot(const OutputStream& source) -{ - const MemoryOutputStream& mos = - dynamic_cast<const MemoryOutputStream&>(source); - std::shared_ptr<std::vector<uint8_t> > result(new std::vector<uint8_t>()); - size_t c = mos.byteCount_; - result->reserve(mos.byteCount_); - for (vector<uint8_t*>::const_iterator it = mos.data_.begin(); - it != mos.data_.end(); ++it) { - size_t n = std::min(c, mos.chunkSize_); - std::copy(*it, *it + n, std::back_inserter(*result)); - c -= n; - } - return result; -} - -} // namespace avro - +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Stream.hh" +#include <vector> + +namespace avro { + +using std::vector; + +class MemoryInputStream : public InputStream { + const std::vector<uint8_t*>& data_; + const size_t chunkSize_; + const size_t size_; + const size_t available_; + size_t cur_; + size_t curLen_; + + size_t maxLen() { + size_t n = (cur_ == (size_ - 1)) ? available_ : chunkSize_; + if (n == curLen_) { + if (cur_ == (size_ - 1)) { + return 0; + } + ++cur_; + n = (cur_ == (size_ - 1)) ? available_ : chunkSize_; + curLen_ = 0; + } + return n; + } + +public: + MemoryInputStream(const std::vector<uint8_t*>& b, + size_t chunkSize, size_t available) : + data_(b), chunkSize_(chunkSize), size_(b.size()), + available_(available), cur_(0), curLen_(0) { } + + bool next(const uint8_t** data, size_t* len) { + if (size_t n = maxLen()) { + *data = data_[cur_] + curLen_; + *len = n - curLen_; + curLen_ = n; + return true; + } + return false; + } + + void backup(size_t len) { + curLen_ -= len; + } + + void skip(size_t len) { + while (len > 0) { + if (size_t n = maxLen()) { + if ((curLen_ + len) < n) { + n = curLen_ + len; + } + len -= n - curLen_; + curLen_ = n; + } else { + break; + } + } + } + + size_t byteCount() const { + return cur_ * chunkSize_ + curLen_; + } +}; + +class MemoryInputStream2 : public InputStream { + const uint8_t* const data_; + const size_t size_; + size_t curLen_; +public: + MemoryInputStream2(const uint8_t *data, size_t len) + : data_(data), size_(len), curLen_(0) { } + + bool next(const uint8_t** data, size_t* len) { + if (curLen_ == size_) { + return false; + } + *data = &data_[curLen_]; + *len = size_ - curLen_; + curLen_ = size_; + return true; + } + + void backup(size_t len) { + curLen_ -= len; + } + + void skip(size_t len) { + if (len > (size_ - curLen_)) { + len = size_ - curLen_; + } + curLen_ += len; + } + + size_t byteCount() const { + return curLen_; + } +}; + +class MemoryOutputStream : public OutputStream { +public: + const size_t chunkSize_; + std::vector<uint8_t*> data_; + size_t available_; + size_t byteCount_; + + MemoryOutputStream(size_t chunkSize) : chunkSize_(chunkSize), + available_(0), byteCount_(0) { } + ~MemoryOutputStream() { + for (std::vector<uint8_t*>::const_iterator it = data_.begin(); + it != data_.end(); ++it) { + delete[] *it; + } + } + + bool next(uint8_t** data, size_t* len) { + if (available_ == 0) { + data_.push_back(new uint8_t[chunkSize_]); + available_ = chunkSize_; + } + *data = &data_.back()[chunkSize_ - available_]; + *len = available_; + byteCount_ += available_; + available_ = 0; + return true; + } + + void backup(size_t len) { + available_ += len; + byteCount_ -= len; + } + + uint64_t byteCount() const { + return byteCount_; + } + + void flush() { } +}; + +std::unique_ptr<OutputStream> memoryOutputStream(size_t chunkSize) +{ + return std::unique_ptr<OutputStream>(new MemoryOutputStream(chunkSize)); +} + +std::unique_ptr<InputStream> memoryInputStream(const uint8_t* data, size_t len) +{ + return std::unique_ptr<InputStream>(new MemoryInputStream2(data, len)); +} + +std::unique_ptr<InputStream> memoryInputStream(const OutputStream& source) +{ + const MemoryOutputStream& mos = + dynamic_cast<const MemoryOutputStream&>(source); + return (mos.data_.empty()) ? + std::unique_ptr<InputStream>(new MemoryInputStream2(0, 0)) : + std::unique_ptr<InputStream>(new MemoryInputStream(mos.data_, + mos.chunkSize_, + (mos.chunkSize_ - mos.available_))); +} + +std::shared_ptr<std::vector<uint8_t> > snapshot(const OutputStream& source) +{ + const MemoryOutputStream& mos = + dynamic_cast<const MemoryOutputStream&>(source); + std::shared_ptr<std::vector<uint8_t> > result(new std::vector<uint8_t>()); + size_t c = mos.byteCount_; + result->reserve(mos.byteCount_); + for (vector<uint8_t*>::const_iterator it = mos.data_.begin(); + it != mos.data_.end(); ++it) { + size_t n = std::min(c, mos.chunkSize_); + std::copy(*it, *it + n, std::back_inserter(*result)); + c -= n; + } + return result; +} + +} // namespace avro + diff --git a/contrib/libs/apache/avro/impl/Types.cc b/contrib/libs/apache/avro/impl/Types.cc index e7485fe8ac..6c9d702b83 100644 --- a/contrib/libs/apache/avro/impl/Types.cc +++ b/contrib/libs/apache/avro/impl/Types.cc @@ -1,82 +1,82 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <iostream> -#include <string> -#include "Types.hh" - -namespace avro { - -namespace strings { -const std::string typeToString[] = { - "string", - "bytes", - "int", - "long", - "float", - "double", - "boolean", - "null", - "record", - "enum", - "array", - "map", - "union", - "fixed", - "symbolic" -}; - -static_assert((sizeof(typeToString)/sizeof(std::string)) == (AVRO_NUM_TYPES+1), - "Incorrect Avro typeToString"); - -} // namespace strings - - -// this static assert exists because a 32 bit integer is used as a bit-flag for each type, -// and it would be a problem for this flag if we ever supported more than 32 types -static_assert(AVRO_NUM_TYPES < 32, "Too many Avro types"); - -const std::string& toString(Type type) -{ - static std::string undefinedType = "Undefined type"; - if (isAvroTypeOrPseudoType(type)) { - return strings::typeToString[type]; - } else { - return undefinedType; - } -} - -std::ostream &operator<< (std::ostream &os, Type type) -{ - if(isAvroTypeOrPseudoType(type)) { - os << strings::typeToString[type]; - } - else { - os << static_cast<int>(type); - } - return os; -} - -std::ostream &operator<< (std::ostream &os, const Null &) -{ - os << "(null value)"; - return os; -} - -} // namespace avro - +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <iostream> +#include <string> +#include "Types.hh" + +namespace avro { + +namespace strings { +const std::string typeToString[] = { + "string", + "bytes", + "int", + "long", + "float", + "double", + "boolean", + "null", + "record", + "enum", + "array", + "map", + "union", + "fixed", + "symbolic" +}; + +static_assert((sizeof(typeToString)/sizeof(std::string)) == (AVRO_NUM_TYPES+1), + "Incorrect Avro typeToString"); + +} // namespace strings + + +// this static assert exists because a 32 bit integer is used as a bit-flag for each type, +// and it would be a problem for this flag if we ever supported more than 32 types +static_assert(AVRO_NUM_TYPES < 32, "Too many Avro types"); + +const std::string& toString(Type type) +{ + static std::string undefinedType = "Undefined type"; + if (isAvroTypeOrPseudoType(type)) { + return strings::typeToString[type]; + } else { + return undefinedType; + } +} + +std::ostream &operator<< (std::ostream &os, Type type) +{ + if(isAvroTypeOrPseudoType(type)) { + os << strings::typeToString[type]; + } + else { + os << static_cast<int>(type); + } + return os; +} + +std::ostream &operator<< (std::ostream &os, const Null &) +{ + os << "(null value)"; + return os; +} + +} // namespace avro + diff --git a/contrib/libs/apache/avro/impl/ValidSchema.cc b/contrib/libs/apache/avro/impl/ValidSchema.cc index 6e9762ccba..74a3f845c5 100644 --- a/contrib/libs/apache/avro/impl/ValidSchema.cc +++ b/contrib/libs/apache/avro/impl/ValidSchema.cc @@ -1,193 +1,193 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <boost/format.hpp> -#include <cctype> -#include <sstream> - -#include "ValidSchema.hh" -#include "Schema.hh" -#include "Node.hh" - -using std::string; -using std::ostringstream; -using std::make_pair; -using boost::format; -using std::shared_ptr; -using std::static_pointer_cast; - -namespace avro { -typedef std::map<Name, NodePtr> SymbolMap; - -static bool validate(const NodePtr &node, SymbolMap &symbolMap) -{ - if (! node->isValid()) { - throw Exception(format("Schema is invalid, due to bad node of type %1%") - % node->type()); - } - - if (node->hasName()) { - const Name& nm = node->name(); - SymbolMap::iterator it = symbolMap.find(nm); - bool found = it != symbolMap.end() && nm == it->first; - - if (node->type() == AVRO_SYMBOLIC) { - if (! found) { - throw Exception(format("Symbolic name \"%1%\" is unknown") % - node->name()); - } - - shared_ptr<NodeSymbolic> symNode = - static_pointer_cast<NodeSymbolic>(node); - - // if the symbolic link is already resolved, we return true, - // otherwise returning false will force it to be resolved - return symNode->isSet(); - } - - if (found) { - return false; - } - symbolMap.insert(it, make_pair(nm, node)); - } - - node->lock(); - size_t leaves = node->leaves(); - for (size_t i = 0; i < leaves; ++i) { - const NodePtr &leaf(node->leafAt(i)); - - if (! validate(leaf, symbolMap)) { - - // if validate returns false it means a node with this name already - // existed in the map, instead of keeping this node twice in the - // map (which could potentially create circular shared pointer - // links that could not be easily freed), replace this node with a - // symbolic link to the original one. - - node->setLeafToSymbolic(i, symbolMap.find(leaf->name())->second); - } - } - - return true; -} - -static void validate(const NodePtr& p) -{ - SymbolMap m; - validate(p, m); -} - -ValidSchema::ValidSchema(const NodePtr &root) : root_(root) -{ - validate(root_); -} - -ValidSchema::ValidSchema(const Schema &schema) : root_(schema.root()) -{ - validate(root_); -} - -ValidSchema::ValidSchema() : root_(NullSchema().root()) -{ - validate(root_); -} - -void -ValidSchema::setSchema(const Schema &schema) -{ - root_ = schema.root(); - validate(root_); -} - -void -ValidSchema::toJson(std::ostream &os) const -{ - root_->printJson(os, 0); - os << '\n'; -} - -string -ValidSchema::toJson(bool prettyPrint) const -{ - ostringstream oss; - toJson(oss); - if (!prettyPrint) { - return compactSchema(oss.str()); - } - return oss.str(); -} - -void -ValidSchema::toFlatList(std::ostream &os) const -{ - root_->printBasicInfo(os); -} - -/* - * compactSchema compacts and returns a formatted string representation - * of a ValidSchema object by removing the whitespaces outside of the quoted - * field names and values. It can handle the cases where the quoted value is - * in UTF-8 format. Note that this method is not responsible for validating - * the schema. - */ -string ValidSchema::compactSchema(const string& schema) { - bool insideQuote = false; - size_t newPos = 0; - string data(schema.data()); - - for (size_t currentPos = 0; currentPos < schema.size(); currentPos++) { - if (!insideQuote && std::isspace(data[currentPos])) { - // Skip the white spaces outside quotes. - continue; - } - - if (data[currentPos] == '\"') { - // It is valid for a quote to be part of the value for some fields, - // e.g., the "doc" field. In that case, the quote is expected to be - // escaped inside the schema. Since the escape character '\\' could - // be escaped itself, we need to check whether there are an even - // number of consecutive slashes prior to the quote. - int leadingSlashes = 0; - for (int i = newPos - 1; i >= 0; i--) { - if (data[i] == '\\') { - leadingSlashes++; - } else { - break; - } - } - if (leadingSlashes % 2 == 0) { - // Found a real quote which identifies either the start or the - // end of a field name or value. - insideQuote = !insideQuote; - } - } - data[newPos++] = data[currentPos]; - } - - if (insideQuote) { - throw Exception("Schema is not well formed with mismatched quotes"); - } - - if (newPos < schema.size()) { - data.resize(newPos); - } - return data; -} - -} // namespace avro - +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <boost/format.hpp> +#include <cctype> +#include <sstream> + +#include "ValidSchema.hh" +#include "Schema.hh" +#include "Node.hh" + +using std::string; +using std::ostringstream; +using std::make_pair; +using boost::format; +using std::shared_ptr; +using std::static_pointer_cast; + +namespace avro { +typedef std::map<Name, NodePtr> SymbolMap; + +static bool validate(const NodePtr &node, SymbolMap &symbolMap) +{ + if (! node->isValid()) { + throw Exception(format("Schema is invalid, due to bad node of type %1%") + % node->type()); + } + + if (node->hasName()) { + const Name& nm = node->name(); + SymbolMap::iterator it = symbolMap.find(nm); + bool found = it != symbolMap.end() && nm == it->first; + + if (node->type() == AVRO_SYMBOLIC) { + if (! found) { + throw Exception(format("Symbolic name \"%1%\" is unknown") % + node->name()); + } + + shared_ptr<NodeSymbolic> symNode = + static_pointer_cast<NodeSymbolic>(node); + + // if the symbolic link is already resolved, we return true, + // otherwise returning false will force it to be resolved + return symNode->isSet(); + } + + if (found) { + return false; + } + symbolMap.insert(it, make_pair(nm, node)); + } + + node->lock(); + size_t leaves = node->leaves(); + for (size_t i = 0; i < leaves; ++i) { + const NodePtr &leaf(node->leafAt(i)); + + if (! validate(leaf, symbolMap)) { + + // if validate returns false it means a node with this name already + // existed in the map, instead of keeping this node twice in the + // map (which could potentially create circular shared pointer + // links that could not be easily freed), replace this node with a + // symbolic link to the original one. + + node->setLeafToSymbolic(i, symbolMap.find(leaf->name())->second); + } + } + + return true; +} + +static void validate(const NodePtr& p) +{ + SymbolMap m; + validate(p, m); +} + +ValidSchema::ValidSchema(const NodePtr &root) : root_(root) +{ + validate(root_); +} + +ValidSchema::ValidSchema(const Schema &schema) : root_(schema.root()) +{ + validate(root_); +} + +ValidSchema::ValidSchema() : root_(NullSchema().root()) +{ + validate(root_); +} + +void +ValidSchema::setSchema(const Schema &schema) +{ + root_ = schema.root(); + validate(root_); +} + +void +ValidSchema::toJson(std::ostream &os) const +{ + root_->printJson(os, 0); + os << '\n'; +} + +string +ValidSchema::toJson(bool prettyPrint) const +{ + ostringstream oss; + toJson(oss); + if (!prettyPrint) { + return compactSchema(oss.str()); + } + return oss.str(); +} + +void +ValidSchema::toFlatList(std::ostream &os) const +{ + root_->printBasicInfo(os); +} + +/* + * compactSchema compacts and returns a formatted string representation + * of a ValidSchema object by removing the whitespaces outside of the quoted + * field names and values. It can handle the cases where the quoted value is + * in UTF-8 format. Note that this method is not responsible for validating + * the schema. + */ +string ValidSchema::compactSchema(const string& schema) { + bool insideQuote = false; + size_t newPos = 0; + string data(schema.data()); + + for (size_t currentPos = 0; currentPos < schema.size(); currentPos++) { + if (!insideQuote && std::isspace(data[currentPos])) { + // Skip the white spaces outside quotes. + continue; + } + + if (data[currentPos] == '\"') { + // It is valid for a quote to be part of the value for some fields, + // e.g., the "doc" field. In that case, the quote is expected to be + // escaped inside the schema. Since the escape character '\\' could + // be escaped itself, we need to check whether there are an even + // number of consecutive slashes prior to the quote. + int leadingSlashes = 0; + for (int i = newPos - 1; i >= 0; i--) { + if (data[i] == '\\') { + leadingSlashes++; + } else { + break; + } + } + if (leadingSlashes % 2 == 0) { + // Found a real quote which identifies either the start or the + // end of a field name or value. + insideQuote = !insideQuote; + } + } + data[newPos++] = data[currentPos]; + } + + if (insideQuote) { + throw Exception("Schema is not well formed with mismatched quotes"); + } + + if (newPos < schema.size()) { + data.resize(newPos); + } + return data; +} + +} // namespace avro + diff --git a/contrib/libs/apache/avro/impl/Validator.cc b/contrib/libs/apache/avro/impl/Validator.cc index 139081693d..2e74b06b66 100644 --- a/contrib/libs/apache/avro/impl/Validator.cc +++ b/contrib/libs/apache/avro/impl/Validator.cc @@ -1,301 +1,301 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "Validator.hh" -#include "ValidSchema.hh" -#include "NodeImpl.hh" - -namespace avro { - -Validator::Validator(const ValidSchema &schema) : - schema_(schema), - nextType_(AVRO_NULL), - expectedTypesFlag_(0), - compoundStarted_(false), - waitingForCount_(false), - count_(0) -{ - setupOperation(schema_.root()); -} - -void -Validator::setWaitingForCount() -{ - waitingForCount_ = true; - count_ = 0; - expectedTypesFlag_ = typeToFlag(AVRO_INT) | typeToFlag(AVRO_LONG); - nextType_ = AVRO_LONG; -} - -void -Validator::enumAdvance() -{ - if(compoundStarted_) { - setWaitingForCount(); - compoundStarted_ = false; - } - else { - waitingForCount_ = false; - compoundStack_.pop_back(); - } -} - -bool -Validator::countingSetup() -{ - bool proceed = true; - if(compoundStarted_) { - setWaitingForCount(); - compoundStarted_ = false; - proceed = false; - } - else if(waitingForCount_) { - waitingForCount_ = false; - if(count_ == 0) { - compoundStack_.pop_back(); - proceed = false; - } - else { - counters_.push_back(static_cast<size_t>(count_)); - } - } - - return proceed; -} - -void -Validator::countingAdvance() -{ - if(countingSetup()) { - - size_t index = (compoundStack_.back().pos)++; - const NodePtr &node = compoundStack_.back().node; - - if(index < node->leaves() ) { - setupOperation(node->leafAt(index)); - } - else { - compoundStack_.back().pos = 0; - int count = --counters_.back(); - if(count == 0) { - counters_.pop_back(); - compoundStarted_ = true; - nextType_ = node->type(); - expectedTypesFlag_ = typeToFlag(nextType_); - } - else { - size_t index = (compoundStack_.back().pos)++; - setupOperation(node->leafAt(index)); - } - } - } -} - -void -Validator::unionAdvance() -{ - if(compoundStarted_) { - setWaitingForCount(); - compoundStarted_ = false; - } - else { - waitingForCount_ = false; - NodePtr node = compoundStack_.back().node; - - if(count_ < static_cast<int64_t>(node->leaves())) { - compoundStack_.pop_back(); - setupOperation(node->leafAt(static_cast<int>(count_))); - } - else { - throw Exception( - boost::format("Union selection out of range, got %1%," \ - " expecting 0-%2%") - % count_ % (node->leaves() -1) - ); - } - } -} - -void -Validator::fixedAdvance() -{ - compoundStarted_ = false; - compoundStack_.pop_back(); -} - -int -Validator::nextSizeExpected() const -{ - return compoundStack_.back().node->fixedSize(); -} - -void -Validator::doAdvance() -{ - typedef void (Validator::*AdvanceFunc)(); - - // only the compound types need advance functions here - static const AdvanceFunc funcs[] = { - 0, // string - 0, // bytes - 0, // int - 0, // long - 0, // float - 0, // double - 0, // bool - 0, // null - &Validator::countingAdvance, // Record is treated like counting with count == 1 - &Validator::enumAdvance, - &Validator::countingAdvance, - &Validator::countingAdvance, - &Validator::unionAdvance, - &Validator::fixedAdvance - }; - static_assert((sizeof(funcs)/sizeof(AdvanceFunc)) == (AVRO_NUM_TYPES), - "Invalid number of advance functions"); - - expectedTypesFlag_ = 0; - // loop until we encounter a next expected type, or we've exited all compound types - while(!expectedTypesFlag_ && !compoundStack_.empty() ) { - - Type type = compoundStack_.back().node->type(); - - AdvanceFunc func = funcs[type]; - - // only compound functions are put on the status stack so it is ok to - // assume that func is not null - assert(func); - - ((this)->*(func))(); - } - - if(compoundStack_.empty()) { - nextType_ = AVRO_NULL; - } -} - -void Validator::advance() -{ - if(!waitingForCount_) { - doAdvance(); - } -} - -void -Validator::setCount(int64_t count) -{ - if(!waitingForCount_) { - throw Exception("Not expecting count"); - } - else if(count_ < 0) { - throw Exception("Count cannot be negative"); - } - count_ = count; - - doAdvance(); -} - -void -Validator::setupFlag(Type type) -{ - // use flags instead of strictly types, so that we can be more lax about the type - // (for example, a long should be able to accept an int type, but not vice versa) - static const flag_t flags[] = { - typeToFlag(AVRO_STRING) | typeToFlag(AVRO_BYTES), - typeToFlag(AVRO_STRING) | typeToFlag(AVRO_BYTES), - typeToFlag(AVRO_INT), - typeToFlag(AVRO_INT) | typeToFlag(AVRO_LONG), - typeToFlag(AVRO_FLOAT), - typeToFlag(AVRO_DOUBLE), - typeToFlag(AVRO_BOOL), - typeToFlag(AVRO_NULL), - typeToFlag(AVRO_RECORD), - typeToFlag(AVRO_ENUM), - typeToFlag(AVRO_ARRAY), - typeToFlag(AVRO_MAP), - typeToFlag(AVRO_UNION), - typeToFlag(AVRO_FIXED) - }; - static_assert((sizeof(flags)/sizeof(flag_t)) == (AVRO_NUM_TYPES), - "Invalid number of avro type flags"); - - expectedTypesFlag_ = flags[type]; -} - -void -Validator::setupOperation(const NodePtr &node) -{ - nextType_ = node->type(); - - if(nextType_ == AVRO_SYMBOLIC) { - NodePtr actualNode = resolveSymbol(node); - assert(actualNode); - setupOperation(actualNode); - return; - } - - assert(nextType_ < AVRO_SYMBOLIC); - - setupFlag(nextType_); - - if(!isPrimitive(nextType_)) { - compoundStack_.push_back(CompoundType(node)); - compoundStarted_ = true; - } -} - -bool -Validator::getCurrentRecordName(std::string &name) const -{ - bool found = false; - name.clear(); - - int idx = -1; - // if the top of the stack is a record I want this record name - if(!compoundStack_.empty() && (isPrimitive(nextType_) || nextType_ == AVRO_RECORD)) { - idx = compoundStack_.size() -1; - } - else { - idx = compoundStack_.size() -2; - } - - if(idx >= 0 && compoundStack_[idx].node->type() == AVRO_RECORD) { - name = compoundStack_[idx].node->name().simpleName(); - found = true; - } - return found; -} - -bool -Validator::getNextFieldName(std::string &name) const -{ - bool found = false; - name.clear(); - int idx = isCompound(nextType_) ? compoundStack_.size()-2 : compoundStack_.size()-1; - if(idx >= 0 && compoundStack_[idx].node->type() == AVRO_RECORD) { - size_t pos = compoundStack_[idx].pos-1; - const NodePtr &node = compoundStack_[idx].node; - if(pos < node->leaves()) { - name = node->nameAt(pos); - found = true; - } - } - return found; -} - -} // namespace avro +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Validator.hh" +#include "ValidSchema.hh" +#include "NodeImpl.hh" + +namespace avro { + +Validator::Validator(const ValidSchema &schema) : + schema_(schema), + nextType_(AVRO_NULL), + expectedTypesFlag_(0), + compoundStarted_(false), + waitingForCount_(false), + count_(0) +{ + setupOperation(schema_.root()); +} + +void +Validator::setWaitingForCount() +{ + waitingForCount_ = true; + count_ = 0; + expectedTypesFlag_ = typeToFlag(AVRO_INT) | typeToFlag(AVRO_LONG); + nextType_ = AVRO_LONG; +} + +void +Validator::enumAdvance() +{ + if(compoundStarted_) { + setWaitingForCount(); + compoundStarted_ = false; + } + else { + waitingForCount_ = false; + compoundStack_.pop_back(); + } +} + +bool +Validator::countingSetup() +{ + bool proceed = true; + if(compoundStarted_) { + setWaitingForCount(); + compoundStarted_ = false; + proceed = false; + } + else if(waitingForCount_) { + waitingForCount_ = false; + if(count_ == 0) { + compoundStack_.pop_back(); + proceed = false; + } + else { + counters_.push_back(static_cast<size_t>(count_)); + } + } + + return proceed; +} + +void +Validator::countingAdvance() +{ + if(countingSetup()) { + + size_t index = (compoundStack_.back().pos)++; + const NodePtr &node = compoundStack_.back().node; + + if(index < node->leaves() ) { + setupOperation(node->leafAt(index)); + } + else { + compoundStack_.back().pos = 0; + int count = --counters_.back(); + if(count == 0) { + counters_.pop_back(); + compoundStarted_ = true; + nextType_ = node->type(); + expectedTypesFlag_ = typeToFlag(nextType_); + } + else { + size_t index = (compoundStack_.back().pos)++; + setupOperation(node->leafAt(index)); + } + } + } +} + +void +Validator::unionAdvance() +{ + if(compoundStarted_) { + setWaitingForCount(); + compoundStarted_ = false; + } + else { + waitingForCount_ = false; + NodePtr node = compoundStack_.back().node; + + if(count_ < static_cast<int64_t>(node->leaves())) { + compoundStack_.pop_back(); + setupOperation(node->leafAt(static_cast<int>(count_))); + } + else { + throw Exception( + boost::format("Union selection out of range, got %1%," \ + " expecting 0-%2%") + % count_ % (node->leaves() -1) + ); + } + } +} + +void +Validator::fixedAdvance() +{ + compoundStarted_ = false; + compoundStack_.pop_back(); +} + +int +Validator::nextSizeExpected() const +{ + return compoundStack_.back().node->fixedSize(); +} + +void +Validator::doAdvance() +{ + typedef void (Validator::*AdvanceFunc)(); + + // only the compound types need advance functions here + static const AdvanceFunc funcs[] = { + 0, // string + 0, // bytes + 0, // int + 0, // long + 0, // float + 0, // double + 0, // bool + 0, // null + &Validator::countingAdvance, // Record is treated like counting with count == 1 + &Validator::enumAdvance, + &Validator::countingAdvance, + &Validator::countingAdvance, + &Validator::unionAdvance, + &Validator::fixedAdvance + }; + static_assert((sizeof(funcs)/sizeof(AdvanceFunc)) == (AVRO_NUM_TYPES), + "Invalid number of advance functions"); + + expectedTypesFlag_ = 0; + // loop until we encounter a next expected type, or we've exited all compound types + while(!expectedTypesFlag_ && !compoundStack_.empty() ) { + + Type type = compoundStack_.back().node->type(); + + AdvanceFunc func = funcs[type]; + + // only compound functions are put on the status stack so it is ok to + // assume that func is not null + assert(func); + + ((this)->*(func))(); + } + + if(compoundStack_.empty()) { + nextType_ = AVRO_NULL; + } +} + +void Validator::advance() +{ + if(!waitingForCount_) { + doAdvance(); + } +} + +void +Validator::setCount(int64_t count) +{ + if(!waitingForCount_) { + throw Exception("Not expecting count"); + } + else if(count_ < 0) { + throw Exception("Count cannot be negative"); + } + count_ = count; + + doAdvance(); +} + +void +Validator::setupFlag(Type type) +{ + // use flags instead of strictly types, so that we can be more lax about the type + // (for example, a long should be able to accept an int type, but not vice versa) + static const flag_t flags[] = { + typeToFlag(AVRO_STRING) | typeToFlag(AVRO_BYTES), + typeToFlag(AVRO_STRING) | typeToFlag(AVRO_BYTES), + typeToFlag(AVRO_INT), + typeToFlag(AVRO_INT) | typeToFlag(AVRO_LONG), + typeToFlag(AVRO_FLOAT), + typeToFlag(AVRO_DOUBLE), + typeToFlag(AVRO_BOOL), + typeToFlag(AVRO_NULL), + typeToFlag(AVRO_RECORD), + typeToFlag(AVRO_ENUM), + typeToFlag(AVRO_ARRAY), + typeToFlag(AVRO_MAP), + typeToFlag(AVRO_UNION), + typeToFlag(AVRO_FIXED) + }; + static_assert((sizeof(flags)/sizeof(flag_t)) == (AVRO_NUM_TYPES), + "Invalid number of avro type flags"); + + expectedTypesFlag_ = flags[type]; +} + +void +Validator::setupOperation(const NodePtr &node) +{ + nextType_ = node->type(); + + if(nextType_ == AVRO_SYMBOLIC) { + NodePtr actualNode = resolveSymbol(node); + assert(actualNode); + setupOperation(actualNode); + return; + } + + assert(nextType_ < AVRO_SYMBOLIC); + + setupFlag(nextType_); + + if(!isPrimitive(nextType_)) { + compoundStack_.push_back(CompoundType(node)); + compoundStarted_ = true; + } +} + +bool +Validator::getCurrentRecordName(std::string &name) const +{ + bool found = false; + name.clear(); + + int idx = -1; + // if the top of the stack is a record I want this record name + if(!compoundStack_.empty() && (isPrimitive(nextType_) || nextType_ == AVRO_RECORD)) { + idx = compoundStack_.size() -1; + } + else { + idx = compoundStack_.size() -2; + } + + if(idx >= 0 && compoundStack_[idx].node->type() == AVRO_RECORD) { + name = compoundStack_[idx].node->name().simpleName(); + found = true; + } + return found; +} + +bool +Validator::getNextFieldName(std::string &name) const +{ + bool found = false; + name.clear(); + int idx = isCompound(nextType_) ? compoundStack_.size()-2 : compoundStack_.size()-1; + if(idx >= 0 && compoundStack_[idx].node->type() == AVRO_RECORD) { + size_t pos = compoundStack_[idx].pos-1; + const NodePtr &node = compoundStack_[idx].node; + if(pos < node->leaves()) { + name = node->nameAt(pos); + found = true; + } + } + return found; +} + +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/Zigzag.cc b/contrib/libs/apache/avro/impl/Zigzag.cc index d4844440a7..06db5b4e7b 100644 --- a/contrib/libs/apache/avro/impl/Zigzag.cc +++ b/contrib/libs/apache/avro/impl/Zigzag.cc @@ -1,86 +1,86 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include "Zigzag.hh" - -namespace avro { - -uint64_t -encodeZigzag64(int64_t input) -{ - // cppcheck-suppress shiftTooManyBitsSigned - return ((input << 1) ^ (input >> 63)); -} - -int64_t -decodeZigzag64(uint64_t input) -{ - return static_cast<int64_t>(((input >> 1) ^ -(static_cast<int64_t>(input) & 1))); -} - -uint32_t -encodeZigzag32(int32_t input) -{ - // cppcheck-suppress shiftTooManyBitsSigned - return ((input << 1) ^ (input >> 31)); -} - -int32_t -decodeZigzag32(uint32_t input) -{ - return static_cast<int32_t>(((input >> 1) ^ -(static_cast<int64_t>(input) & 1))); -} - -size_t -encodeInt64(int64_t input, std::array<uint8_t, 10> &output) -{ - // get the zigzag encoding - uint64_t val = encodeZigzag64(input); - - // put values in an array of bytes with variable length encoding - const int mask = 0x7F; - output[0] = val & mask; - size_t bytesOut = 1; - while( val >>=7 ) { - output[bytesOut-1] |= 0x80; - output[bytesOut++] = (val & mask); - } - - return bytesOut; -} - -size_t -encodeInt32(int32_t input, std::array<uint8_t, 5> &output) -{ - // get the zigzag encoding - uint32_t val = encodeZigzag32(input); - - // put values in an array of bytes with variable length encoding - const int mask = 0x7F; - output[0] = val & mask; - size_t bytesOut = 1; - while( val >>=7 ) { - output[bytesOut-1] |= 0x80; - output[bytesOut++] = (val & mask); - } - - return bytesOut; -} - -} // namespace avro +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include "Zigzag.hh" + +namespace avro { + +uint64_t +encodeZigzag64(int64_t input) +{ + // cppcheck-suppress shiftTooManyBitsSigned + return ((input << 1) ^ (input >> 63)); +} + +int64_t +decodeZigzag64(uint64_t input) +{ + return static_cast<int64_t>(((input >> 1) ^ -(static_cast<int64_t>(input) & 1))); +} + +uint32_t +encodeZigzag32(int32_t input) +{ + // cppcheck-suppress shiftTooManyBitsSigned + return ((input << 1) ^ (input >> 31)); +} + +int32_t +decodeZigzag32(uint32_t input) +{ + return static_cast<int32_t>(((input >> 1) ^ -(static_cast<int64_t>(input) & 1))); +} + +size_t +encodeInt64(int64_t input, std::array<uint8_t, 10> &output) +{ + // get the zigzag encoding + uint64_t val = encodeZigzag64(input); + + // put values in an array of bytes with variable length encoding + const int mask = 0x7F; + output[0] = val & mask; + size_t bytesOut = 1; + while( val >>=7 ) { + output[bytesOut-1] |= 0x80; + output[bytesOut++] = (val & mask); + } + + return bytesOut; +} + +size_t +encodeInt32(int32_t input, std::array<uint8_t, 5> &output) +{ + // get the zigzag encoding + uint32_t val = encodeZigzag32(input); + + // put values in an array of bytes with variable length encoding + const int mask = 0x7F; + output[0] = val & mask; + size_t bytesOut = 1; + while( val >>=7 ) { + output[bytesOut-1] |= 0x80; + output[bytesOut++] = (val & mask); + } + + return bytesOut; +} + +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/json/JsonDom.cc b/contrib/libs/apache/avro/impl/json/JsonDom.cc index 8a41912be2..ac4d8c9bfc 100644 --- a/contrib/libs/apache/avro/impl/json/JsonDom.cc +++ b/contrib/libs/apache/avro/impl/json/JsonDom.cc @@ -1,203 +1,203 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "JsonDom.hh" - -#include <stdexcept> - -#include <string.h> - -#include "Stream.hh" -#include "JsonIO.hh" - -using std::string; -using boost::format; - -namespace avro { -namespace json { -const char* typeToString(EntityType t) -{ - switch (t) { - case etNull: return "null"; - case etBool: return "bool"; - case etLong: return "long"; - case etDouble: return "double"; - case etString: return "string"; - case etArray: return "array"; - case etObject: return "object"; - default: return "unknown"; - } -} - -Entity readEntity(JsonParser& p) -{ - switch (p.peek()) { - case JsonParser::tkNull: - p.advance(); - return Entity(p.line()); - case JsonParser::tkBool: - p.advance(); - return Entity(p.boolValue(), p.line()); - case JsonParser::tkLong: - p.advance(); - return Entity(p.longValue(), p.line()); - case JsonParser::tkDouble: - p.advance(); - return Entity(p.doubleValue(), p.line()); - case JsonParser::tkString: - p.advance(); - return Entity(std::make_shared<String>(p.rawString()), p.line()); - case JsonParser::tkArrayStart: - { - size_t l = p.line(); - p.advance(); - std::shared_ptr<Array> v = std::make_shared<Array>(); - while (p.peek() != JsonParser::tkArrayEnd) { - v->push_back(readEntity(p)); - } - p.advance(); - return Entity(v, l); - } - case JsonParser::tkObjectStart: - { - size_t l = p.line(); - p.advance(); - std::shared_ptr<Object> v = std::make_shared<Object>(); - while (p.peek() != JsonParser::tkObjectEnd) { - p.advance(); - std::string k = p.stringValue(); - Entity n = readEntity(p); - v->insert(std::make_pair(k, n)); - } - p.advance(); - return Entity(v, l); - } - default: - throw std::domain_error(JsonParser::toString(p.peek())); - } - -} - -Entity loadEntity(const char* text) -{ - return loadEntity(reinterpret_cast<const uint8_t*>(text), ::strlen(text)); -} - -Entity loadEntity(InputStream& in) -{ - JsonParser p; - p.init(in); - return readEntity(p); -} - -Entity loadEntity(const uint8_t* text, size_t len) -{ - std::unique_ptr<InputStream> in = memoryInputStream(text, len); - return loadEntity(*in); -} - -void writeEntity(JsonGenerator<JsonNullFormatter>& g, const Entity& n) -{ - switch (n.type()) { - case etNull: - g.encodeNull(); - break; - case etBool: - g.encodeBool(n.boolValue()); - break; - case etLong: - g.encodeNumber(n.longValue()); - break; - case etDouble: - g.encodeNumber(n.doubleValue()); - break; - case etString: - g.encodeString(n.stringValue()); - break; - case etArray: - { - g.arrayStart(); - const Array& v = n.arrayValue(); - for (Array::const_iterator it = v.begin(); - it != v.end(); ++it) { - writeEntity(g, *it); - } - g.arrayEnd(); - } - break; - case etObject: - { - g.objectStart(); - const Object& v = n.objectValue(); - for (Object::const_iterator it = v.begin(); it != v.end(); ++it) { - g.encodeString(it->first); - writeEntity(g, it->second); - } - g.objectEnd(); - } - break; - } -} - -void Entity::ensureType(EntityType type) const -{ - if (type_ != type) { - format msg = format("Invalid type. Expected \"%1%\" actual %2%") % - typeToString(type) % typeToString(type_); - throw Exception(msg); - } -} - -String Entity::stringValue() const { - ensureType(etString); - return JsonParser::toStringValue(**boost::any_cast<std::shared_ptr<String> >(&value_)); -} - -String Entity::bytesValue() const { - ensureType(etString); - return JsonParser::toBytesValue(**boost::any_cast<std::shared_ptr<String> >(&value_)); -} - -std::string Entity::toString() const -{ - std::unique_ptr<OutputStream> out = memoryOutputStream(); - JsonGenerator<JsonNullFormatter> g; - g.init(*out); - writeEntity(g, *this); - g.flush(); - std::unique_ptr<InputStream> in = memoryInputStream(*out); - const uint8_t *p = 0; - size_t n = 0; - size_t c = 0; - while (in->next(&p, &n)) { - c += n; - } - std::string result; - result.resize(c); - c = 0; - std::unique_ptr<InputStream> in2 = memoryInputStream(*out); - while (in2->next(&p, &n)) { - ::memcpy(&result[c], p, n); - c += n; - } - return result; -} - -} -} - +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JsonDom.hh" + +#include <stdexcept> + +#include <string.h> + +#include "Stream.hh" +#include "JsonIO.hh" + +using std::string; +using boost::format; + +namespace avro { +namespace json { +const char* typeToString(EntityType t) +{ + switch (t) { + case etNull: return "null"; + case etBool: return "bool"; + case etLong: return "long"; + case etDouble: return "double"; + case etString: return "string"; + case etArray: return "array"; + case etObject: return "object"; + default: return "unknown"; + } +} + +Entity readEntity(JsonParser& p) +{ + switch (p.peek()) { + case JsonParser::tkNull: + p.advance(); + return Entity(p.line()); + case JsonParser::tkBool: + p.advance(); + return Entity(p.boolValue(), p.line()); + case JsonParser::tkLong: + p.advance(); + return Entity(p.longValue(), p.line()); + case JsonParser::tkDouble: + p.advance(); + return Entity(p.doubleValue(), p.line()); + case JsonParser::tkString: + p.advance(); + return Entity(std::make_shared<String>(p.rawString()), p.line()); + case JsonParser::tkArrayStart: + { + size_t l = p.line(); + p.advance(); + std::shared_ptr<Array> v = std::make_shared<Array>(); + while (p.peek() != JsonParser::tkArrayEnd) { + v->push_back(readEntity(p)); + } + p.advance(); + return Entity(v, l); + } + case JsonParser::tkObjectStart: + { + size_t l = p.line(); + p.advance(); + std::shared_ptr<Object> v = std::make_shared<Object>(); + while (p.peek() != JsonParser::tkObjectEnd) { + p.advance(); + std::string k = p.stringValue(); + Entity n = readEntity(p); + v->insert(std::make_pair(k, n)); + } + p.advance(); + return Entity(v, l); + } + default: + throw std::domain_error(JsonParser::toString(p.peek())); + } + +} + +Entity loadEntity(const char* text) +{ + return loadEntity(reinterpret_cast<const uint8_t*>(text), ::strlen(text)); +} + +Entity loadEntity(InputStream& in) +{ + JsonParser p; + p.init(in); + return readEntity(p); +} + +Entity loadEntity(const uint8_t* text, size_t len) +{ + std::unique_ptr<InputStream> in = memoryInputStream(text, len); + return loadEntity(*in); +} + +void writeEntity(JsonGenerator<JsonNullFormatter>& g, const Entity& n) +{ + switch (n.type()) { + case etNull: + g.encodeNull(); + break; + case etBool: + g.encodeBool(n.boolValue()); + break; + case etLong: + g.encodeNumber(n.longValue()); + break; + case etDouble: + g.encodeNumber(n.doubleValue()); + break; + case etString: + g.encodeString(n.stringValue()); + break; + case etArray: + { + g.arrayStart(); + const Array& v = n.arrayValue(); + for (Array::const_iterator it = v.begin(); + it != v.end(); ++it) { + writeEntity(g, *it); + } + g.arrayEnd(); + } + break; + case etObject: + { + g.objectStart(); + const Object& v = n.objectValue(); + for (Object::const_iterator it = v.begin(); it != v.end(); ++it) { + g.encodeString(it->first); + writeEntity(g, it->second); + } + g.objectEnd(); + } + break; + } +} + +void Entity::ensureType(EntityType type) const +{ + if (type_ != type) { + format msg = format("Invalid type. Expected \"%1%\" actual %2%") % + typeToString(type) % typeToString(type_); + throw Exception(msg); + } +} + +String Entity::stringValue() const { + ensureType(etString); + return JsonParser::toStringValue(**boost::any_cast<std::shared_ptr<String> >(&value_)); +} + +String Entity::bytesValue() const { + ensureType(etString); + return JsonParser::toBytesValue(**boost::any_cast<std::shared_ptr<String> >(&value_)); +} + +std::string Entity::toString() const +{ + std::unique_ptr<OutputStream> out = memoryOutputStream(); + JsonGenerator<JsonNullFormatter> g; + g.init(*out); + writeEntity(g, *this); + g.flush(); + std::unique_ptr<InputStream> in = memoryInputStream(*out); + const uint8_t *p = 0; + size_t n = 0; + size_t c = 0; + while (in->next(&p, &n)) { + c += n; + } + std::string result; + result.resize(c); + c = 0; + std::unique_ptr<InputStream> in2 = memoryInputStream(*out); + while (in2->next(&p, &n)) { + ::memcpy(&result[c], p, n); + c += n; + } + return result; +} + +} +} + diff --git a/contrib/libs/apache/avro/impl/json/JsonDom.hh b/contrib/libs/apache/avro/impl/json/JsonDom.hh index 7eb412aa2f..e1f549dfea 100644 --- a/contrib/libs/apache/avro/impl/json/JsonDom.hh +++ b/contrib/libs/apache/avro/impl/json/JsonDom.hh @@ -1,162 +1,162 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_json_JsonDom_hh__ -#define avro_json_JsonDom_hh__ - -#include <iostream> -#include <stdint.h> -#include <map> -#include <string> -#include <vector> -#include <memory> - -#include "boost/any.hpp" -#include "Config.hh" - -namespace avro { - -class AVRO_DECL InputStream; - -namespace json { -class Entity; - -typedef bool Bool; -typedef int64_t Long; -typedef double Double; -typedef std::string String; -typedef std::vector<Entity> Array; -typedef std::map<std::string, Entity> Object; - -class AVRO_DECL JsonParser; -class JsonNullFormatter; - -template<typename F = JsonNullFormatter> -class AVRO_DECL JsonGenerator; - -enum EntityType { - etNull, - etBool, - etLong, - etDouble, - etString, - etArray, - etObject -}; - -const char* typeToString(EntityType t); - -class AVRO_DECL Entity { - EntityType type_; - boost::any value_; - size_t line_; // can't be const else noncopyable... - - void ensureType(EntityType) const; -public: - Entity(size_t line = 0) : type_(etNull), line_(line) { } - Entity(Bool v, size_t line = 0) : type_(etBool), value_(v), line_(line) { } - Entity(Long v, size_t line = 0) : type_(etLong), value_(v), line_(line) { } - Entity(Double v, size_t line = 0) : type_(etDouble), value_(v), line_(line) { } - Entity(const std::shared_ptr<String>& v, size_t line = 0) : type_(etString), value_(v), line_(line) { } - Entity(const std::shared_ptr<Array>& v, size_t line = 0) : type_(etArray), value_(v), line_(line) { } - Entity(const std::shared_ptr<Object>& v, size_t line = 0) : type_(etObject), value_(v), line_(line) { } - - EntityType type() const { return type_; } - - size_t line() const { return line_; } - - Bool boolValue() const { - ensureType(etBool); - return boost::any_cast<Bool>(value_); - } - - Long longValue() const { - ensureType(etLong); - return boost::any_cast<Long>(value_); - } - - Double doubleValue() const { - ensureType(etDouble); - return boost::any_cast<Double>(value_); - } - - String stringValue() const; - - String bytesValue() const; - - const Array& arrayValue() const { - ensureType(etArray); - return **boost::any_cast<std::shared_ptr<Array> >(&value_); - } - - const Object& objectValue() const { - ensureType(etObject); - return **boost::any_cast<std::shared_ptr<Object> >(&value_); - } - - std::string toString() const; -}; - -template <typename T> -struct type_traits { -}; - -template <> struct type_traits<bool> { - static EntityType type() { return etBool; } - static const char* name() { return "bool"; } -}; - -template <> struct type_traits<int64_t> { - static EntityType type() { return etLong; } - static const char* name() { return "long"; } -}; - -template <> struct type_traits<double> { - static EntityType type() { return etDouble; } - static const char* name() { return "double"; } -}; - -template <> struct type_traits<std::string> { - static EntityType type() { return etString; } - static const char* name() { return "string"; } -}; - -template <> struct type_traits<std::vector<Entity> > { - static EntityType type() { return etArray; } - static const char* name() { return "array"; } -}; - -template <> struct type_traits<std::map<std::string, Entity> > { - static EntityType type() { return etObject; } - static const char* name() { return "object"; } -}; - -AVRO_DECL Entity readEntity(JsonParser& p); - -AVRO_DECL Entity loadEntity(InputStream& in); -AVRO_DECL Entity loadEntity(const char* text); -AVRO_DECL Entity loadEntity(const uint8_t* text, size_t len); - -void writeEntity(JsonGenerator<JsonNullFormatter>& g, const Entity& n); - -} -} - -#endif - - +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_json_JsonDom_hh__ +#define avro_json_JsonDom_hh__ + +#include <iostream> +#include <stdint.h> +#include <map> +#include <string> +#include <vector> +#include <memory> + +#include "boost/any.hpp" +#include "Config.hh" + +namespace avro { + +class AVRO_DECL InputStream; + +namespace json { +class Entity; + +typedef bool Bool; +typedef int64_t Long; +typedef double Double; +typedef std::string String; +typedef std::vector<Entity> Array; +typedef std::map<std::string, Entity> Object; + +class AVRO_DECL JsonParser; +class JsonNullFormatter; + +template<typename F = JsonNullFormatter> +class AVRO_DECL JsonGenerator; + +enum EntityType { + etNull, + etBool, + etLong, + etDouble, + etString, + etArray, + etObject +}; + +const char* typeToString(EntityType t); + +class AVRO_DECL Entity { + EntityType type_; + boost::any value_; + size_t line_; // can't be const else noncopyable... + + void ensureType(EntityType) const; +public: + Entity(size_t line = 0) : type_(etNull), line_(line) { } + Entity(Bool v, size_t line = 0) : type_(etBool), value_(v), line_(line) { } + Entity(Long v, size_t line = 0) : type_(etLong), value_(v), line_(line) { } + Entity(Double v, size_t line = 0) : type_(etDouble), value_(v), line_(line) { } + Entity(const std::shared_ptr<String>& v, size_t line = 0) : type_(etString), value_(v), line_(line) { } + Entity(const std::shared_ptr<Array>& v, size_t line = 0) : type_(etArray), value_(v), line_(line) { } + Entity(const std::shared_ptr<Object>& v, size_t line = 0) : type_(etObject), value_(v), line_(line) { } + + EntityType type() const { return type_; } + + size_t line() const { return line_; } + + Bool boolValue() const { + ensureType(etBool); + return boost::any_cast<Bool>(value_); + } + + Long longValue() const { + ensureType(etLong); + return boost::any_cast<Long>(value_); + } + + Double doubleValue() const { + ensureType(etDouble); + return boost::any_cast<Double>(value_); + } + + String stringValue() const; + + String bytesValue() const; + + const Array& arrayValue() const { + ensureType(etArray); + return **boost::any_cast<std::shared_ptr<Array> >(&value_); + } + + const Object& objectValue() const { + ensureType(etObject); + return **boost::any_cast<std::shared_ptr<Object> >(&value_); + } + + std::string toString() const; +}; + +template <typename T> +struct type_traits { +}; + +template <> struct type_traits<bool> { + static EntityType type() { return etBool; } + static const char* name() { return "bool"; } +}; + +template <> struct type_traits<int64_t> { + static EntityType type() { return etLong; } + static const char* name() { return "long"; } +}; + +template <> struct type_traits<double> { + static EntityType type() { return etDouble; } + static const char* name() { return "double"; } +}; + +template <> struct type_traits<std::string> { + static EntityType type() { return etString; } + static const char* name() { return "string"; } +}; + +template <> struct type_traits<std::vector<Entity> > { + static EntityType type() { return etArray; } + static const char* name() { return "array"; } +}; + +template <> struct type_traits<std::map<std::string, Entity> > { + static EntityType type() { return etObject; } + static const char* name() { return "object"; } +}; + +AVRO_DECL Entity readEntity(JsonParser& p); + +AVRO_DECL Entity loadEntity(InputStream& in); +AVRO_DECL Entity loadEntity(const char* text); +AVRO_DECL Entity loadEntity(const uint8_t* text, size_t len); + +void writeEntity(JsonGenerator<JsonNullFormatter>& g, const Entity& n); + +} +} + +#endif + + diff --git a/contrib/libs/apache/avro/impl/json/JsonIO.cc b/contrib/libs/apache/avro/impl/json/JsonIO.cc index d09ea2315f..c11a722ad4 100644 --- a/contrib/libs/apache/avro/impl/json/JsonIO.cc +++ b/contrib/libs/apache/avro/impl/json/JsonIO.cc @@ -1,442 +1,442 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "JsonIO.hh" - -namespace avro { -namespace json { - -using std::ostringstream; -using std::string; - -const char* const -JsonParser::tokenNames[] = { - "Null", - "Bool", - "Integer", - "Double", - "String", - "Array start", - "Array end", - "Object start", - "Object end", -}; - -char JsonParser::next() -{ - char ch = hasNext ? nextChar : ' '; - while (isspace(ch)) { - if (ch == '\n') { - line_++; - } - ch = in_.read(); - } - hasNext = false; - return ch; -} - -void JsonParser::expectToken(Token tk) -{ - if (advance() != tk) { - if (tk == tkDouble) { - if(cur() == tkString - && (sv == "Infinity" || sv == "-Infinity" || sv == "NaN")) { - curToken = tkDouble; - dv = sv == "Infinity" ? - std::numeric_limits<double>::infinity() : - sv == "-Infinity" ? - -std::numeric_limits<double>::infinity() : - std::numeric_limits<double>::quiet_NaN(); - return; - } else if (cur() == tkLong) { - dv = double(lv); - return; - } - } - ostringstream oss; - oss << "Incorrect token in the stream. Expected: " - << JsonParser::toString(tk) << ", found " - << JsonParser::toString(cur()); - throw Exception(oss.str()); - } -} - -JsonParser::Token JsonParser::doAdvance() -{ - char ch = next(); - if (ch == ']') { - if (curState == stArray0 || curState == stArrayN) { - curState = stateStack.top(); - stateStack.pop(); - return tkArrayEnd; - } else { - throw unexpected(ch); - } - } else if (ch == '}') { - if (curState == stObject0 || curState == stObjectN) { - curState = stateStack.top(); - stateStack.pop(); - return tkObjectEnd; - } else { - throw unexpected(ch); - } - } else if (ch == ',') { - if (curState != stObjectN && curState != stArrayN) { - throw unexpected(ch); - } - if (curState == stObjectN) { - curState = stObject0; - } - ch = next(); - } else if (ch == ':') { - if (curState != stKey) { - throw unexpected(ch); - } - curState = stObjectN; - ch = next(); - } - - if (curState == stObject0) { - if (ch != '"') { - throw unexpected(ch); - } - curState = stKey; - } else if (curState == stArray0) { - curState = stArrayN; - } - - switch (ch) { - case '[': - stateStack.push(curState); - curState = stArray0; - return tkArrayStart; - case '{': - stateStack.push(curState); - curState = stObject0; - return tkObjectStart; - case '"': - return tryString(); - case 't': - bv = true; - return tryLiteral("rue", 3, tkBool); - case 'f': - bv = false; - return tryLiteral("alse", 4, tkBool); - case 'n': - return tryLiteral("ull", 3, tkNull); - default: - if (isdigit(ch) || ch == '-') { - return tryNumber(ch); - } else { - throw unexpected(ch); - } - } -} - -JsonParser::Token JsonParser::tryNumber(char ch) -{ - sv.clear(); - sv.push_back(ch); - - hasNext = false; - int state = (ch == '-') ? 0 : (ch == '0') ? 1 : 2; - for (; ;) { - switch (state) { - case 0: - if (in_.hasMore()) { - ch = in_.read(); - if (isdigit(ch)) { - state = (ch == '0') ? 1 : 2; - sv.push_back(ch); - continue; - } - hasNext = true; - } - break; - case 1: - if (in_.hasMore()) { - ch = in_.read(); - if (ch == '.') { - state = 3; - sv.push_back(ch); - continue; - } else if (ch == 'e' || ch == 'E') { - sv.push_back(ch); - state = 5; - continue; - } - hasNext = true; - } - break; - case 2: - if (in_.hasMore()) { - ch = in_.read(); - if (isdigit(ch)) { - sv.push_back(ch); - continue; - } else if (ch == '.') { - state = 3; - sv.push_back(ch); - continue; - } else if (ch == 'e' || ch == 'E') { - sv.push_back(ch); - state = 5; - continue; - } - hasNext = true; - } - break; - case 3: - case 6: - if (in_.hasMore()) { - ch = in_.read(); - if (isdigit(ch)) { - sv.push_back(ch); - state++; - continue; - } - hasNext = true; - } - break; - case 4: - if (in_.hasMore()) { - ch = in_.read(); - if (isdigit(ch)) { - sv.push_back(ch); - continue; - } else if (ch == 'e' || ch == 'E') { - sv.push_back(ch); - state = 5; - continue; - } - hasNext = true; - } - break; - case 5: - if (in_.hasMore()) { - ch = in_.read(); - if (ch == '+' || ch == '-') { - sv.push_back(ch); - state = 6; - continue; - } else if (isdigit(ch)) { - sv.push_back(ch); - state = 7; - continue; - } - hasNext = true; - } - break; - case 7: - if (in_.hasMore()) { - ch = in_.read(); - if (isdigit(ch)) { - sv.push_back(ch); - continue; - } - hasNext = true; - } - break; - } - if (state == 1 || state == 2 || state == 4 || state == 7) { - if (hasNext) { - nextChar = ch; - } - std::istringstream iss(sv); - if (state == 1 || state == 2) { - iss >> lv; - return tkLong; - } else { - iss >> dv; - return tkDouble; - } - } else { - if (hasNext) { - throw unexpected(ch); - } else { - throw Exception("Unexpected EOF"); - } - } - } -} - -JsonParser::Token JsonParser::tryString() -{ - sv.clear(); - for ( ; ;) { - char ch = in_.read(); - if (ch == '"') { - return tkString; - } else if (ch == '\\') { - ch = in_.read(); - switch (ch) { - case '"': - case '\\': - case '/': - case 'b': - case 'f': - case 'n': - case 'r': - case 't': - sv.push_back('\\'); - sv.push_back(ch); - break; - case 'u': - case 'U': - { - uint32_t n = 0; - char e[4]; - in_.readBytes(reinterpret_cast<uint8_t*>(e), 4); - sv.push_back('\\'); - sv.push_back(ch); - for (int i = 0; i < 4; i++) { - n *= 16; - char c = e[i]; - if (isdigit(c) || - (c >= 'a' && c <= 'f') || - (c >= 'A' && c <= 'F')) { - sv.push_back(c); - } else { - throw unexpected(c); - } - } - } - break; - default: - throw unexpected(ch); - } - } else { - sv.push_back(ch); - } - } -} - - -string JsonParser::decodeString(const string& s, bool binary) -{ - string result; - for (string::const_iterator it = s.begin(); it != s.end(); ++it) { - char ch = *it; - if (ch == '\\') { - ch = *++it; - switch (ch) { - case '"': - case '\\': - case '/': - result.push_back(ch); - continue; - case 'b': - result.push_back('\b'); - continue; - case 'f': - result.push_back('\f'); - continue; - case 'n': - result.push_back('\n'); - continue; - case 'r': - result.push_back('\r'); - continue; - case 't': - result.push_back('\t'); - continue; - case 'u': - case 'U': - { - uint32_t n = 0; - char e[4]; - for (int i = 0; i < 4; i++) { - n *= 16; - char c = *++it; - e[i] = c; - if (isdigit(c)) { - n += c - '0'; - } else if (c >= 'a' && c <= 'f') { - n += c - 'a' + 10; - } else if (c >= 'A' && c <= 'F') { - n += c - 'A' + 10; - } - } - if (binary) { - if (n > 0xff) { - throw Exception(boost::format( - "Invalid byte for binary: %1%%2%") % ch % - string(e, 4)); - } else { - result.push_back(n); - continue; - } - } - if (n < 0x80) { - result.push_back(n); - } else if (n < 0x800) { - result.push_back((n >> 6) | 0xc0); - result.push_back((n & 0x3f) | 0x80); - } else if (n < 0x10000) { - result.push_back((n >> 12) | 0xe0); - result.push_back(((n >> 6)& 0x3f) | 0x80); - result.push_back((n & 0x3f) | 0x80); - } else if (n < 110000) { - result.push_back((n >> 18) | 0xf0); - result.push_back(((n >> 12)& 0x3f) | 0x80); - result.push_back(((n >> 6)& 0x3f) | 0x80); - result.push_back((n & 0x3f) | 0x80); - } else { - throw Exception(boost::format( - "Invalid unicode value: %1%i%2%") % ch % - string(e, 4)); - } - } - continue; - } - } else { - result.push_back(ch); - } - } - return result; -} - -Exception JsonParser::unexpected(unsigned char c) -{ - std::ostringstream oss; - oss << "Unexpected character in json " << toHex(c / 16) << toHex(c % 16); - return Exception(oss.str()); -} - -JsonParser::Token JsonParser::tryLiteral(const char exp[], size_t n, Token tk) -{ - char c[100]; - in_.readBytes(reinterpret_cast<uint8_t*>(c), n); - for (size_t i = 0; i < n; ++i) { - if (c[i] != exp[i]) { - throw unexpected(c[i]); - } - } - if (in_.hasMore()) { - nextChar = in_.read(); - if (isdigit(nextChar) || isalpha(nextChar)) { - throw unexpected(nextChar); - } - hasNext = true; - } - return tk; -} - -} -} - +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JsonIO.hh" + +namespace avro { +namespace json { + +using std::ostringstream; +using std::string; + +const char* const +JsonParser::tokenNames[] = { + "Null", + "Bool", + "Integer", + "Double", + "String", + "Array start", + "Array end", + "Object start", + "Object end", +}; + +char JsonParser::next() +{ + char ch = hasNext ? nextChar : ' '; + while (isspace(ch)) { + if (ch == '\n') { + line_++; + } + ch = in_.read(); + } + hasNext = false; + return ch; +} + +void JsonParser::expectToken(Token tk) +{ + if (advance() != tk) { + if (tk == tkDouble) { + if(cur() == tkString + && (sv == "Infinity" || sv == "-Infinity" || sv == "NaN")) { + curToken = tkDouble; + dv = sv == "Infinity" ? + std::numeric_limits<double>::infinity() : + sv == "-Infinity" ? + -std::numeric_limits<double>::infinity() : + std::numeric_limits<double>::quiet_NaN(); + return; + } else if (cur() == tkLong) { + dv = double(lv); + return; + } + } + ostringstream oss; + oss << "Incorrect token in the stream. Expected: " + << JsonParser::toString(tk) << ", found " + << JsonParser::toString(cur()); + throw Exception(oss.str()); + } +} + +JsonParser::Token JsonParser::doAdvance() +{ + char ch = next(); + if (ch == ']') { + if (curState == stArray0 || curState == stArrayN) { + curState = stateStack.top(); + stateStack.pop(); + return tkArrayEnd; + } else { + throw unexpected(ch); + } + } else if (ch == '}') { + if (curState == stObject0 || curState == stObjectN) { + curState = stateStack.top(); + stateStack.pop(); + return tkObjectEnd; + } else { + throw unexpected(ch); + } + } else if (ch == ',') { + if (curState != stObjectN && curState != stArrayN) { + throw unexpected(ch); + } + if (curState == stObjectN) { + curState = stObject0; + } + ch = next(); + } else if (ch == ':') { + if (curState != stKey) { + throw unexpected(ch); + } + curState = stObjectN; + ch = next(); + } + + if (curState == stObject0) { + if (ch != '"') { + throw unexpected(ch); + } + curState = stKey; + } else if (curState == stArray0) { + curState = stArrayN; + } + + switch (ch) { + case '[': + stateStack.push(curState); + curState = stArray0; + return tkArrayStart; + case '{': + stateStack.push(curState); + curState = stObject0; + return tkObjectStart; + case '"': + return tryString(); + case 't': + bv = true; + return tryLiteral("rue", 3, tkBool); + case 'f': + bv = false; + return tryLiteral("alse", 4, tkBool); + case 'n': + return tryLiteral("ull", 3, tkNull); + default: + if (isdigit(ch) || ch == '-') { + return tryNumber(ch); + } else { + throw unexpected(ch); + } + } +} + +JsonParser::Token JsonParser::tryNumber(char ch) +{ + sv.clear(); + sv.push_back(ch); + + hasNext = false; + int state = (ch == '-') ? 0 : (ch == '0') ? 1 : 2; + for (; ;) { + switch (state) { + case 0: + if (in_.hasMore()) { + ch = in_.read(); + if (isdigit(ch)) { + state = (ch == '0') ? 1 : 2; + sv.push_back(ch); + continue; + } + hasNext = true; + } + break; + case 1: + if (in_.hasMore()) { + ch = in_.read(); + if (ch == '.') { + state = 3; + sv.push_back(ch); + continue; + } else if (ch == 'e' || ch == 'E') { + sv.push_back(ch); + state = 5; + continue; + } + hasNext = true; + } + break; + case 2: + if (in_.hasMore()) { + ch = in_.read(); + if (isdigit(ch)) { + sv.push_back(ch); + continue; + } else if (ch == '.') { + state = 3; + sv.push_back(ch); + continue; + } else if (ch == 'e' || ch == 'E') { + sv.push_back(ch); + state = 5; + continue; + } + hasNext = true; + } + break; + case 3: + case 6: + if (in_.hasMore()) { + ch = in_.read(); + if (isdigit(ch)) { + sv.push_back(ch); + state++; + continue; + } + hasNext = true; + } + break; + case 4: + if (in_.hasMore()) { + ch = in_.read(); + if (isdigit(ch)) { + sv.push_back(ch); + continue; + } else if (ch == 'e' || ch == 'E') { + sv.push_back(ch); + state = 5; + continue; + } + hasNext = true; + } + break; + case 5: + if (in_.hasMore()) { + ch = in_.read(); + if (ch == '+' || ch == '-') { + sv.push_back(ch); + state = 6; + continue; + } else if (isdigit(ch)) { + sv.push_back(ch); + state = 7; + continue; + } + hasNext = true; + } + break; + case 7: + if (in_.hasMore()) { + ch = in_.read(); + if (isdigit(ch)) { + sv.push_back(ch); + continue; + } + hasNext = true; + } + break; + } + if (state == 1 || state == 2 || state == 4 || state == 7) { + if (hasNext) { + nextChar = ch; + } + std::istringstream iss(sv); + if (state == 1 || state == 2) { + iss >> lv; + return tkLong; + } else { + iss >> dv; + return tkDouble; + } + } else { + if (hasNext) { + throw unexpected(ch); + } else { + throw Exception("Unexpected EOF"); + } + } + } +} + +JsonParser::Token JsonParser::tryString() +{ + sv.clear(); + for ( ; ;) { + char ch = in_.read(); + if (ch == '"') { + return tkString; + } else if (ch == '\\') { + ch = in_.read(); + switch (ch) { + case '"': + case '\\': + case '/': + case 'b': + case 'f': + case 'n': + case 'r': + case 't': + sv.push_back('\\'); + sv.push_back(ch); + break; + case 'u': + case 'U': + { + uint32_t n = 0; + char e[4]; + in_.readBytes(reinterpret_cast<uint8_t*>(e), 4); + sv.push_back('\\'); + sv.push_back(ch); + for (int i = 0; i < 4; i++) { + n *= 16; + char c = e[i]; + if (isdigit(c) || + (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F')) { + sv.push_back(c); + } else { + throw unexpected(c); + } + } + } + break; + default: + throw unexpected(ch); + } + } else { + sv.push_back(ch); + } + } +} + + +string JsonParser::decodeString(const string& s, bool binary) +{ + string result; + for (string::const_iterator it = s.begin(); it != s.end(); ++it) { + char ch = *it; + if (ch == '\\') { + ch = *++it; + switch (ch) { + case '"': + case '\\': + case '/': + result.push_back(ch); + continue; + case 'b': + result.push_back('\b'); + continue; + case 'f': + result.push_back('\f'); + continue; + case 'n': + result.push_back('\n'); + continue; + case 'r': + result.push_back('\r'); + continue; + case 't': + result.push_back('\t'); + continue; + case 'u': + case 'U': + { + uint32_t n = 0; + char e[4]; + for (int i = 0; i < 4; i++) { + n *= 16; + char c = *++it; + e[i] = c; + if (isdigit(c)) { + n += c - '0'; + } else if (c >= 'a' && c <= 'f') { + n += c - 'a' + 10; + } else if (c >= 'A' && c <= 'F') { + n += c - 'A' + 10; + } + } + if (binary) { + if (n > 0xff) { + throw Exception(boost::format( + "Invalid byte for binary: %1%%2%") % ch % + string(e, 4)); + } else { + result.push_back(n); + continue; + } + } + if (n < 0x80) { + result.push_back(n); + } else if (n < 0x800) { + result.push_back((n >> 6) | 0xc0); + result.push_back((n & 0x3f) | 0x80); + } else if (n < 0x10000) { + result.push_back((n >> 12) | 0xe0); + result.push_back(((n >> 6)& 0x3f) | 0x80); + result.push_back((n & 0x3f) | 0x80); + } else if (n < 110000) { + result.push_back((n >> 18) | 0xf0); + result.push_back(((n >> 12)& 0x3f) | 0x80); + result.push_back(((n >> 6)& 0x3f) | 0x80); + result.push_back((n & 0x3f) | 0x80); + } else { + throw Exception(boost::format( + "Invalid unicode value: %1%i%2%") % ch % + string(e, 4)); + } + } + continue; + } + } else { + result.push_back(ch); + } + } + return result; +} + +Exception JsonParser::unexpected(unsigned char c) +{ + std::ostringstream oss; + oss << "Unexpected character in json " << toHex(c / 16) << toHex(c % 16); + return Exception(oss.str()); +} + +JsonParser::Token JsonParser::tryLiteral(const char exp[], size_t n, Token tk) +{ + char c[100]; + in_.readBytes(reinterpret_cast<uint8_t*>(c), n); + for (size_t i = 0; i < n; ++i) { + if (c[i] != exp[i]) { + throw unexpected(c[i]); + } + } + if (in_.hasMore()) { + nextChar = in_.read(); + if (isdigit(nextChar) || isalpha(nextChar)) { + throw unexpected(nextChar); + } + hasNext = true; + } + return tk; +} + +} +} + diff --git a/contrib/libs/apache/avro/impl/json/JsonIO.hh b/contrib/libs/apache/avro/impl/json/JsonIO.hh index 705c5fc4fd..5ae7ae07dc 100644 --- a/contrib/libs/apache/avro/impl/json/JsonIO.hh +++ b/contrib/libs/apache/avro/impl/json/JsonIO.hh @@ -1,482 +1,482 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_json_JsonIO_hh__ -#define avro_json_JsonIO_hh__ - -#include <locale> -#include <stack> -#include <string> -#include <sstream> -#include <boost/math/special_functions/fpclassify.hpp> -#include <boost/lexical_cast.hpp> -#include <boost/utility.hpp> - -#include "Config.hh" -#include "Stream.hh" - -namespace avro { -namespace json { - -inline char toHex(unsigned int n) { - return (n < 10) ? (n + '0') : (n + 'a' - 10); -} - - -class AVRO_DECL JsonParser : boost::noncopyable { -public: - enum Token { - tkNull, - tkBool, - tkLong, - tkDouble, - tkString, - tkArrayStart, - tkArrayEnd, - tkObjectStart, - tkObjectEnd - }; - - size_t line() const { return line_; } - -private: - enum State { - stValue, // Expect a data type - stArray0, // Expect a data type or ']' - stArrayN, // Expect a ',' or ']' - stObject0, // Expect a string or a '}' - stObjectN, // Expect a ',' or '}' - stKey // Expect a ':' - }; - std::stack<State> stateStack; - State curState; - bool hasNext; - char nextChar; - bool peeked; - - StreamReader in_; - Token curToken; - bool bv; - int64_t lv; - double dv; - std::string sv; - size_t line_; - - Token doAdvance(); - Token tryLiteral(const char exp[], size_t n, Token tk); - Token tryNumber(char ch); - Token tryString(); - Exception unexpected(unsigned char ch); - char next(); - - static std::string decodeString(const std::string& s, bool binary); - -public: - JsonParser() : curState(stValue), hasNext(false), peeked(false), line_(1) { } - - void init(InputStream& is) { - // Clear by swapping with an empty stack - std::stack<State>().swap(stateStack); - curState = stValue; - hasNext = false; - peeked = false; - line_ = 1; - in_.reset(is); - } - - Token advance() { - if (! peeked) { - curToken = doAdvance(); - } else { - peeked = false; - } - return curToken; - } - - Token peek() { - if (! peeked) { - curToken = doAdvance(); - peeked = true; - } - return curToken; - } - - void expectToken(Token tk); - - bool boolValue() const { - return bv; - } - - Token cur() const { - return curToken; - } - - double doubleValue() const { - return dv; - } - - int64_t longValue() const { - return lv; - } - - const std::string& rawString() const { - return sv; - } - - std::string stringValue() const { - return decodeString(sv, false); - } - - std::string bytesValue() const { - return decodeString(sv, true); - } - - void drain() { - if (!stateStack.empty() || peeked) { - throw Exception("Invalid state for draining"); - } - in_.drain(hasNext); - hasNext = false; - } - - /** - * Return UTF-8 encoded string value. - */ - static std::string toStringValue(const std::string& sv) { - return decodeString(sv, false); - } - - /** - * Return byte-encoded string value. It is an error if the input - * JSON string contained unicode characters more than "\u00ff'. - */ - static std::string toBytesValue(const std::string& sv) { - return decodeString(sv, true); - } - - static const char* const tokenNames[]; - - static const char* toString(Token tk) { - return tokenNames[tk]; - } -}; - -class AVRO_DECL JsonNullFormatter { -public: - JsonNullFormatter(StreamWriter&) { } - - void handleObjectStart() {} - void handleObjectEnd() {} - void handleValueEnd() {} - void handleColon() {} -}; - -class AVRO_DECL JsonPrettyFormatter { - StreamWriter& out_; - size_t level_; - std::vector<uint8_t> indent_; - - static const int CHARS_PER_LEVEL = 2; - - void printIndent() { - size_t charsToIndent = level_ * CHARS_PER_LEVEL; - if (indent_.size() < charsToIndent) { - indent_.resize(charsToIndent * 2, ' '); - } - out_.writeBytes(indent_.data(), charsToIndent); - } -public: - JsonPrettyFormatter(StreamWriter& out) : out_(out), level_(0), indent_(10, ' ') { } - - void handleObjectStart() { - out_.write('\n'); - ++level_; - printIndent(); - } - - void handleObjectEnd() { - out_.write('\n'); - --level_; - printIndent(); - } - - void handleValueEnd() { - out_.write('\n'); - printIndent(); - } - - void handleColon() { - out_.write(' '); - } -}; - -template <class F> -class AVRO_DECL JsonGenerator { - StreamWriter out_; - F formatter_; - enum State { - stStart, - stArray0, - stArrayN, - stMap0, - stMapN, - stKey, - }; - - std::stack<State> stateStack; - State top; - - void write(const char *b, const char* p) { - if (b != p) { - out_.writeBytes(reinterpret_cast<const uint8_t*>(b), p - b); - } - } - - void escape(char c, const char* b, const char *p) { - write(b, p); - out_.write('\\'); - out_.write(c); - } - - void escapeCtl(char c) { - escapeUnicode(static_cast<uint8_t>(c)); - } - - void writeHex(char c) { - out_.write(toHex((static_cast<unsigned char>(c)) / 16)); - out_.write(toHex((static_cast<unsigned char>(c)) % 16)); - } - - void escapeUnicode(uint32_t c) { - out_.write('\\'); - out_.write('u'); - writeHex((c >> 8) & 0xff); - writeHex(c & 0xff); - } - void doEncodeString(const char* b, size_t len, bool binary) { - const char* e = b + len; - out_.write('"'); - for (const char* p = b; p != e; p++) { - if ((*p & 0x80) != 0) { - write(b, p); - if (binary) { - escapeCtl(*p); - } else if ((*p & 0x40) == 0) { - throw Exception("Invalid UTF-8 sequence"); - } else { - int more = 1; - uint32_t value = 0; - if ((*p & 0x20) != 0) { - more++; - if ((*p & 0x10) != 0) { - more++; - if ((*p & 0x08) != 0) { - throw Exception("Invalid UTF-8 sequence"); - } else { - value = *p & 0x07; - } - } else { - value = *p & 0x0f; - } - } else { - value = *p & 0x1f; - } - for (int i = 0; i < more; ++i) { - if (++p == e || (*p & 0xc0) != 0x80) { - throw Exception("Invalid UTF-8 sequence"); - } - value <<= 6; - value |= *p & 0x3f; - } - escapeUnicode(value); - } - } else { - switch (*p) { - case '\\': - case '"': - case '/': - escape(*p, b, p); - break; - case '\b': - escape('b', b, p); - break; - case '\f': - escape('f', b, p); - break; - case '\n': - escape('n', b, p); - break; - case '\r': - escape('r', b, p); - break; - case '\t': - escape('t', b, p); - break; - default: - if (std::iscntrl(*p, std::locale::classic())) { - write(b, p); - escapeCtl(*p); - break; - } else { - continue; - } - } - } - b = p + 1; - } - write(b, e); - out_.write('"'); - } - - void sep() { - if (top == stArrayN) { - out_.write(','); - formatter_.handleValueEnd(); - } else if (top == stArray0) { - top = stArrayN; - } - } - - void sep2() { - if (top == stKey) { - top = stMapN; - } - } - -public: - JsonGenerator() : formatter_(out_), top(stStart) { } - - void init(OutputStream& os) { - out_.reset(os); - } - - void flush() { - out_.flush(); - } - - int64_t byteCount() const { - return out_.byteCount(); - } - - void encodeNull() { - sep(); - out_.writeBytes(reinterpret_cast<const uint8_t*>("null"), 4); - sep2(); - } - - void encodeBool(bool b) { - sep(); - if (b) { - out_.writeBytes(reinterpret_cast<const uint8_t*>("true"), 4); - } else { - out_.writeBytes(reinterpret_cast<const uint8_t*>("false"), 5); - } - sep2(); - } - - template <typename T> - void encodeNumber(T t) { - sep(); - std::ostringstream oss; - oss << boost::lexical_cast<std::string>(t); - const std::string& s = oss.str(); - out_.writeBytes(reinterpret_cast<const uint8_t*>(s.data()), s.size()); - sep2(); - } - - void encodeNumber(double t) { - sep(); - std::ostringstream oss; - if (boost::math::isfinite(t)) { - oss << boost::lexical_cast<std::string>(t); - } else if (boost::math::isnan(t)) { - oss << "NaN"; - } else if (t == std::numeric_limits<double>::infinity()) { - oss << "Infinity"; - } else { - oss << "-Infinity"; - } - const std::string& s = oss.str(); - out_.writeBytes(reinterpret_cast<const uint8_t*>(s.data()), s.size()); - sep2(); - } - - - void encodeString(const std::string& s) { - if (top == stMap0) { - top = stKey; - } else if (top == stMapN) { - out_.write(','); - formatter_.handleValueEnd(); - top = stKey; - } else if (top == stKey) { - top = stMapN; - } else { - sep(); - } - doEncodeString(s.c_str(), s.size(), false); - if (top == stKey) { - out_.write(':'); - formatter_.handleColon(); - } - } - - void encodeBinary(const uint8_t* bytes, size_t len) { - sep(); - doEncodeString(reinterpret_cast<const char *>(bytes), len, true); - sep2(); - } - - void arrayStart() { - sep(); - stateStack.push(top); - top = stArray0; - out_.write('['); - formatter_.handleObjectStart(); - } - - void arrayEnd() { - top = stateStack.top(); - stateStack.pop(); - formatter_.handleObjectEnd(); - out_.write(']'); - sep2(); - } - - void objectStart() { - sep(); - stateStack.push(top); - top = stMap0; - out_.write('{'); - formatter_.handleObjectStart(); - } - - void objectEnd() { - top = stateStack.top(); - stateStack.pop(); - formatter_.handleObjectEnd(); - out_.write('}'); - sep2(); - } - -}; - -} -} - -#endif +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_json_JsonIO_hh__ +#define avro_json_JsonIO_hh__ + +#include <locale> +#include <stack> +#include <string> +#include <sstream> +#include <boost/math/special_functions/fpclassify.hpp> +#include <boost/lexical_cast.hpp> +#include <boost/utility.hpp> + +#include "Config.hh" +#include "Stream.hh" + +namespace avro { +namespace json { + +inline char toHex(unsigned int n) { + return (n < 10) ? (n + '0') : (n + 'a' - 10); +} + + +class AVRO_DECL JsonParser : boost::noncopyable { +public: + enum Token { + tkNull, + tkBool, + tkLong, + tkDouble, + tkString, + tkArrayStart, + tkArrayEnd, + tkObjectStart, + tkObjectEnd + }; + + size_t line() const { return line_; } + +private: + enum State { + stValue, // Expect a data type + stArray0, // Expect a data type or ']' + stArrayN, // Expect a ',' or ']' + stObject0, // Expect a string or a '}' + stObjectN, // Expect a ',' or '}' + stKey // Expect a ':' + }; + std::stack<State> stateStack; + State curState; + bool hasNext; + char nextChar; + bool peeked; + + StreamReader in_; + Token curToken; + bool bv; + int64_t lv; + double dv; + std::string sv; + size_t line_; + + Token doAdvance(); + Token tryLiteral(const char exp[], size_t n, Token tk); + Token tryNumber(char ch); + Token tryString(); + Exception unexpected(unsigned char ch); + char next(); + + static std::string decodeString(const std::string& s, bool binary); + +public: + JsonParser() : curState(stValue), hasNext(false), peeked(false), line_(1) { } + + void init(InputStream& is) { + // Clear by swapping with an empty stack + std::stack<State>().swap(stateStack); + curState = stValue; + hasNext = false; + peeked = false; + line_ = 1; + in_.reset(is); + } + + Token advance() { + if (! peeked) { + curToken = doAdvance(); + } else { + peeked = false; + } + return curToken; + } + + Token peek() { + if (! peeked) { + curToken = doAdvance(); + peeked = true; + } + return curToken; + } + + void expectToken(Token tk); + + bool boolValue() const { + return bv; + } + + Token cur() const { + return curToken; + } + + double doubleValue() const { + return dv; + } + + int64_t longValue() const { + return lv; + } + + const std::string& rawString() const { + return sv; + } + + std::string stringValue() const { + return decodeString(sv, false); + } + + std::string bytesValue() const { + return decodeString(sv, true); + } + + void drain() { + if (!stateStack.empty() || peeked) { + throw Exception("Invalid state for draining"); + } + in_.drain(hasNext); + hasNext = false; + } + + /** + * Return UTF-8 encoded string value. + */ + static std::string toStringValue(const std::string& sv) { + return decodeString(sv, false); + } + + /** + * Return byte-encoded string value. It is an error if the input + * JSON string contained unicode characters more than "\u00ff'. + */ + static std::string toBytesValue(const std::string& sv) { + return decodeString(sv, true); + } + + static const char* const tokenNames[]; + + static const char* toString(Token tk) { + return tokenNames[tk]; + } +}; + +class AVRO_DECL JsonNullFormatter { +public: + JsonNullFormatter(StreamWriter&) { } + + void handleObjectStart() {} + void handleObjectEnd() {} + void handleValueEnd() {} + void handleColon() {} +}; + +class AVRO_DECL JsonPrettyFormatter { + StreamWriter& out_; + size_t level_; + std::vector<uint8_t> indent_; + + static const int CHARS_PER_LEVEL = 2; + + void printIndent() { + size_t charsToIndent = level_ * CHARS_PER_LEVEL; + if (indent_.size() < charsToIndent) { + indent_.resize(charsToIndent * 2, ' '); + } + out_.writeBytes(indent_.data(), charsToIndent); + } +public: + JsonPrettyFormatter(StreamWriter& out) : out_(out), level_(0), indent_(10, ' ') { } + + void handleObjectStart() { + out_.write('\n'); + ++level_; + printIndent(); + } + + void handleObjectEnd() { + out_.write('\n'); + --level_; + printIndent(); + } + + void handleValueEnd() { + out_.write('\n'); + printIndent(); + } + + void handleColon() { + out_.write(' '); + } +}; + +template <class F> +class AVRO_DECL JsonGenerator { + StreamWriter out_; + F formatter_; + enum State { + stStart, + stArray0, + stArrayN, + stMap0, + stMapN, + stKey, + }; + + std::stack<State> stateStack; + State top; + + void write(const char *b, const char* p) { + if (b != p) { + out_.writeBytes(reinterpret_cast<const uint8_t*>(b), p - b); + } + } + + void escape(char c, const char* b, const char *p) { + write(b, p); + out_.write('\\'); + out_.write(c); + } + + void escapeCtl(char c) { + escapeUnicode(static_cast<uint8_t>(c)); + } + + void writeHex(char c) { + out_.write(toHex((static_cast<unsigned char>(c)) / 16)); + out_.write(toHex((static_cast<unsigned char>(c)) % 16)); + } + + void escapeUnicode(uint32_t c) { + out_.write('\\'); + out_.write('u'); + writeHex((c >> 8) & 0xff); + writeHex(c & 0xff); + } + void doEncodeString(const char* b, size_t len, bool binary) { + const char* e = b + len; + out_.write('"'); + for (const char* p = b; p != e; p++) { + if ((*p & 0x80) != 0) { + write(b, p); + if (binary) { + escapeCtl(*p); + } else if ((*p & 0x40) == 0) { + throw Exception("Invalid UTF-8 sequence"); + } else { + int more = 1; + uint32_t value = 0; + if ((*p & 0x20) != 0) { + more++; + if ((*p & 0x10) != 0) { + more++; + if ((*p & 0x08) != 0) { + throw Exception("Invalid UTF-8 sequence"); + } else { + value = *p & 0x07; + } + } else { + value = *p & 0x0f; + } + } else { + value = *p & 0x1f; + } + for (int i = 0; i < more; ++i) { + if (++p == e || (*p & 0xc0) != 0x80) { + throw Exception("Invalid UTF-8 sequence"); + } + value <<= 6; + value |= *p & 0x3f; + } + escapeUnicode(value); + } + } else { + switch (*p) { + case '\\': + case '"': + case '/': + escape(*p, b, p); + break; + case '\b': + escape('b', b, p); + break; + case '\f': + escape('f', b, p); + break; + case '\n': + escape('n', b, p); + break; + case '\r': + escape('r', b, p); + break; + case '\t': + escape('t', b, p); + break; + default: + if (std::iscntrl(*p, std::locale::classic())) { + write(b, p); + escapeCtl(*p); + break; + } else { + continue; + } + } + } + b = p + 1; + } + write(b, e); + out_.write('"'); + } + + void sep() { + if (top == stArrayN) { + out_.write(','); + formatter_.handleValueEnd(); + } else if (top == stArray0) { + top = stArrayN; + } + } + + void sep2() { + if (top == stKey) { + top = stMapN; + } + } + +public: + JsonGenerator() : formatter_(out_), top(stStart) { } + + void init(OutputStream& os) { + out_.reset(os); + } + + void flush() { + out_.flush(); + } + + int64_t byteCount() const { + return out_.byteCount(); + } + + void encodeNull() { + sep(); + out_.writeBytes(reinterpret_cast<const uint8_t*>("null"), 4); + sep2(); + } + + void encodeBool(bool b) { + sep(); + if (b) { + out_.writeBytes(reinterpret_cast<const uint8_t*>("true"), 4); + } else { + out_.writeBytes(reinterpret_cast<const uint8_t*>("false"), 5); + } + sep2(); + } + + template <typename T> + void encodeNumber(T t) { + sep(); + std::ostringstream oss; + oss << boost::lexical_cast<std::string>(t); + const std::string& s = oss.str(); + out_.writeBytes(reinterpret_cast<const uint8_t*>(s.data()), s.size()); + sep2(); + } + + void encodeNumber(double t) { + sep(); + std::ostringstream oss; + if (boost::math::isfinite(t)) { + oss << boost::lexical_cast<std::string>(t); + } else if (boost::math::isnan(t)) { + oss << "NaN"; + } else if (t == std::numeric_limits<double>::infinity()) { + oss << "Infinity"; + } else { + oss << "-Infinity"; + } + const std::string& s = oss.str(); + out_.writeBytes(reinterpret_cast<const uint8_t*>(s.data()), s.size()); + sep2(); + } + + + void encodeString(const std::string& s) { + if (top == stMap0) { + top = stKey; + } else if (top == stMapN) { + out_.write(','); + formatter_.handleValueEnd(); + top = stKey; + } else if (top == stKey) { + top = stMapN; + } else { + sep(); + } + doEncodeString(s.c_str(), s.size(), false); + if (top == stKey) { + out_.write(':'); + formatter_.handleColon(); + } + } + + void encodeBinary(const uint8_t* bytes, size_t len) { + sep(); + doEncodeString(reinterpret_cast<const char *>(bytes), len, true); + sep2(); + } + + void arrayStart() { + sep(); + stateStack.push(top); + top = stArray0; + out_.write('['); + formatter_.handleObjectStart(); + } + + void arrayEnd() { + top = stateStack.top(); + stateStack.pop(); + formatter_.handleObjectEnd(); + out_.write(']'); + sep2(); + } + + void objectStart() { + sep(); + stateStack.push(top); + top = stMap0; + out_.write('{'); + formatter_.handleObjectStart(); + } + + void objectEnd() { + top = stateStack.top(); + stateStack.pop(); + formatter_.handleObjectEnd(); + out_.write('}'); + sep2(); + } + +}; + +} +} + +#endif diff --git a/contrib/libs/apache/avro/impl/parsing/JsonCodec.cc b/contrib/libs/apache/avro/impl/parsing/JsonCodec.cc index 73271fca55..8bca2984ae 100644 --- a/contrib/libs/apache/avro/impl/parsing/JsonCodec.cc +++ b/contrib/libs/apache/avro/impl/parsing/JsonCodec.cc @@ -1,718 +1,718 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define __STDC_LIMIT_MACROS - -#include <string> -#include <map> -#include <algorithm> -#include <ctype.h> -#include <memory> -#include <boost/math/special_functions/fpclassify.hpp> - -#include "ValidatingCodec.hh" -#include "Symbol.hh" -#include "ValidSchema.hh" -#include "Decoder.hh" -#include "Encoder.hh" -#include "NodeImpl.hh" - -#include "../json/JsonIO.hh" - -namespace avro { - -namespace parsing { - -using std::make_shared; - -using std::map; -using std::vector; -using std::string; -using std::reverse; -using std::ostringstream; -using std::istringstream; - -using avro::json::JsonParser; -using avro::json::JsonGenerator; -using avro::json::JsonNullFormatter; - -class JsonGrammarGenerator : public ValidatingGrammarGenerator { - ProductionPtr doGenerate(const NodePtr& n, - std::map<NodePtr, ProductionPtr> &m); -}; - -static std::string nameOf(const NodePtr& n) -{ - if (n->hasName()) { - return n->name(); - } - std::ostringstream oss; - oss << n->type(); - return oss.str(); -} - -ProductionPtr JsonGrammarGenerator::doGenerate(const NodePtr& n, - std::map<NodePtr, ProductionPtr> &m) { - switch (n->type()) { - case AVRO_NULL: - case AVRO_BOOL: - case AVRO_INT: - case AVRO_LONG: - case AVRO_FLOAT: - case AVRO_DOUBLE: - case AVRO_STRING: - case AVRO_BYTES: - case AVRO_FIXED: - case AVRO_ARRAY: - case AVRO_MAP: - case AVRO_SYMBOLIC: - return ValidatingGrammarGenerator::doGenerate(n, m); - case AVRO_RECORD: - { - ProductionPtr result = make_shared<Production>(); - - m.erase(n); - - size_t c = n->leaves(); - result->reserve(2 + 2 * c); - result->push_back(Symbol::recordStartSymbol()); - for (size_t i = 0; i < c; ++i) { - const NodePtr& leaf = n->leafAt(i); - ProductionPtr v = doGenerate(leaf, m); - result->push_back(Symbol::fieldSymbol(n->nameAt(i))); - copy(v->rbegin(), v->rend(), back_inserter(*result)); - } - result->push_back(Symbol::recordEndSymbol()); - reverse(result->begin(), result->end()); - - m[n] = result; - return make_shared<Production>(1, Symbol::indirect(result)); - } - case AVRO_ENUM: - { - vector<string> nn; - size_t c = n->names(); - nn.reserve(c); - for (size_t i = 0; i < c; ++i) { - nn.push_back(n->nameAt(i)); - } - ProductionPtr result = make_shared<Production>(); - result->push_back(Symbol::nameListSymbol(nn)); - result->push_back(Symbol::enumSymbol()); - m[n] = result; - return result; - } - case AVRO_UNION: - { - size_t c = n->leaves(); - - vector<ProductionPtr> vv; - vv.reserve(c); - - vector<string> names; - names.reserve(c); - - for (size_t i = 0; i < c; ++i) { - const NodePtr& nn = n->leafAt(i); - ProductionPtr v = doGenerate(nn, m); - if (nn->type() != AVRO_NULL) { - ProductionPtr v2 = make_shared<Production>(); - v2->push_back(Symbol::recordEndSymbol()); - copy(v->begin(), v->end(), back_inserter(*v2)); - v.swap(v2); - } - vv.push_back(v); - names.push_back(nameOf(nn)); - } - ProductionPtr result = make_shared<Production>(); - result->push_back(Symbol::alternative(vv)); - result->push_back(Symbol::nameListSymbol(names)); - result->push_back(Symbol::unionSymbol()); - return result; - } - default: - throw Exception("Unknown node type"); - } -} - -static void expectToken(JsonParser& in, JsonParser::Token tk) -{ - in.expectToken(tk); -} - -class JsonDecoderHandler { - JsonParser& in_; -public: - JsonDecoderHandler(JsonParser& p) : in_(p) { } - size_t handle(const Symbol& s) { - switch (s.kind()) { - case Symbol::sRecordStart: - expectToken(in_, JsonParser::tkObjectStart); - break; - case Symbol::sRecordEnd: - expectToken(in_, JsonParser::tkObjectEnd); - break; - case Symbol::sField: - expectToken(in_, JsonParser::tkString); - if (s.extra<string>() != in_.stringValue()) { - throw Exception("Incorrect field"); - } - break; - default: - break; - } - return 0; - } -}; - -template <typename P> -class JsonDecoder : public Decoder { - JsonParser in_; - JsonDecoderHandler handler_; - P parser_; - - void init(InputStream& is); - void decodeNull(); - bool decodeBool(); - int32_t decodeInt(); - int64_t decodeLong(); - float decodeFloat(); - double decodeDouble(); - void decodeString(string& value); - void skipString(); - void decodeBytes(vector<uint8_t>& value); - void skipBytes(); - void decodeFixed(size_t n, vector<uint8_t>& value); - void skipFixed(size_t n); - size_t decodeEnum(); - size_t arrayStart(); - size_t arrayNext(); - size_t skipArray(); - size_t mapStart(); - size_t mapNext(); - size_t skipMap(); - size_t decodeUnionIndex(); - - void expect(JsonParser::Token tk); - void skipComposite(); - void drain(); -public: - - JsonDecoder(const ValidSchema& s) : - handler_(in_), - parser_(JsonGrammarGenerator().generate(s), NULL, handler_) { } - -}; - -template <typename P> -void JsonDecoder<P>::init(InputStream& is) -{ - in_.init(is); - parser_.reset(); -} - -template <typename P> -void JsonDecoder<P>::expect(JsonParser::Token tk) -{ - expectToken(in_, tk); -} - -template <typename P> -void JsonDecoder<P>::decodeNull() -{ - parser_.advance(Symbol::sNull); - expect(JsonParser::tkNull); -} - -template <typename P> -bool JsonDecoder<P>::decodeBool() -{ - parser_.advance(Symbol::sBool); - expect(JsonParser::tkBool); - bool result = in_.boolValue(); - return result; -} - -template <typename P> -int32_t JsonDecoder<P>::decodeInt() -{ - parser_.advance(Symbol::sInt); - expect(JsonParser::tkLong); - int64_t result = in_.longValue(); - if (result < INT32_MIN || result > INT32_MAX) { - throw Exception(boost::format("Value out of range for Avro int: %1%") - % result); - } - return static_cast<int32_t>(result); -} - -template <typename P> -int64_t JsonDecoder<P>::decodeLong() -{ - parser_.advance(Symbol::sLong); - expect(JsonParser::tkLong); - int64_t result = in_.longValue(); - return result; -} - -template <typename P> -float JsonDecoder<P>::decodeFloat() -{ - parser_.advance(Symbol::sFloat); - expect(JsonParser::tkDouble); - double result = in_.doubleValue(); - return static_cast<float>(result); -} - -template <typename P> -double JsonDecoder<P>::decodeDouble() -{ - parser_.advance(Symbol::sDouble); - expect(JsonParser::tkDouble); - double result = in_.doubleValue(); - return result; -} - -template <typename P> -void JsonDecoder<P>::decodeString(string& value) -{ - parser_.advance(Symbol::sString); - expect(JsonParser::tkString); - value = in_.stringValue(); -} - -template <typename P> -void JsonDecoder<P>::skipString() -{ - parser_.advance(Symbol::sString); - expect(JsonParser::tkString); -} - -static vector<uint8_t> toBytes(const string& s) -{ - return vector<uint8_t>(s.begin(), s.end()); -} - -template <typename P> -void JsonDecoder<P>::decodeBytes(vector<uint8_t>& value ) -{ - parser_.advance(Symbol::sBytes); - expect(JsonParser::tkString); - value = toBytes(in_.bytesValue()); -} - -template <typename P> -void JsonDecoder<P>::skipBytes() -{ - parser_.advance(Symbol::sBytes); - expect(JsonParser::tkString); -} - -template <typename P> -void JsonDecoder<P>::decodeFixed(size_t n, vector<uint8_t>& value) -{ - parser_.advance(Symbol::sFixed); - parser_.assertSize(n); - expect(JsonParser::tkString); - value = toBytes(in_.bytesValue()); - if (value.size() != n) { - throw Exception("Incorrect value for fixed"); - } -} - -template <typename P> -void JsonDecoder<P>::skipFixed(size_t n) -{ - parser_.advance(Symbol::sFixed); - parser_.assertSize(n); - expect(JsonParser::tkString); - vector<uint8_t> result = toBytes(in_.bytesValue()); - if (result.size() != n) { - throw Exception("Incorrect value for fixed"); - } -} - -template <typename P> -size_t JsonDecoder<P>::decodeEnum() -{ - parser_.advance(Symbol::sEnum); - expect(JsonParser::tkString); - size_t result = parser_.indexForName(in_.stringValue()); - return result; -} - -template <typename P> -size_t JsonDecoder<P>::arrayStart() -{ - parser_.advance(Symbol::sArrayStart); - parser_.pushRepeatCount(0); - expect(JsonParser::tkArrayStart); - return arrayNext(); -} - -template <typename P> -size_t JsonDecoder<P>::arrayNext() -{ - parser_.processImplicitActions(); - if (in_.peek() == JsonParser::tkArrayEnd) { - in_.advance(); - parser_.popRepeater(); - parser_.advance(Symbol::sArrayEnd); - return 0; - } - parser_.nextRepeatCount(1); - return 1; -} - -template<typename P> -void JsonDecoder<P>::skipComposite() -{ - size_t level = 0; - for (; ;) { - switch (in_.advance()) { - case JsonParser::tkArrayStart: - case JsonParser::tkObjectStart: - ++level; - continue; - case JsonParser::tkArrayEnd: - case JsonParser::tkObjectEnd: - if (level == 0) { - return; - } - --level; - continue; - default: - continue; - } - } -} - -template<typename P> -void JsonDecoder<P>::drain() -{ - parser_.processImplicitActions(); - in_.drain(); -} - -template <typename P> -size_t JsonDecoder<P>::skipArray() -{ - parser_.advance(Symbol::sArrayStart); - parser_.pop(); - parser_.advance(Symbol::sArrayEnd); - expect(JsonParser::tkArrayStart); - skipComposite(); - return 0; -} - -template <typename P> -size_t JsonDecoder<P>::mapStart() -{ - parser_.advance(Symbol::sMapStart); - parser_.pushRepeatCount(0); - expect(JsonParser::tkObjectStart); - return mapNext(); -} - -template <typename P> -size_t JsonDecoder<P>::mapNext() -{ - parser_.processImplicitActions(); - if (in_.peek() == JsonParser::tkObjectEnd) { - in_.advance(); - parser_.popRepeater(); - parser_.advance(Symbol::sMapEnd); - return 0; - } - parser_.nextRepeatCount(1); - return 1; -} - -template <typename P> -size_t JsonDecoder<P>::skipMap() -{ - parser_.advance(Symbol::sMapStart); - parser_.pop(); - parser_.advance(Symbol::sMapEnd); - expect(JsonParser::tkObjectStart); - skipComposite(); - return 0; -} - -template <typename P> -size_t JsonDecoder<P>::decodeUnionIndex() -{ - parser_.advance(Symbol::sUnion); - - size_t result; - if (in_.peek() == JsonParser::tkNull) { - result = parser_.indexForName("null"); - } else { - expect(JsonParser::tkObjectStart); - expect(JsonParser::tkString); - result = parser_.indexForName(in_.stringValue()); - } - parser_.selectBranch(result); - return result; -} - -template<typename F = JsonNullFormatter> -class JsonHandler { - JsonGenerator<F>& generator_; -public: - JsonHandler(JsonGenerator<F>& g) : generator_(g) { } - size_t handle(const Symbol& s) { - switch (s.kind()) { - case Symbol::sRecordStart: - generator_.objectStart(); - break; - case Symbol::sRecordEnd: - generator_.objectEnd(); - break; - case Symbol::sField: - generator_.encodeString(s.extra<string>()); - break; - default: - break; - } - return 0; - } -}; - -template <typename P, typename F = JsonNullFormatter> -class JsonEncoder : public Encoder { - JsonGenerator<F> out_; - JsonHandler<F> handler_; - P parser_; - - void init(OutputStream& os); - void flush(); - int64_t byteCount() const; - void encodeNull(); - void encodeBool(bool b); - void encodeInt(int32_t i); - void encodeLong(int64_t l); - void encodeFloat(float f); - void encodeDouble(double d); - void encodeString(const std::string& s); - void encodeBytes(const uint8_t *bytes, size_t len); - void encodeFixed(const uint8_t *bytes, size_t len); - void encodeEnum(size_t e); - void arrayStart(); - void arrayEnd(); - void mapStart(); - void mapEnd(); - void setItemCount(size_t count); - void startItem(); - void encodeUnionIndex(size_t e); -public: - JsonEncoder(const ValidSchema& schema) : - handler_(out_), - parser_(JsonGrammarGenerator().generate(schema), NULL, handler_) { } -}; - -template<typename P, typename F> -void JsonEncoder<P, F>::init(OutputStream& os) -{ - out_.init(os); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::flush() -{ - parser_.processImplicitActions(); - out_.flush(); -} - -template<typename P, typename F> -int64_t JsonEncoder<P, F>::byteCount() const -{ - return out_.byteCount(); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::encodeNull() -{ - parser_.advance(Symbol::sNull); - out_.encodeNull(); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::encodeBool(bool b) -{ - parser_.advance(Symbol::sBool); - out_.encodeBool(b); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::encodeInt(int32_t i) -{ - parser_.advance(Symbol::sInt); - out_.encodeNumber(i); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::encodeLong(int64_t l) -{ - parser_.advance(Symbol::sLong); - out_.encodeNumber(l); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::encodeFloat(float f) -{ - parser_.advance(Symbol::sFloat); - if (f == std::numeric_limits<float>::infinity()) { - out_.encodeString("Infinity"); - } else if (f == -std::numeric_limits<float>::infinity()) { - out_.encodeString("-Infinity"); - } else if (boost::math::isnan(f)) { - out_.encodeString("NaN"); - } else { - out_.encodeNumber(f); - } -} - -template<typename P, typename F> -void JsonEncoder<P, F>::encodeDouble(double d) -{ - parser_.advance(Symbol::sDouble); - if (d == std::numeric_limits<double>::infinity()) { - out_.encodeString("Infinity"); - } else if (d == -std::numeric_limits<double>::infinity()) { - out_.encodeString("-Infinity"); - } else if (boost::math::isnan(d)) { - out_.encodeString("NaN"); - } else { - out_.encodeNumber(d); - } -} - -template<typename P, typename F> -void JsonEncoder<P, F>::encodeString(const std::string& s) -{ - parser_.advance(Symbol::sString); - out_.encodeString(s); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::encodeBytes(const uint8_t *bytes, size_t len) -{ - parser_.advance(Symbol::sBytes); - out_.encodeBinary(bytes, len); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::encodeFixed(const uint8_t *bytes, size_t len) -{ - parser_.advance(Symbol::sFixed); - parser_.assertSize(len); - out_.encodeBinary(bytes, len); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::encodeEnum(size_t e) -{ - parser_.advance(Symbol::sEnum); - const string& s = parser_.nameForIndex(e); - out_.encodeString(s); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::arrayStart() -{ - parser_.advance(Symbol::sArrayStart); - parser_.pushRepeatCount(0); - out_.arrayStart(); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::arrayEnd() -{ - parser_.popRepeater(); - parser_.advance(Symbol::sArrayEnd); - out_.arrayEnd(); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::mapStart() -{ - parser_.advance(Symbol::sMapStart); - parser_.pushRepeatCount(0); - out_.objectStart(); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::mapEnd() -{ - parser_.popRepeater(); - parser_.advance(Symbol::sMapEnd); - out_.objectEnd(); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::setItemCount(size_t count) -{ - parser_.nextRepeatCount(count); -} - -template<typename P, typename F> -void JsonEncoder<P, F>::startItem() -{ - parser_.processImplicitActions(); - if (parser_.top() != Symbol::sRepeater) { - throw Exception("startItem at not an item boundary"); - } -} - -template<typename P, typename F> -void JsonEncoder<P, F>::encodeUnionIndex(size_t e) -{ - parser_.advance(Symbol::sUnion); - - const std::string name = parser_.nameForIndex(e); - - if (name != "null") { - out_.objectStart(); - out_.encodeString(name); - } - parser_.selectBranch(e); -} - -} // namespace parsing - -DecoderPtr jsonDecoder(const ValidSchema& s) -{ - return std::make_shared<parsing::JsonDecoder< - parsing::SimpleParser<parsing::JsonDecoderHandler> > >(s); -} - -EncoderPtr jsonEncoder(const ValidSchema& schema) -{ - return std::make_shared<parsing::JsonEncoder< - parsing::SimpleParser<parsing::JsonHandler<avro::json::JsonNullFormatter> >, avro::json::JsonNullFormatter> >(schema); -} - -EncoderPtr jsonPrettyEncoder(const ValidSchema& schema) -{ - return std::make_shared<parsing::JsonEncoder< - parsing::SimpleParser<parsing::JsonHandler<avro::json::JsonPrettyFormatter> >, avro::json::JsonPrettyFormatter> >(schema); -} - -} // namespace avro - +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define __STDC_LIMIT_MACROS + +#include <string> +#include <map> +#include <algorithm> +#include <ctype.h> +#include <memory> +#include <boost/math/special_functions/fpclassify.hpp> + +#include "ValidatingCodec.hh" +#include "Symbol.hh" +#include "ValidSchema.hh" +#include "Decoder.hh" +#include "Encoder.hh" +#include "NodeImpl.hh" + +#include "../json/JsonIO.hh" + +namespace avro { + +namespace parsing { + +using std::make_shared; + +using std::map; +using std::vector; +using std::string; +using std::reverse; +using std::ostringstream; +using std::istringstream; + +using avro::json::JsonParser; +using avro::json::JsonGenerator; +using avro::json::JsonNullFormatter; + +class JsonGrammarGenerator : public ValidatingGrammarGenerator { + ProductionPtr doGenerate(const NodePtr& n, + std::map<NodePtr, ProductionPtr> &m); +}; + +static std::string nameOf(const NodePtr& n) +{ + if (n->hasName()) { + return n->name(); + } + std::ostringstream oss; + oss << n->type(); + return oss.str(); +} + +ProductionPtr JsonGrammarGenerator::doGenerate(const NodePtr& n, + std::map<NodePtr, ProductionPtr> &m) { + switch (n->type()) { + case AVRO_NULL: + case AVRO_BOOL: + case AVRO_INT: + case AVRO_LONG: + case AVRO_FLOAT: + case AVRO_DOUBLE: + case AVRO_STRING: + case AVRO_BYTES: + case AVRO_FIXED: + case AVRO_ARRAY: + case AVRO_MAP: + case AVRO_SYMBOLIC: + return ValidatingGrammarGenerator::doGenerate(n, m); + case AVRO_RECORD: + { + ProductionPtr result = make_shared<Production>(); + + m.erase(n); + + size_t c = n->leaves(); + result->reserve(2 + 2 * c); + result->push_back(Symbol::recordStartSymbol()); + for (size_t i = 0; i < c; ++i) { + const NodePtr& leaf = n->leafAt(i); + ProductionPtr v = doGenerate(leaf, m); + result->push_back(Symbol::fieldSymbol(n->nameAt(i))); + copy(v->rbegin(), v->rend(), back_inserter(*result)); + } + result->push_back(Symbol::recordEndSymbol()); + reverse(result->begin(), result->end()); + + m[n] = result; + return make_shared<Production>(1, Symbol::indirect(result)); + } + case AVRO_ENUM: + { + vector<string> nn; + size_t c = n->names(); + nn.reserve(c); + for (size_t i = 0; i < c; ++i) { + nn.push_back(n->nameAt(i)); + } + ProductionPtr result = make_shared<Production>(); + result->push_back(Symbol::nameListSymbol(nn)); + result->push_back(Symbol::enumSymbol()); + m[n] = result; + return result; + } + case AVRO_UNION: + { + size_t c = n->leaves(); + + vector<ProductionPtr> vv; + vv.reserve(c); + + vector<string> names; + names.reserve(c); + + for (size_t i = 0; i < c; ++i) { + const NodePtr& nn = n->leafAt(i); + ProductionPtr v = doGenerate(nn, m); + if (nn->type() != AVRO_NULL) { + ProductionPtr v2 = make_shared<Production>(); + v2->push_back(Symbol::recordEndSymbol()); + copy(v->begin(), v->end(), back_inserter(*v2)); + v.swap(v2); + } + vv.push_back(v); + names.push_back(nameOf(nn)); + } + ProductionPtr result = make_shared<Production>(); + result->push_back(Symbol::alternative(vv)); + result->push_back(Symbol::nameListSymbol(names)); + result->push_back(Symbol::unionSymbol()); + return result; + } + default: + throw Exception("Unknown node type"); + } +} + +static void expectToken(JsonParser& in, JsonParser::Token tk) +{ + in.expectToken(tk); +} + +class JsonDecoderHandler { + JsonParser& in_; +public: + JsonDecoderHandler(JsonParser& p) : in_(p) { } + size_t handle(const Symbol& s) { + switch (s.kind()) { + case Symbol::sRecordStart: + expectToken(in_, JsonParser::tkObjectStart); + break; + case Symbol::sRecordEnd: + expectToken(in_, JsonParser::tkObjectEnd); + break; + case Symbol::sField: + expectToken(in_, JsonParser::tkString); + if (s.extra<string>() != in_.stringValue()) { + throw Exception("Incorrect field"); + } + break; + default: + break; + } + return 0; + } +}; + +template <typename P> +class JsonDecoder : public Decoder { + JsonParser in_; + JsonDecoderHandler handler_; + P parser_; + + void init(InputStream& is); + void decodeNull(); + bool decodeBool(); + int32_t decodeInt(); + int64_t decodeLong(); + float decodeFloat(); + double decodeDouble(); + void decodeString(string& value); + void skipString(); + void decodeBytes(vector<uint8_t>& value); + void skipBytes(); + void decodeFixed(size_t n, vector<uint8_t>& value); + void skipFixed(size_t n); + size_t decodeEnum(); + size_t arrayStart(); + size_t arrayNext(); + size_t skipArray(); + size_t mapStart(); + size_t mapNext(); + size_t skipMap(); + size_t decodeUnionIndex(); + + void expect(JsonParser::Token tk); + void skipComposite(); + void drain(); +public: + + JsonDecoder(const ValidSchema& s) : + handler_(in_), + parser_(JsonGrammarGenerator().generate(s), NULL, handler_) { } + +}; + +template <typename P> +void JsonDecoder<P>::init(InputStream& is) +{ + in_.init(is); + parser_.reset(); +} + +template <typename P> +void JsonDecoder<P>::expect(JsonParser::Token tk) +{ + expectToken(in_, tk); +} + +template <typename P> +void JsonDecoder<P>::decodeNull() +{ + parser_.advance(Symbol::sNull); + expect(JsonParser::tkNull); +} + +template <typename P> +bool JsonDecoder<P>::decodeBool() +{ + parser_.advance(Symbol::sBool); + expect(JsonParser::tkBool); + bool result = in_.boolValue(); + return result; +} + +template <typename P> +int32_t JsonDecoder<P>::decodeInt() +{ + parser_.advance(Symbol::sInt); + expect(JsonParser::tkLong); + int64_t result = in_.longValue(); + if (result < INT32_MIN || result > INT32_MAX) { + throw Exception(boost::format("Value out of range for Avro int: %1%") + % result); + } + return static_cast<int32_t>(result); +} + +template <typename P> +int64_t JsonDecoder<P>::decodeLong() +{ + parser_.advance(Symbol::sLong); + expect(JsonParser::tkLong); + int64_t result = in_.longValue(); + return result; +} + +template <typename P> +float JsonDecoder<P>::decodeFloat() +{ + parser_.advance(Symbol::sFloat); + expect(JsonParser::tkDouble); + double result = in_.doubleValue(); + return static_cast<float>(result); +} + +template <typename P> +double JsonDecoder<P>::decodeDouble() +{ + parser_.advance(Symbol::sDouble); + expect(JsonParser::tkDouble); + double result = in_.doubleValue(); + return result; +} + +template <typename P> +void JsonDecoder<P>::decodeString(string& value) +{ + parser_.advance(Symbol::sString); + expect(JsonParser::tkString); + value = in_.stringValue(); +} + +template <typename P> +void JsonDecoder<P>::skipString() +{ + parser_.advance(Symbol::sString); + expect(JsonParser::tkString); +} + +static vector<uint8_t> toBytes(const string& s) +{ + return vector<uint8_t>(s.begin(), s.end()); +} + +template <typename P> +void JsonDecoder<P>::decodeBytes(vector<uint8_t>& value ) +{ + parser_.advance(Symbol::sBytes); + expect(JsonParser::tkString); + value = toBytes(in_.bytesValue()); +} + +template <typename P> +void JsonDecoder<P>::skipBytes() +{ + parser_.advance(Symbol::sBytes); + expect(JsonParser::tkString); +} + +template <typename P> +void JsonDecoder<P>::decodeFixed(size_t n, vector<uint8_t>& value) +{ + parser_.advance(Symbol::sFixed); + parser_.assertSize(n); + expect(JsonParser::tkString); + value = toBytes(in_.bytesValue()); + if (value.size() != n) { + throw Exception("Incorrect value for fixed"); + } +} + +template <typename P> +void JsonDecoder<P>::skipFixed(size_t n) +{ + parser_.advance(Symbol::sFixed); + parser_.assertSize(n); + expect(JsonParser::tkString); + vector<uint8_t> result = toBytes(in_.bytesValue()); + if (result.size() != n) { + throw Exception("Incorrect value for fixed"); + } +} + +template <typename P> +size_t JsonDecoder<P>::decodeEnum() +{ + parser_.advance(Symbol::sEnum); + expect(JsonParser::tkString); + size_t result = parser_.indexForName(in_.stringValue()); + return result; +} + +template <typename P> +size_t JsonDecoder<P>::arrayStart() +{ + parser_.advance(Symbol::sArrayStart); + parser_.pushRepeatCount(0); + expect(JsonParser::tkArrayStart); + return arrayNext(); +} + +template <typename P> +size_t JsonDecoder<P>::arrayNext() +{ + parser_.processImplicitActions(); + if (in_.peek() == JsonParser::tkArrayEnd) { + in_.advance(); + parser_.popRepeater(); + parser_.advance(Symbol::sArrayEnd); + return 0; + } + parser_.nextRepeatCount(1); + return 1; +} + +template<typename P> +void JsonDecoder<P>::skipComposite() +{ + size_t level = 0; + for (; ;) { + switch (in_.advance()) { + case JsonParser::tkArrayStart: + case JsonParser::tkObjectStart: + ++level; + continue; + case JsonParser::tkArrayEnd: + case JsonParser::tkObjectEnd: + if (level == 0) { + return; + } + --level; + continue; + default: + continue; + } + } +} + +template<typename P> +void JsonDecoder<P>::drain() +{ + parser_.processImplicitActions(); + in_.drain(); +} + +template <typename P> +size_t JsonDecoder<P>::skipArray() +{ + parser_.advance(Symbol::sArrayStart); + parser_.pop(); + parser_.advance(Symbol::sArrayEnd); + expect(JsonParser::tkArrayStart); + skipComposite(); + return 0; +} + +template <typename P> +size_t JsonDecoder<P>::mapStart() +{ + parser_.advance(Symbol::sMapStart); + parser_.pushRepeatCount(0); + expect(JsonParser::tkObjectStart); + return mapNext(); +} + +template <typename P> +size_t JsonDecoder<P>::mapNext() +{ + parser_.processImplicitActions(); + if (in_.peek() == JsonParser::tkObjectEnd) { + in_.advance(); + parser_.popRepeater(); + parser_.advance(Symbol::sMapEnd); + return 0; + } + parser_.nextRepeatCount(1); + return 1; +} + +template <typename P> +size_t JsonDecoder<P>::skipMap() +{ + parser_.advance(Symbol::sMapStart); + parser_.pop(); + parser_.advance(Symbol::sMapEnd); + expect(JsonParser::tkObjectStart); + skipComposite(); + return 0; +} + +template <typename P> +size_t JsonDecoder<P>::decodeUnionIndex() +{ + parser_.advance(Symbol::sUnion); + + size_t result; + if (in_.peek() == JsonParser::tkNull) { + result = parser_.indexForName("null"); + } else { + expect(JsonParser::tkObjectStart); + expect(JsonParser::tkString); + result = parser_.indexForName(in_.stringValue()); + } + parser_.selectBranch(result); + return result; +} + +template<typename F = JsonNullFormatter> +class JsonHandler { + JsonGenerator<F>& generator_; +public: + JsonHandler(JsonGenerator<F>& g) : generator_(g) { } + size_t handle(const Symbol& s) { + switch (s.kind()) { + case Symbol::sRecordStart: + generator_.objectStart(); + break; + case Symbol::sRecordEnd: + generator_.objectEnd(); + break; + case Symbol::sField: + generator_.encodeString(s.extra<string>()); + break; + default: + break; + } + return 0; + } +}; + +template <typename P, typename F = JsonNullFormatter> +class JsonEncoder : public Encoder { + JsonGenerator<F> out_; + JsonHandler<F> handler_; + P parser_; + + void init(OutputStream& os); + void flush(); + int64_t byteCount() const; + void encodeNull(); + void encodeBool(bool b); + void encodeInt(int32_t i); + void encodeLong(int64_t l); + void encodeFloat(float f); + void encodeDouble(double d); + void encodeString(const std::string& s); + void encodeBytes(const uint8_t *bytes, size_t len); + void encodeFixed(const uint8_t *bytes, size_t len); + void encodeEnum(size_t e); + void arrayStart(); + void arrayEnd(); + void mapStart(); + void mapEnd(); + void setItemCount(size_t count); + void startItem(); + void encodeUnionIndex(size_t e); +public: + JsonEncoder(const ValidSchema& schema) : + handler_(out_), + parser_(JsonGrammarGenerator().generate(schema), NULL, handler_) { } +}; + +template<typename P, typename F> +void JsonEncoder<P, F>::init(OutputStream& os) +{ + out_.init(os); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::flush() +{ + parser_.processImplicitActions(); + out_.flush(); +} + +template<typename P, typename F> +int64_t JsonEncoder<P, F>::byteCount() const +{ + return out_.byteCount(); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::encodeNull() +{ + parser_.advance(Symbol::sNull); + out_.encodeNull(); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::encodeBool(bool b) +{ + parser_.advance(Symbol::sBool); + out_.encodeBool(b); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::encodeInt(int32_t i) +{ + parser_.advance(Symbol::sInt); + out_.encodeNumber(i); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::encodeLong(int64_t l) +{ + parser_.advance(Symbol::sLong); + out_.encodeNumber(l); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::encodeFloat(float f) +{ + parser_.advance(Symbol::sFloat); + if (f == std::numeric_limits<float>::infinity()) { + out_.encodeString("Infinity"); + } else if (f == -std::numeric_limits<float>::infinity()) { + out_.encodeString("-Infinity"); + } else if (boost::math::isnan(f)) { + out_.encodeString("NaN"); + } else { + out_.encodeNumber(f); + } +} + +template<typename P, typename F> +void JsonEncoder<P, F>::encodeDouble(double d) +{ + parser_.advance(Symbol::sDouble); + if (d == std::numeric_limits<double>::infinity()) { + out_.encodeString("Infinity"); + } else if (d == -std::numeric_limits<double>::infinity()) { + out_.encodeString("-Infinity"); + } else if (boost::math::isnan(d)) { + out_.encodeString("NaN"); + } else { + out_.encodeNumber(d); + } +} + +template<typename P, typename F> +void JsonEncoder<P, F>::encodeString(const std::string& s) +{ + parser_.advance(Symbol::sString); + out_.encodeString(s); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::encodeBytes(const uint8_t *bytes, size_t len) +{ + parser_.advance(Symbol::sBytes); + out_.encodeBinary(bytes, len); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::encodeFixed(const uint8_t *bytes, size_t len) +{ + parser_.advance(Symbol::sFixed); + parser_.assertSize(len); + out_.encodeBinary(bytes, len); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::encodeEnum(size_t e) +{ + parser_.advance(Symbol::sEnum); + const string& s = parser_.nameForIndex(e); + out_.encodeString(s); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::arrayStart() +{ + parser_.advance(Symbol::sArrayStart); + parser_.pushRepeatCount(0); + out_.arrayStart(); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::arrayEnd() +{ + parser_.popRepeater(); + parser_.advance(Symbol::sArrayEnd); + out_.arrayEnd(); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::mapStart() +{ + parser_.advance(Symbol::sMapStart); + parser_.pushRepeatCount(0); + out_.objectStart(); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::mapEnd() +{ + parser_.popRepeater(); + parser_.advance(Symbol::sMapEnd); + out_.objectEnd(); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::setItemCount(size_t count) +{ + parser_.nextRepeatCount(count); +} + +template<typename P, typename F> +void JsonEncoder<P, F>::startItem() +{ + parser_.processImplicitActions(); + if (parser_.top() != Symbol::sRepeater) { + throw Exception("startItem at not an item boundary"); + } +} + +template<typename P, typename F> +void JsonEncoder<P, F>::encodeUnionIndex(size_t e) +{ + parser_.advance(Symbol::sUnion); + + const std::string name = parser_.nameForIndex(e); + + if (name != "null") { + out_.objectStart(); + out_.encodeString(name); + } + parser_.selectBranch(e); +} + +} // namespace parsing + +DecoderPtr jsonDecoder(const ValidSchema& s) +{ + return std::make_shared<parsing::JsonDecoder< + parsing::SimpleParser<parsing::JsonDecoderHandler> > >(s); +} + +EncoderPtr jsonEncoder(const ValidSchema& schema) +{ + return std::make_shared<parsing::JsonEncoder< + parsing::SimpleParser<parsing::JsonHandler<avro::json::JsonNullFormatter> >, avro::json::JsonNullFormatter> >(schema); +} + +EncoderPtr jsonPrettyEncoder(const ValidSchema& schema) +{ + return std::make_shared<parsing::JsonEncoder< + parsing::SimpleParser<parsing::JsonHandler<avro::json::JsonPrettyFormatter> >, avro::json::JsonPrettyFormatter> >(schema); +} + +} // namespace avro + diff --git a/contrib/libs/apache/avro/impl/parsing/ResolvingDecoder.cc b/contrib/libs/apache/avro/impl/parsing/ResolvingDecoder.cc index 2e33eaa8d0..f6dbacabcf 100644 --- a/contrib/libs/apache/avro/impl/parsing/ResolvingDecoder.cc +++ b/contrib/libs/apache/avro/impl/parsing/ResolvingDecoder.cc @@ -1,740 +1,740 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define __STDC_LIMIT_MACROS - -#include <string> -#include <stack> -#include <map> -#include <algorithm> -#include <memory> -#include <ctype.h> - -#include "ValidatingCodec.hh" -#include "Symbol.hh" -#include "Types.hh" -#include "ValidSchema.hh" -#include "Decoder.hh" -#include "Encoder.hh" -#include "NodeImpl.hh" -#include "Generic.hh" -#include "Stream.hh" - -namespace avro { - -using std::make_shared; - -namespace parsing { - -using std::shared_ptr; -using std::static_pointer_cast; -using std::make_shared; - -using std::unique_ptr; -using std::map; -using std::pair; -using std::vector; -using std::string; -using std::reverse; -using std::ostringstream; -using std::istringstream; -using std::stack; -using std::find_if; -using std::make_pair; - -typedef pair<NodePtr, NodePtr> NodePair; - -class ResolvingGrammarGenerator : public ValidatingGrammarGenerator { - ProductionPtr doGenerate2(const NodePtr& writer, - const NodePtr& reader, map<NodePair, ProductionPtr> &m, - map<NodePtr, ProductionPtr> &m2); - ProductionPtr resolveRecords(const NodePtr& writer, - const NodePtr& reader, map<NodePair, ProductionPtr> &m, - map<NodePtr, ProductionPtr> &m2); - ProductionPtr resolveUnion(const NodePtr& writer, - const NodePtr& reader, map<NodePair, ProductionPtr> &m, - map<NodePtr, ProductionPtr> &m2); - - static vector<pair<string, size_t> > fields(const NodePtr& n) { - vector<pair<string, size_t> > result; - size_t c = n->names(); - for (size_t i = 0; i < c; ++i) { - result.push_back(make_pair(n->nameAt(i), i)); - } - return result; - } - - static int bestBranch(const NodePtr& writer, const NodePtr& reader); - - ProductionPtr getWriterProduction(const NodePtr& n, - map<NodePtr, ProductionPtr>& m2); - -public: - Symbol generate( - const ValidSchema& writer, const ValidSchema& reader); -}; - -Symbol ResolvingGrammarGenerator::generate( - const ValidSchema& writer, const ValidSchema& reader) { - map<NodePtr, ProductionPtr> m2; - - const NodePtr& rr = reader.root(); - const NodePtr& rw = writer.root(); - ProductionPtr backup = ValidatingGrammarGenerator::doGenerate(rw, m2); - fixup(backup, m2); - - map<NodePair, ProductionPtr> m; - ProductionPtr main = doGenerate2(rw, rr, m, m2); - fixup(main, m); - return Symbol::rootSymbol(main, backup); -} - -int ResolvingGrammarGenerator::bestBranch(const NodePtr& writer, - const NodePtr& reader) -{ - Type t = writer->type(); - - const size_t c = reader->leaves(); - for (size_t j = 0; j < c; ++j) { - NodePtr r = reader->leafAt(j); - if (r->type() == AVRO_SYMBOLIC) { - r = resolveSymbol(r); - } - if (t == r->type()) { - if (r->hasName()) { - if (r->name() == writer->name()) { - return j; - } - } else { - return j; - } - } - } - - for (size_t j = 0; j < c; ++j) { - const NodePtr& r = reader->leafAt(j); - Type rt = r->type(); - switch (t) { - case AVRO_INT: - if (rt == AVRO_LONG || rt == AVRO_DOUBLE || rt == AVRO_FLOAT) { - return j; - } - break; - case AVRO_LONG: - case AVRO_FLOAT: - if (rt == AVRO_DOUBLE) { - return j; - } - break; - default: - break; - } - } - return -1; -} - -static shared_ptr<vector<uint8_t> > getAvroBinary( - const GenericDatum& defaultValue) -{ - EncoderPtr e = binaryEncoder(); - unique_ptr<OutputStream> os = memoryOutputStream(); - e->init(*os); - GenericWriter::write(*e, defaultValue); - e->flush(); - return snapshot(*os); -} - -template<typename T1, typename T2> -struct equalsFirst -{ - const T1& v_; - equalsFirst(const T1& v) : v_(v) { } - bool operator()(const pair<T1, T2>& p) { - return p.first == v_; - } -}; - -ProductionPtr ResolvingGrammarGenerator::getWriterProduction( - const NodePtr& n, map<NodePtr, ProductionPtr>& m2) -{ - const NodePtr& nn = (n->type() == AVRO_SYMBOLIC) ? - static_cast<const NodeSymbolic& >(*n).getNode() : n; - map<NodePtr, ProductionPtr>::const_iterator it2 = m2.find(nn); - if (it2 != m2.end()) { - return it2->second; - } else { - ProductionPtr result = ValidatingGrammarGenerator::doGenerate(nn, m2); - fixup(result, m2); - return result; - } -} - -ProductionPtr ResolvingGrammarGenerator::resolveRecords( - const NodePtr& writer, const NodePtr& reader, - map<NodePair, ProductionPtr>& m, - map<NodePtr, ProductionPtr>& m2) -{ - ProductionPtr result = make_shared<Production>(); - - vector<pair<string, size_t> > wf = fields(writer); - vector<pair<string, size_t> > rf = fields(reader); - vector<size_t> fieldOrder; - fieldOrder.reserve(reader->names()); - - /* - * We look for all writer fields in the reader. If found, recursively - * resolve the corresponding fields. Then erase the reader field. - * If no matching field is found for reader, arrange to skip the writer - * field. - */ - for (vector<pair<string, size_t> >::const_iterator it = wf.begin(); - it != wf.end(); ++it) { - vector<pair<string, size_t> >::iterator it2 = - find_if(rf.begin(), rf.end(), - equalsFirst<string, size_t>(it->first)); - if (it2 != rf.end()) { - ProductionPtr p = doGenerate2(writer->leafAt(it->second), - reader->leafAt(it2->second), m, m2); - copy(p->rbegin(), p->rend(), back_inserter(*result)); - fieldOrder.push_back(it2->second); - rf.erase(it2); - } else { - ProductionPtr p = getWriterProduction( - writer->leafAt(it->second), m2); - result->push_back(Symbol::skipStart()); - if (p->size() == 1) { - result->push_back((*p)[0]); - } else { - result->push_back(Symbol::indirect(p)); - } - } - } - - /* - * Examine the reader fields left out, (i.e. those didn't have corresponding - * writer field). - */ - for (vector<pair<string, size_t> >::const_iterator it = rf.begin(); - it != rf.end(); ++it) { - - NodePtr s = reader->leafAt(it->second); - fieldOrder.push_back(it->second); - - if (s->type() == AVRO_SYMBOLIC) { - s = resolveSymbol(s); - } - shared_ptr<vector<uint8_t> > defaultBinary = - getAvroBinary(reader->defaultValueAt(it->second)); - result->push_back(Symbol::defaultStartAction(defaultBinary)); - map<NodePair, shared_ptr<Production> >::const_iterator it2 = - m.find(NodePair(s, s)); - ProductionPtr p = (it2 == m.end()) ? - doGenerate2(s, s, m, m2) : it2->second; - copy(p->rbegin(), p->rend(), back_inserter(*result)); - result->push_back(Symbol::defaultEndAction()); - } - reverse(result->begin(), result->end()); - result->push_back(Symbol::sizeListAction(fieldOrder)); - result->push_back(Symbol::recordAction()); - - return result; - -} - -ProductionPtr ResolvingGrammarGenerator::resolveUnion( - const NodePtr& writer, const NodePtr& reader, - map<NodePair, ProductionPtr>& m, - map<NodePtr, ProductionPtr>& m2) -{ - vector<ProductionPtr> v; - size_t c = writer->leaves(); - v.reserve(c); - for (size_t i = 0; i < c; ++i) { - ProductionPtr p = doGenerate2(writer->leafAt(i), reader, m, m2); - v.push_back(p); - } - ProductionPtr result = make_shared<Production>(); - result->push_back(Symbol::alternative(v)); - result->push_back(Symbol::writerUnionAction()); - return result; -} - -ProductionPtr ResolvingGrammarGenerator::doGenerate2( - const NodePtr& w, const NodePtr& r, - map<NodePair, ProductionPtr> &m, - map<NodePtr, ProductionPtr> &m2) -{ - const NodePtr writer = w->type() == AVRO_SYMBOLIC ? resolveSymbol(w) : w; - const NodePtr reader = r->type() == AVRO_SYMBOLIC ? resolveSymbol(r) : r; - Type writerType = writer->type(); - Type readerType = reader->type(); - - if (writerType == readerType) { - switch (writerType) { - case AVRO_NULL: - return make_shared<Production>(1, Symbol::nullSymbol()); - case AVRO_BOOL: - return make_shared<Production>(1, Symbol::boolSymbol()); - case AVRO_INT: - return make_shared<Production>(1, Symbol::intSymbol()); - case AVRO_LONG: - return make_shared<Production>(1, Symbol::longSymbol()); - case AVRO_FLOAT: - return make_shared<Production>(1, Symbol::floatSymbol()); - case AVRO_DOUBLE: - return make_shared<Production>(1, Symbol::doubleSymbol()); - case AVRO_STRING: - return make_shared<Production>(1, Symbol::stringSymbol()); - case AVRO_BYTES: - return make_shared<Production>(1, Symbol::bytesSymbol()); - case AVRO_FIXED: - if (writer->name() == reader->name() && - writer->fixedSize() == reader->fixedSize()) { - ProductionPtr result = make_shared<Production>(); - result->push_back(Symbol::sizeCheckSymbol(reader->fixedSize())); - result->push_back(Symbol::fixedSymbol()); - m[make_pair(writer, reader)] = result; - return result; - } - break; - case AVRO_RECORD: - if (writer->name() == reader->name()) { - const pair<NodePtr, NodePtr> key(writer, reader); - map<NodePair, ProductionPtr>::const_iterator kp = m.find(key); - if (kp != m.end()) { - return (kp->second) ? kp->second : - make_shared<Production>(1, Symbol::placeholder(key)); - } - m[key] = ProductionPtr(); - ProductionPtr result = resolveRecords(writer, reader, m, m2); - m[key] = result; - return make_shared<Production>(1, Symbol::indirect(result)); - } - break; - - case AVRO_ENUM: - if (writer->name() == reader->name()) { - ProductionPtr result = make_shared<Production>(); - result->push_back(Symbol::enumAdjustSymbol(writer, reader)); - result->push_back(Symbol::enumSymbol()); - m[make_pair(writer, reader)] = result; - return result; - } - break; - - case AVRO_ARRAY: - { - ProductionPtr p = getWriterProduction(writer->leafAt(0), m2); - ProductionPtr p2 = doGenerate2(writer->leafAt(0), reader->leafAt(0), m, m2); - ProductionPtr result = make_shared<Production>(); - result->push_back(Symbol::arrayEndSymbol()); - result->push_back(Symbol::repeater(p2, p, true)); - result->push_back(Symbol::arrayStartSymbol()); - return result; - } - case AVRO_MAP: - { - ProductionPtr pp = - doGenerate2(writer->leafAt(1),reader->leafAt(1), m, m2); - ProductionPtr v(new Production(*pp)); - v->push_back(Symbol::stringSymbol()); - - ProductionPtr pp2 = getWriterProduction(writer->leafAt(1), m2); - ProductionPtr v2(new Production(*pp2)); - - v2->push_back(Symbol::stringSymbol()); - - ProductionPtr result = make_shared<Production>(); - result->push_back(Symbol::mapEndSymbol()); - result->push_back(Symbol::repeater(v, v2, false)); - result->push_back(Symbol::mapStartSymbol()); - return result; - } - case AVRO_UNION: - return resolveUnion(writer, reader, m, m2); - case AVRO_SYMBOLIC: - { - shared_ptr<NodeSymbolic> w = - static_pointer_cast<NodeSymbolic>(writer); - shared_ptr<NodeSymbolic> r = - static_pointer_cast<NodeSymbolic>(reader); - NodePair p(w->getNode(), r->getNode()); - map<NodePair, ProductionPtr>::iterator it = m.find(p); - if (it != m.end() && it->second) { - return it->second; - } else { - m[p] = ProductionPtr(); - return make_shared<Production>(1, Symbol::placeholder(p)); - } - } - default: - throw Exception("Unknown node type"); - } - } else if (writerType == AVRO_UNION) { - return resolveUnion(writer, reader, m, m2); - } else { - switch (readerType) { - case AVRO_LONG: - if (writerType == AVRO_INT) { - return make_shared<Production>(1, - Symbol::resolveSymbol(Symbol::sInt, Symbol::sLong)); - } - break; - case AVRO_FLOAT: - if (writerType == AVRO_INT || writerType == AVRO_LONG) { - return make_shared<Production>(1, - Symbol::resolveSymbol(writerType == AVRO_INT ? - Symbol::sInt : Symbol::sLong, Symbol::sFloat)); - } - break; - case AVRO_DOUBLE: - if (writerType == AVRO_INT || writerType == AVRO_LONG - || writerType == AVRO_FLOAT) { - return make_shared<Production>(1, - Symbol::resolveSymbol(writerType == AVRO_INT ? - Symbol::sInt : writerType == AVRO_LONG ? - Symbol::sLong : Symbol::sFloat, Symbol::sDouble)); - } - break; - - case AVRO_UNION: - { - int j = bestBranch(writer, reader); - if (j >= 0) { - ProductionPtr p = doGenerate2(writer, reader->leafAt(j), m, m2); - ProductionPtr result = make_shared<Production>(); - result->push_back(Symbol::unionAdjustSymbol(j, p)); - result->push_back(Symbol::unionSymbol()); - return result; - } - } - break; - case AVRO_NULL: - case AVRO_BOOL: - case AVRO_INT: - case AVRO_STRING: - case AVRO_BYTES: - case AVRO_ENUM: - case AVRO_ARRAY: - case AVRO_MAP: - case AVRO_RECORD: - break; - default: - throw Exception("Unknown node type"); - } - } - return make_shared<Production>(1, Symbol::error(writer, reader)); -} - -class ResolvingDecoderHandler { - shared_ptr<vector<uint8_t> > defaultData_; - unique_ptr<InputStream> inp_; - DecoderPtr backup_; - DecoderPtr& base_; - const DecoderPtr binDecoder; - public: - ResolvingDecoderHandler(DecoderPtr& base) : base_(base), - binDecoder(binaryDecoder()) { } - size_t handle(const Symbol& s) { - switch (s.kind()) { - case Symbol::sWriterUnion: - return base_->decodeUnionIndex(); - case Symbol::sDefaultStart: - defaultData_ = s.extra<shared_ptr<vector<uint8_t> > >(); - backup_ = base_; - inp_ = memoryInputStream(&(*defaultData_)[0], defaultData_->size()); - base_ = binDecoder; - base_->init(*inp_); - return 0; - case Symbol::sDefaultEnd: - base_= backup_; - backup_.reset(); - return 0; - default: - return 0; - } - } - - void reset() - { - if (backup_ != NULL) - { - base_= backup_; - backup_.reset(); - } - } -}; - -template <typename Parser> -class ResolvingDecoderImpl : public ResolvingDecoder -{ - DecoderPtr base_; - ResolvingDecoderHandler handler_; - Parser parser_; - - void init(InputStream& is); - void decodeNull(); - bool decodeBool(); - int32_t decodeInt(); - int64_t decodeLong(); - float decodeFloat(); - double decodeDouble(); - void decodeString(string& value); - void skipString(); - void decodeBytes(vector<uint8_t>& value); - void skipBytes(); - void decodeFixed(size_t n, vector<uint8_t>& value); - void skipFixed(size_t n); - size_t decodeEnum(); - size_t arrayStart(); - size_t arrayNext(); - size_t skipArray(); - size_t mapStart(); - size_t mapNext(); - size_t skipMap(); - size_t decodeUnionIndex(); - const vector<size_t>& fieldOrder(); - void drain() { - parser_.processImplicitActions(); - base_->drain(); - } -public: - ResolvingDecoderImpl(const ValidSchema& writer, const ValidSchema& reader, - const DecoderPtr& base) : - base_(base), - handler_(base_), - parser_(ResolvingGrammarGenerator().generate(writer, reader), - &(*base_), handler_) - { - } -}; - -template <typename P> -void ResolvingDecoderImpl<P>::init(InputStream& is) -{ - handler_.reset(); - base_->init(is); - parser_.reset(); -} - -template <typename P> -void ResolvingDecoderImpl<P>::decodeNull() -{ - parser_.advance(Symbol::sNull); - base_->decodeNull(); -} - -template <typename P> -bool ResolvingDecoderImpl<P>::decodeBool() -{ - parser_.advance(Symbol::sBool); - return base_->decodeBool(); -} - -template <typename P> -int32_t ResolvingDecoderImpl<P>::decodeInt() -{ - parser_.advance(Symbol::sInt); - return base_->decodeInt(); -} - -template <typename P> -int64_t ResolvingDecoderImpl<P>::decodeLong() -{ - Symbol::Kind k = parser_.advance(Symbol::sLong); - return k == Symbol::sInt ? base_->decodeInt() : base_->decodeLong(); -} - -template <typename P> -float ResolvingDecoderImpl<P>::decodeFloat() -{ - Symbol::Kind k = parser_.advance(Symbol::sFloat); - return k == Symbol::sInt ? base_->decodeInt() : - k == Symbol::sLong ? base_->decodeLong() : - base_->decodeFloat(); -} - -template <typename P> -double ResolvingDecoderImpl<P>::decodeDouble() -{ - Symbol::Kind k = parser_.advance(Symbol::sDouble); - return k == Symbol::sInt ? base_->decodeInt() : - k == Symbol::sLong ? base_->decodeLong() : - k == Symbol::sFloat ? base_->decodeFloat() : - base_->decodeDouble(); -} - -template <typename P> -void ResolvingDecoderImpl<P>::decodeString(string& value) -{ - parser_.advance(Symbol::sString); - base_->decodeString(value); -} - -template <typename P> -void ResolvingDecoderImpl<P>::skipString() -{ - parser_.advance(Symbol::sString); - base_->skipString(); -} - -template <typename P> -void ResolvingDecoderImpl<P>::decodeBytes(vector<uint8_t>& value) -{ - parser_.advance(Symbol::sBytes); - base_->decodeBytes(value); -} - -template <typename P> -void ResolvingDecoderImpl<P>::skipBytes() -{ - parser_.advance(Symbol::sBytes); - base_->skipBytes(); -} - -template <typename P> -void ResolvingDecoderImpl<P>::decodeFixed(size_t n, vector<uint8_t>& value) -{ - parser_.advance(Symbol::sFixed); - parser_.assertSize(n); - return base_->decodeFixed(n, value); -} - -template <typename P> -void ResolvingDecoderImpl<P>::skipFixed(size_t n) -{ - parser_.advance(Symbol::sFixed); - parser_.assertSize(n); - base_->skipFixed(n); -} - -template <typename P> -size_t ResolvingDecoderImpl<P>::decodeEnum() -{ - parser_.advance(Symbol::sEnum); - size_t n = base_->decodeEnum(); - return parser_.enumAdjust(n); -} - -template <typename P> -size_t ResolvingDecoderImpl<P>::arrayStart() -{ - parser_.advance(Symbol::sArrayStart); - size_t result = base_->arrayStart(); - parser_.pushRepeatCount(result); - if (result == 0) { - parser_.popRepeater(); - parser_.advance(Symbol::sArrayEnd); - } - return result; -} - -template <typename P> -size_t ResolvingDecoderImpl<P>::arrayNext() -{ - parser_.processImplicitActions(); - size_t result = base_->arrayNext(); - parser_.nextRepeatCount(result); - if (result == 0) { - parser_.popRepeater(); - parser_.advance(Symbol::sArrayEnd); - } - return result; -} - -template <typename P> -size_t ResolvingDecoderImpl<P>::skipArray() -{ - parser_.advance(Symbol::sArrayStart); - size_t n = base_->skipArray(); - if (n == 0) { - parser_.pop(); - } else { - parser_.pushRepeatCount(n); - parser_.skip(*base_); - } - parser_.advance(Symbol::sArrayEnd); - return 0; -} - -template <typename P> -size_t ResolvingDecoderImpl<P>::mapStart() -{ - parser_.advance(Symbol::sMapStart); - size_t result = base_->mapStart(); - parser_.pushRepeatCount(result); - if (result == 0) { - parser_.popRepeater(); - parser_.advance(Symbol::sMapEnd); - } - return result; -} - -template <typename P> -size_t ResolvingDecoderImpl<P>::mapNext() -{ - parser_.processImplicitActions(); - size_t result = base_->mapNext(); - parser_.nextRepeatCount(result); - if (result == 0) { - parser_.popRepeater(); - parser_.advance(Symbol::sMapEnd); - } - return result; -} - -template <typename P> -size_t ResolvingDecoderImpl<P>::skipMap() -{ - parser_.advance(Symbol::sMapStart); - size_t n = base_->skipMap(); - if (n == 0) { - parser_.pop(); - } else { - parser_.pushRepeatCount(n); - parser_.skip(*base_); - } - parser_.advance(Symbol::sMapEnd); - return 0; -} - -template <typename P> -size_t ResolvingDecoderImpl<P>::decodeUnionIndex() -{ - parser_.advance(Symbol::sUnion); - return parser_.unionAdjust(); -} - -template <typename P> -const vector<size_t>& ResolvingDecoderImpl<P>::fieldOrder() -{ - parser_.advance(Symbol::sRecord); - return parser_.sizeList(); -} - -} // namespace parsing - -ResolvingDecoderPtr resolvingDecoder(const ValidSchema& writer, - const ValidSchema& reader, const DecoderPtr& base) { - return make_shared<parsing::ResolvingDecoderImpl - <parsing::SimpleParser<parsing::ResolvingDecoderHandler> > >( - writer, reader, base); -} - -} // namespace avro - +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define __STDC_LIMIT_MACROS + +#include <string> +#include <stack> +#include <map> +#include <algorithm> +#include <memory> +#include <ctype.h> + +#include "ValidatingCodec.hh" +#include "Symbol.hh" +#include "Types.hh" +#include "ValidSchema.hh" +#include "Decoder.hh" +#include "Encoder.hh" +#include "NodeImpl.hh" +#include "Generic.hh" +#include "Stream.hh" + +namespace avro { + +using std::make_shared; + +namespace parsing { + +using std::shared_ptr; +using std::static_pointer_cast; +using std::make_shared; + +using std::unique_ptr; +using std::map; +using std::pair; +using std::vector; +using std::string; +using std::reverse; +using std::ostringstream; +using std::istringstream; +using std::stack; +using std::find_if; +using std::make_pair; + +typedef pair<NodePtr, NodePtr> NodePair; + +class ResolvingGrammarGenerator : public ValidatingGrammarGenerator { + ProductionPtr doGenerate2(const NodePtr& writer, + const NodePtr& reader, map<NodePair, ProductionPtr> &m, + map<NodePtr, ProductionPtr> &m2); + ProductionPtr resolveRecords(const NodePtr& writer, + const NodePtr& reader, map<NodePair, ProductionPtr> &m, + map<NodePtr, ProductionPtr> &m2); + ProductionPtr resolveUnion(const NodePtr& writer, + const NodePtr& reader, map<NodePair, ProductionPtr> &m, + map<NodePtr, ProductionPtr> &m2); + + static vector<pair<string, size_t> > fields(const NodePtr& n) { + vector<pair<string, size_t> > result; + size_t c = n->names(); + for (size_t i = 0; i < c; ++i) { + result.push_back(make_pair(n->nameAt(i), i)); + } + return result; + } + + static int bestBranch(const NodePtr& writer, const NodePtr& reader); + + ProductionPtr getWriterProduction(const NodePtr& n, + map<NodePtr, ProductionPtr>& m2); + +public: + Symbol generate( + const ValidSchema& writer, const ValidSchema& reader); +}; + +Symbol ResolvingGrammarGenerator::generate( + const ValidSchema& writer, const ValidSchema& reader) { + map<NodePtr, ProductionPtr> m2; + + const NodePtr& rr = reader.root(); + const NodePtr& rw = writer.root(); + ProductionPtr backup = ValidatingGrammarGenerator::doGenerate(rw, m2); + fixup(backup, m2); + + map<NodePair, ProductionPtr> m; + ProductionPtr main = doGenerate2(rw, rr, m, m2); + fixup(main, m); + return Symbol::rootSymbol(main, backup); +} + +int ResolvingGrammarGenerator::bestBranch(const NodePtr& writer, + const NodePtr& reader) +{ + Type t = writer->type(); + + const size_t c = reader->leaves(); + for (size_t j = 0; j < c; ++j) { + NodePtr r = reader->leafAt(j); + if (r->type() == AVRO_SYMBOLIC) { + r = resolveSymbol(r); + } + if (t == r->type()) { + if (r->hasName()) { + if (r->name() == writer->name()) { + return j; + } + } else { + return j; + } + } + } + + for (size_t j = 0; j < c; ++j) { + const NodePtr& r = reader->leafAt(j); + Type rt = r->type(); + switch (t) { + case AVRO_INT: + if (rt == AVRO_LONG || rt == AVRO_DOUBLE || rt == AVRO_FLOAT) { + return j; + } + break; + case AVRO_LONG: + case AVRO_FLOAT: + if (rt == AVRO_DOUBLE) { + return j; + } + break; + default: + break; + } + } + return -1; +} + +static shared_ptr<vector<uint8_t> > getAvroBinary( + const GenericDatum& defaultValue) +{ + EncoderPtr e = binaryEncoder(); + unique_ptr<OutputStream> os = memoryOutputStream(); + e->init(*os); + GenericWriter::write(*e, defaultValue); + e->flush(); + return snapshot(*os); +} + +template<typename T1, typename T2> +struct equalsFirst +{ + const T1& v_; + equalsFirst(const T1& v) : v_(v) { } + bool operator()(const pair<T1, T2>& p) { + return p.first == v_; + } +}; + +ProductionPtr ResolvingGrammarGenerator::getWriterProduction( + const NodePtr& n, map<NodePtr, ProductionPtr>& m2) +{ + const NodePtr& nn = (n->type() == AVRO_SYMBOLIC) ? + static_cast<const NodeSymbolic& >(*n).getNode() : n; + map<NodePtr, ProductionPtr>::const_iterator it2 = m2.find(nn); + if (it2 != m2.end()) { + return it2->second; + } else { + ProductionPtr result = ValidatingGrammarGenerator::doGenerate(nn, m2); + fixup(result, m2); + return result; + } +} + +ProductionPtr ResolvingGrammarGenerator::resolveRecords( + const NodePtr& writer, const NodePtr& reader, + map<NodePair, ProductionPtr>& m, + map<NodePtr, ProductionPtr>& m2) +{ + ProductionPtr result = make_shared<Production>(); + + vector<pair<string, size_t> > wf = fields(writer); + vector<pair<string, size_t> > rf = fields(reader); + vector<size_t> fieldOrder; + fieldOrder.reserve(reader->names()); + + /* + * We look for all writer fields in the reader. If found, recursively + * resolve the corresponding fields. Then erase the reader field. + * If no matching field is found for reader, arrange to skip the writer + * field. + */ + for (vector<pair<string, size_t> >::const_iterator it = wf.begin(); + it != wf.end(); ++it) { + vector<pair<string, size_t> >::iterator it2 = + find_if(rf.begin(), rf.end(), + equalsFirst<string, size_t>(it->first)); + if (it2 != rf.end()) { + ProductionPtr p = doGenerate2(writer->leafAt(it->second), + reader->leafAt(it2->second), m, m2); + copy(p->rbegin(), p->rend(), back_inserter(*result)); + fieldOrder.push_back(it2->second); + rf.erase(it2); + } else { + ProductionPtr p = getWriterProduction( + writer->leafAt(it->second), m2); + result->push_back(Symbol::skipStart()); + if (p->size() == 1) { + result->push_back((*p)[0]); + } else { + result->push_back(Symbol::indirect(p)); + } + } + } + + /* + * Examine the reader fields left out, (i.e. those didn't have corresponding + * writer field). + */ + for (vector<pair<string, size_t> >::const_iterator it = rf.begin(); + it != rf.end(); ++it) { + + NodePtr s = reader->leafAt(it->second); + fieldOrder.push_back(it->second); + + if (s->type() == AVRO_SYMBOLIC) { + s = resolveSymbol(s); + } + shared_ptr<vector<uint8_t> > defaultBinary = + getAvroBinary(reader->defaultValueAt(it->second)); + result->push_back(Symbol::defaultStartAction(defaultBinary)); + map<NodePair, shared_ptr<Production> >::const_iterator it2 = + m.find(NodePair(s, s)); + ProductionPtr p = (it2 == m.end()) ? + doGenerate2(s, s, m, m2) : it2->second; + copy(p->rbegin(), p->rend(), back_inserter(*result)); + result->push_back(Symbol::defaultEndAction()); + } + reverse(result->begin(), result->end()); + result->push_back(Symbol::sizeListAction(fieldOrder)); + result->push_back(Symbol::recordAction()); + + return result; + +} + +ProductionPtr ResolvingGrammarGenerator::resolveUnion( + const NodePtr& writer, const NodePtr& reader, + map<NodePair, ProductionPtr>& m, + map<NodePtr, ProductionPtr>& m2) +{ + vector<ProductionPtr> v; + size_t c = writer->leaves(); + v.reserve(c); + for (size_t i = 0; i < c; ++i) { + ProductionPtr p = doGenerate2(writer->leafAt(i), reader, m, m2); + v.push_back(p); + } + ProductionPtr result = make_shared<Production>(); + result->push_back(Symbol::alternative(v)); + result->push_back(Symbol::writerUnionAction()); + return result; +} + +ProductionPtr ResolvingGrammarGenerator::doGenerate2( + const NodePtr& w, const NodePtr& r, + map<NodePair, ProductionPtr> &m, + map<NodePtr, ProductionPtr> &m2) +{ + const NodePtr writer = w->type() == AVRO_SYMBOLIC ? resolveSymbol(w) : w; + const NodePtr reader = r->type() == AVRO_SYMBOLIC ? resolveSymbol(r) : r; + Type writerType = writer->type(); + Type readerType = reader->type(); + + if (writerType == readerType) { + switch (writerType) { + case AVRO_NULL: + return make_shared<Production>(1, Symbol::nullSymbol()); + case AVRO_BOOL: + return make_shared<Production>(1, Symbol::boolSymbol()); + case AVRO_INT: + return make_shared<Production>(1, Symbol::intSymbol()); + case AVRO_LONG: + return make_shared<Production>(1, Symbol::longSymbol()); + case AVRO_FLOAT: + return make_shared<Production>(1, Symbol::floatSymbol()); + case AVRO_DOUBLE: + return make_shared<Production>(1, Symbol::doubleSymbol()); + case AVRO_STRING: + return make_shared<Production>(1, Symbol::stringSymbol()); + case AVRO_BYTES: + return make_shared<Production>(1, Symbol::bytesSymbol()); + case AVRO_FIXED: + if (writer->name() == reader->name() && + writer->fixedSize() == reader->fixedSize()) { + ProductionPtr result = make_shared<Production>(); + result->push_back(Symbol::sizeCheckSymbol(reader->fixedSize())); + result->push_back(Symbol::fixedSymbol()); + m[make_pair(writer, reader)] = result; + return result; + } + break; + case AVRO_RECORD: + if (writer->name() == reader->name()) { + const pair<NodePtr, NodePtr> key(writer, reader); + map<NodePair, ProductionPtr>::const_iterator kp = m.find(key); + if (kp != m.end()) { + return (kp->second) ? kp->second : + make_shared<Production>(1, Symbol::placeholder(key)); + } + m[key] = ProductionPtr(); + ProductionPtr result = resolveRecords(writer, reader, m, m2); + m[key] = result; + return make_shared<Production>(1, Symbol::indirect(result)); + } + break; + + case AVRO_ENUM: + if (writer->name() == reader->name()) { + ProductionPtr result = make_shared<Production>(); + result->push_back(Symbol::enumAdjustSymbol(writer, reader)); + result->push_back(Symbol::enumSymbol()); + m[make_pair(writer, reader)] = result; + return result; + } + break; + + case AVRO_ARRAY: + { + ProductionPtr p = getWriterProduction(writer->leafAt(0), m2); + ProductionPtr p2 = doGenerate2(writer->leafAt(0), reader->leafAt(0), m, m2); + ProductionPtr result = make_shared<Production>(); + result->push_back(Symbol::arrayEndSymbol()); + result->push_back(Symbol::repeater(p2, p, true)); + result->push_back(Symbol::arrayStartSymbol()); + return result; + } + case AVRO_MAP: + { + ProductionPtr pp = + doGenerate2(writer->leafAt(1),reader->leafAt(1), m, m2); + ProductionPtr v(new Production(*pp)); + v->push_back(Symbol::stringSymbol()); + + ProductionPtr pp2 = getWriterProduction(writer->leafAt(1), m2); + ProductionPtr v2(new Production(*pp2)); + + v2->push_back(Symbol::stringSymbol()); + + ProductionPtr result = make_shared<Production>(); + result->push_back(Symbol::mapEndSymbol()); + result->push_back(Symbol::repeater(v, v2, false)); + result->push_back(Symbol::mapStartSymbol()); + return result; + } + case AVRO_UNION: + return resolveUnion(writer, reader, m, m2); + case AVRO_SYMBOLIC: + { + shared_ptr<NodeSymbolic> w = + static_pointer_cast<NodeSymbolic>(writer); + shared_ptr<NodeSymbolic> r = + static_pointer_cast<NodeSymbolic>(reader); + NodePair p(w->getNode(), r->getNode()); + map<NodePair, ProductionPtr>::iterator it = m.find(p); + if (it != m.end() && it->second) { + return it->second; + } else { + m[p] = ProductionPtr(); + return make_shared<Production>(1, Symbol::placeholder(p)); + } + } + default: + throw Exception("Unknown node type"); + } + } else if (writerType == AVRO_UNION) { + return resolveUnion(writer, reader, m, m2); + } else { + switch (readerType) { + case AVRO_LONG: + if (writerType == AVRO_INT) { + return make_shared<Production>(1, + Symbol::resolveSymbol(Symbol::sInt, Symbol::sLong)); + } + break; + case AVRO_FLOAT: + if (writerType == AVRO_INT || writerType == AVRO_LONG) { + return make_shared<Production>(1, + Symbol::resolveSymbol(writerType == AVRO_INT ? + Symbol::sInt : Symbol::sLong, Symbol::sFloat)); + } + break; + case AVRO_DOUBLE: + if (writerType == AVRO_INT || writerType == AVRO_LONG + || writerType == AVRO_FLOAT) { + return make_shared<Production>(1, + Symbol::resolveSymbol(writerType == AVRO_INT ? + Symbol::sInt : writerType == AVRO_LONG ? + Symbol::sLong : Symbol::sFloat, Symbol::sDouble)); + } + break; + + case AVRO_UNION: + { + int j = bestBranch(writer, reader); + if (j >= 0) { + ProductionPtr p = doGenerate2(writer, reader->leafAt(j), m, m2); + ProductionPtr result = make_shared<Production>(); + result->push_back(Symbol::unionAdjustSymbol(j, p)); + result->push_back(Symbol::unionSymbol()); + return result; + } + } + break; + case AVRO_NULL: + case AVRO_BOOL: + case AVRO_INT: + case AVRO_STRING: + case AVRO_BYTES: + case AVRO_ENUM: + case AVRO_ARRAY: + case AVRO_MAP: + case AVRO_RECORD: + break; + default: + throw Exception("Unknown node type"); + } + } + return make_shared<Production>(1, Symbol::error(writer, reader)); +} + +class ResolvingDecoderHandler { + shared_ptr<vector<uint8_t> > defaultData_; + unique_ptr<InputStream> inp_; + DecoderPtr backup_; + DecoderPtr& base_; + const DecoderPtr binDecoder; + public: + ResolvingDecoderHandler(DecoderPtr& base) : base_(base), + binDecoder(binaryDecoder()) { } + size_t handle(const Symbol& s) { + switch (s.kind()) { + case Symbol::sWriterUnion: + return base_->decodeUnionIndex(); + case Symbol::sDefaultStart: + defaultData_ = s.extra<shared_ptr<vector<uint8_t> > >(); + backup_ = base_; + inp_ = memoryInputStream(&(*defaultData_)[0], defaultData_->size()); + base_ = binDecoder; + base_->init(*inp_); + return 0; + case Symbol::sDefaultEnd: + base_= backup_; + backup_.reset(); + return 0; + default: + return 0; + } + } + + void reset() + { + if (backup_ != NULL) + { + base_= backup_; + backup_.reset(); + } + } +}; + +template <typename Parser> +class ResolvingDecoderImpl : public ResolvingDecoder +{ + DecoderPtr base_; + ResolvingDecoderHandler handler_; + Parser parser_; + + void init(InputStream& is); + void decodeNull(); + bool decodeBool(); + int32_t decodeInt(); + int64_t decodeLong(); + float decodeFloat(); + double decodeDouble(); + void decodeString(string& value); + void skipString(); + void decodeBytes(vector<uint8_t>& value); + void skipBytes(); + void decodeFixed(size_t n, vector<uint8_t>& value); + void skipFixed(size_t n); + size_t decodeEnum(); + size_t arrayStart(); + size_t arrayNext(); + size_t skipArray(); + size_t mapStart(); + size_t mapNext(); + size_t skipMap(); + size_t decodeUnionIndex(); + const vector<size_t>& fieldOrder(); + void drain() { + parser_.processImplicitActions(); + base_->drain(); + } +public: + ResolvingDecoderImpl(const ValidSchema& writer, const ValidSchema& reader, + const DecoderPtr& base) : + base_(base), + handler_(base_), + parser_(ResolvingGrammarGenerator().generate(writer, reader), + &(*base_), handler_) + { + } +}; + +template <typename P> +void ResolvingDecoderImpl<P>::init(InputStream& is) +{ + handler_.reset(); + base_->init(is); + parser_.reset(); +} + +template <typename P> +void ResolvingDecoderImpl<P>::decodeNull() +{ + parser_.advance(Symbol::sNull); + base_->decodeNull(); +} + +template <typename P> +bool ResolvingDecoderImpl<P>::decodeBool() +{ + parser_.advance(Symbol::sBool); + return base_->decodeBool(); +} + +template <typename P> +int32_t ResolvingDecoderImpl<P>::decodeInt() +{ + parser_.advance(Symbol::sInt); + return base_->decodeInt(); +} + +template <typename P> +int64_t ResolvingDecoderImpl<P>::decodeLong() +{ + Symbol::Kind k = parser_.advance(Symbol::sLong); + return k == Symbol::sInt ? base_->decodeInt() : base_->decodeLong(); +} + +template <typename P> +float ResolvingDecoderImpl<P>::decodeFloat() +{ + Symbol::Kind k = parser_.advance(Symbol::sFloat); + return k == Symbol::sInt ? base_->decodeInt() : + k == Symbol::sLong ? base_->decodeLong() : + base_->decodeFloat(); +} + +template <typename P> +double ResolvingDecoderImpl<P>::decodeDouble() +{ + Symbol::Kind k = parser_.advance(Symbol::sDouble); + return k == Symbol::sInt ? base_->decodeInt() : + k == Symbol::sLong ? base_->decodeLong() : + k == Symbol::sFloat ? base_->decodeFloat() : + base_->decodeDouble(); +} + +template <typename P> +void ResolvingDecoderImpl<P>::decodeString(string& value) +{ + parser_.advance(Symbol::sString); + base_->decodeString(value); +} + +template <typename P> +void ResolvingDecoderImpl<P>::skipString() +{ + parser_.advance(Symbol::sString); + base_->skipString(); +} + +template <typename P> +void ResolvingDecoderImpl<P>::decodeBytes(vector<uint8_t>& value) +{ + parser_.advance(Symbol::sBytes); + base_->decodeBytes(value); +} + +template <typename P> +void ResolvingDecoderImpl<P>::skipBytes() +{ + parser_.advance(Symbol::sBytes); + base_->skipBytes(); +} + +template <typename P> +void ResolvingDecoderImpl<P>::decodeFixed(size_t n, vector<uint8_t>& value) +{ + parser_.advance(Symbol::sFixed); + parser_.assertSize(n); + return base_->decodeFixed(n, value); +} + +template <typename P> +void ResolvingDecoderImpl<P>::skipFixed(size_t n) +{ + parser_.advance(Symbol::sFixed); + parser_.assertSize(n); + base_->skipFixed(n); +} + +template <typename P> +size_t ResolvingDecoderImpl<P>::decodeEnum() +{ + parser_.advance(Symbol::sEnum); + size_t n = base_->decodeEnum(); + return parser_.enumAdjust(n); +} + +template <typename P> +size_t ResolvingDecoderImpl<P>::arrayStart() +{ + parser_.advance(Symbol::sArrayStart); + size_t result = base_->arrayStart(); + parser_.pushRepeatCount(result); + if (result == 0) { + parser_.popRepeater(); + parser_.advance(Symbol::sArrayEnd); + } + return result; +} + +template <typename P> +size_t ResolvingDecoderImpl<P>::arrayNext() +{ + parser_.processImplicitActions(); + size_t result = base_->arrayNext(); + parser_.nextRepeatCount(result); + if (result == 0) { + parser_.popRepeater(); + parser_.advance(Symbol::sArrayEnd); + } + return result; +} + +template <typename P> +size_t ResolvingDecoderImpl<P>::skipArray() +{ + parser_.advance(Symbol::sArrayStart); + size_t n = base_->skipArray(); + if (n == 0) { + parser_.pop(); + } else { + parser_.pushRepeatCount(n); + parser_.skip(*base_); + } + parser_.advance(Symbol::sArrayEnd); + return 0; +} + +template <typename P> +size_t ResolvingDecoderImpl<P>::mapStart() +{ + parser_.advance(Symbol::sMapStart); + size_t result = base_->mapStart(); + parser_.pushRepeatCount(result); + if (result == 0) { + parser_.popRepeater(); + parser_.advance(Symbol::sMapEnd); + } + return result; +} + +template <typename P> +size_t ResolvingDecoderImpl<P>::mapNext() +{ + parser_.processImplicitActions(); + size_t result = base_->mapNext(); + parser_.nextRepeatCount(result); + if (result == 0) { + parser_.popRepeater(); + parser_.advance(Symbol::sMapEnd); + } + return result; +} + +template <typename P> +size_t ResolvingDecoderImpl<P>::skipMap() +{ + parser_.advance(Symbol::sMapStart); + size_t n = base_->skipMap(); + if (n == 0) { + parser_.pop(); + } else { + parser_.pushRepeatCount(n); + parser_.skip(*base_); + } + parser_.advance(Symbol::sMapEnd); + return 0; +} + +template <typename P> +size_t ResolvingDecoderImpl<P>::decodeUnionIndex() +{ + parser_.advance(Symbol::sUnion); + return parser_.unionAdjust(); +} + +template <typename P> +const vector<size_t>& ResolvingDecoderImpl<P>::fieldOrder() +{ + parser_.advance(Symbol::sRecord); + return parser_.sizeList(); +} + +} // namespace parsing + +ResolvingDecoderPtr resolvingDecoder(const ValidSchema& writer, + const ValidSchema& reader, const DecoderPtr& base) { + return make_shared<parsing::ResolvingDecoderImpl + <parsing::SimpleParser<parsing::ResolvingDecoderHandler> > >( + writer, reader, base); +} + +} // namespace avro + diff --git a/contrib/libs/apache/avro/impl/parsing/Symbol.cc b/contrib/libs/apache/avro/impl/parsing/Symbol.cc index b59b965133..6eb83309be 100644 --- a/contrib/libs/apache/avro/impl/parsing/Symbol.cc +++ b/contrib/libs/apache/avro/impl/parsing/Symbol.cc @@ -1,111 +1,111 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include "Symbol.hh" - -namespace avro { -namespace parsing { - -using std::vector; -using std::string; -using std::ostringstream; - -const char* Symbol::stringValues[] = { - "TerminalLow", - "Null", - "Bool", - "Int", - "Long", - "Float", - "Double", - "String", - "Bytes", - "ArrayStart", - "ArrayEnd", - "MapStart", - "MapEnd", - "Fixed", - "Enum", - "Union", - "TerminalHigh", - "SizeCheck", - "NameList", - "Root", - "Repeater", - "Alternative", - "Placeholder", - "Indirect", - "Symbolic", - "EnumAdjust", - "UnionAdjust", - "SkipStart", - "Resolve", - "ImplicitActionLow", - "RecordStart", - "RecordEnd", - "Field", - "Record", - "SizeList", - "WriterUnion", - "DefaultStart", - "DefaultEnd", - "ImplicitActionHigh", - "Error" -}; - -Symbol Symbol::enumAdjustSymbol(const NodePtr& writer, const NodePtr& reader) -{ - vector<string> rs; - size_t rc = reader->names(); - for (size_t i = 0; i < rc; ++i) { - rs.push_back(reader->nameAt(i)); - } - - size_t wc = writer->names(); - vector<int> adj; - adj.reserve(wc); - - vector<string> err; - - for (size_t i = 0; i < wc; ++i) { - const string& s = writer->nameAt(i); - vector<string>::const_iterator it = find(rs.begin(), rs.end(), s); - if (it == rs.end()) { - int pos = err.size() + 1; - adj.push_back(-pos); - err.push_back(s); - } else { - adj.push_back(it - rs.begin()); - } - } - return Symbol(sEnumAdjust, make_pair(adj, err)); -} - -Symbol Symbol::error(const NodePtr& writer, const NodePtr& reader) -{ - ostringstream oss; - oss << "Cannot resolve: " << std::endl; - writer->printJson(oss, 0); - oss << std::endl << "with" << std::endl; - reader->printJson(oss, 0); - return Symbol(sError, oss.str()); -} - -} // namespace parsing -} // namespace avro +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include "Symbol.hh" + +namespace avro { +namespace parsing { + +using std::vector; +using std::string; +using std::ostringstream; + +const char* Symbol::stringValues[] = { + "TerminalLow", + "Null", + "Bool", + "Int", + "Long", + "Float", + "Double", + "String", + "Bytes", + "ArrayStart", + "ArrayEnd", + "MapStart", + "MapEnd", + "Fixed", + "Enum", + "Union", + "TerminalHigh", + "SizeCheck", + "NameList", + "Root", + "Repeater", + "Alternative", + "Placeholder", + "Indirect", + "Symbolic", + "EnumAdjust", + "UnionAdjust", + "SkipStart", + "Resolve", + "ImplicitActionLow", + "RecordStart", + "RecordEnd", + "Field", + "Record", + "SizeList", + "WriterUnion", + "DefaultStart", + "DefaultEnd", + "ImplicitActionHigh", + "Error" +}; + +Symbol Symbol::enumAdjustSymbol(const NodePtr& writer, const NodePtr& reader) +{ + vector<string> rs; + size_t rc = reader->names(); + for (size_t i = 0; i < rc; ++i) { + rs.push_back(reader->nameAt(i)); + } + + size_t wc = writer->names(); + vector<int> adj; + adj.reserve(wc); + + vector<string> err; + + for (size_t i = 0; i < wc; ++i) { + const string& s = writer->nameAt(i); + vector<string>::const_iterator it = find(rs.begin(), rs.end(), s); + if (it == rs.end()) { + int pos = err.size() + 1; + adj.push_back(-pos); + err.push_back(s); + } else { + adj.push_back(it - rs.begin()); + } + } + return Symbol(sEnumAdjust, make_pair(adj, err)); +} + +Symbol Symbol::error(const NodePtr& writer, const NodePtr& reader) +{ + ostringstream oss; + oss << "Cannot resolve: " << std::endl; + writer->printJson(oss, 0); + oss << std::endl << "with" << std::endl; + reader->printJson(oss, 0); + return Symbol(sError, oss.str()); +} + +} // namespace parsing +} // namespace avro diff --git a/contrib/libs/apache/avro/impl/parsing/Symbol.hh b/contrib/libs/apache/avro/impl/parsing/Symbol.hh index d642341e16..f4ecfe6e83 100644 --- a/contrib/libs/apache/avro/impl/parsing/Symbol.hh +++ b/contrib/libs/apache/avro/impl/parsing/Symbol.hh @@ -1,854 +1,854 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_parsing_Symbol_hh__ -#define avro_parsing_Symbol_hh__ - -#include <vector> -#include <map> -#include <set> -#include <stack> -#include <sstream> - -#include <boost/any.hpp> -#include <boost/tuple/tuple.hpp> - -#include "Node.hh" -#include "Decoder.hh" -#include "Exception.hh" - -namespace avro { -namespace parsing { - -class Symbol; - -typedef std::vector<Symbol> Production; -typedef std::shared_ptr<Production> ProductionPtr; -typedef boost::tuple<std::stack<ssize_t>, bool, ProductionPtr, ProductionPtr> RepeaterInfo; -typedef boost::tuple<ProductionPtr, ProductionPtr> RootInfo; - -class Symbol { -public: - enum Kind { - sTerminalLow, // extra has nothing - sNull, - sBool, - sInt, - sLong, - sFloat, - sDouble, - sString, - sBytes, - sArrayStart, - sArrayEnd, - sMapStart, - sMapEnd, - sFixed, - sEnum, - sUnion, - sTerminalHigh, - sSizeCheck, // Extra has size - sNameList, // Extra has a vector<string> - sRoot, // Root for a schema, extra is Symbol - sRepeater, // Array or Map, extra is symbol - sAlternative, // One of many (union), extra is Union - sPlaceholder, // To be fixed up later. - sIndirect, // extra is shared_ptr<Production> - sSymbolic, // extra is weal_ptr<Production> - sEnumAdjust, - sUnionAdjust, - sSkipStart, - sResolve, - - sImplicitActionLow, - sRecordStart, - sRecordEnd, - sField, // extra is string - sRecord, - sSizeList, - sWriterUnion, - sDefaultStart, // extra has default value in Avro binary encoding - sDefaultEnd, - sImplicitActionHigh, - sError - }; - -private: - Kind kind_; - boost::any extra_; - - - explicit Symbol(Kind k) : kind_(k) { } - template <typename T> Symbol(Kind k, T t) : kind_(k), extra_(t) { } -public: - - Kind kind() const { - return kind_; - } - - template <typename T> T extra() const { - return boost::any_cast<T>(extra_); - } - - template <typename T> T* extrap() { - return boost::any_cast<T>(&extra_); - } - - template <typename T> const T* extrap() const { - return boost::any_cast<T>(&extra_); - } - - template <typename T> void extra(const T& t) { - extra_ = t; - } - - bool isTerminal() const { - return kind_ > sTerminalLow && kind_ < sTerminalHigh; - } - - bool isImplicitAction() const { - return kind_ > sImplicitActionLow && kind_ < sImplicitActionHigh; - } - - static const char* stringValues[]; - static const char* toString(Kind k) { - return stringValues[k]; - } - - static Symbol rootSymbol(ProductionPtr& s) - { - return Symbol(Symbol::sRoot, RootInfo(s, std::make_shared<Production>())); - } - - static Symbol rootSymbol(const ProductionPtr& main, - const ProductionPtr& backup) - { - return Symbol(Symbol::sRoot, RootInfo(main, backup)); - } - - static Symbol nullSymbol() { - return Symbol(sNull); - } - - static Symbol boolSymbol() { - return Symbol(sBool); - } - - static Symbol intSymbol() { - return Symbol(sInt); - } - - static Symbol longSymbol() { - return Symbol(sLong); - } - - static Symbol floatSymbol() { - return Symbol(sFloat); - } - - static Symbol doubleSymbol() { - return Symbol(sDouble); - } - - static Symbol stringSymbol() { - return Symbol(sString); - } - - static Symbol bytesSymbol() { - return Symbol(sBytes); - } - - static Symbol sizeCheckSymbol(size_t s) { - return Symbol(sSizeCheck, s); - } - - static Symbol fixedSymbol() { - return Symbol(sFixed); - } - - static Symbol enumSymbol() { - return Symbol(sEnum); - } - - static Symbol arrayStartSymbol() { - return Symbol(sArrayStart); - } - - static Symbol arrayEndSymbol() { - return Symbol(sArrayEnd); - } - - static Symbol mapStartSymbol() { - return Symbol(sMapStart); - } - - static Symbol mapEndSymbol() { - return Symbol(sMapEnd); - } - - static Symbol repeater(const ProductionPtr& p, - bool isArray) { - return repeater(p, p, isArray); - } - - static Symbol repeater(const ProductionPtr& read, - const ProductionPtr& skip, - bool isArray) { - std::stack<ssize_t> s; - return Symbol(sRepeater, RepeaterInfo(s, isArray, read, skip)); - } - - static Symbol defaultStartAction(std::shared_ptr<std::vector<uint8_t> > bb) - { - return Symbol(sDefaultStart, bb); - } - - static Symbol defaultEndAction() - { - return Symbol(sDefaultEnd); - } - - static Symbol alternative( - const std::vector<ProductionPtr>& branches) - { - return Symbol(Symbol::sAlternative, branches); - } - - static Symbol unionSymbol() { - return Symbol(sUnion); - } - - static Symbol recordStartSymbol() { - return Symbol(sRecordStart); - } - - static Symbol recordEndSymbol() { - return Symbol(sRecordEnd); - } - - static Symbol fieldSymbol(const std::string& name) { - return Symbol(sField, name); - } - - static Symbol writerUnionAction() { - return Symbol(sWriterUnion); - } - - static Symbol nameListSymbol( - const std::vector<std::string>& v) { - return Symbol(sNameList, v); - } - - template <typename T> - static Symbol placeholder(const T& n) { - return Symbol(sPlaceholder, n); - } - - static Symbol indirect(const ProductionPtr& p) { - return Symbol(sIndirect, p); - } - - static Symbol symbolic(const std::weak_ptr<Production>& p) { - return Symbol(sSymbolic, p); - } - - static Symbol enumAdjustSymbol(const NodePtr& writer, - const NodePtr& reader); - - static Symbol unionAdjustSymbol(size_t branch, - const ProductionPtr& p) { - return Symbol(sUnionAdjust, std::make_pair(branch, p)); - } - - static Symbol sizeListAction(std::vector<size_t> order) { - return Symbol(sSizeList, order); - } - - static Symbol recordAction() { - return Symbol(sRecord); - } - - static Symbol error(const NodePtr& writer, const NodePtr& reader); - - static Symbol resolveSymbol(Kind w, Kind r) { - return Symbol(sResolve, std::make_pair(w, r)); - } - - static Symbol skipStart() { - return Symbol(sSkipStart); - } - -}; - -/** - * Recursively replaces all placeholders in the production with the - * corresponding values. - */ -template<typename T> -void fixup(const ProductionPtr& p, - const std::map<T, ProductionPtr> &m) -{ - std::set<ProductionPtr> seen; - for (Production::iterator it = p->begin(); it != p->end(); ++it) { - fixup(*it, m, seen); - } -} - - -/** - * Recursively replaces all placeholders in the symbol with the values with the - * corresponding values. - */ -template<typename T> -void fixup_internal(const ProductionPtr& p, - const std::map<T, ProductionPtr> &m, - std::set<ProductionPtr>& seen) -{ - if (seen.find(p) == seen.end()) { - seen.insert(p); - for (Production::iterator it = p->begin(); it != p->end(); ++it) { - fixup(*it, m, seen); - } - } -} - -template<typename T> -void fixup(Symbol& s, const std::map<T, ProductionPtr> &m, - std::set<ProductionPtr>& seen) -{ - switch (s.kind()) { - case Symbol::sIndirect: - fixup_internal(s.extra<ProductionPtr>(), m, seen); - break; - case Symbol::sAlternative: - { - const std::vector<ProductionPtr> *vv = - s.extrap<std::vector<ProductionPtr> >(); - for (std::vector<ProductionPtr>::const_iterator it = vv->begin(); - it != vv->end(); ++it) { - fixup_internal(*it, m, seen); - } - } - break; - case Symbol::sRepeater: - { - const RepeaterInfo& ri = *s.extrap<RepeaterInfo>(); - fixup_internal(boost::tuples::get<2>(ri), m, seen); - fixup_internal(boost::tuples::get<3>(ri), m, seen); - } - break; - case Symbol::sPlaceholder: - { - typename std::map<T, std::shared_ptr<Production> >::const_iterator it = - m.find(s.extra<T>()); - if (it == m.end()) { - throw Exception("Placeholder symbol cannot be resolved"); - } - s = Symbol::symbolic(std::weak_ptr<Production>(it->second)); - } - break; - case Symbol::sUnionAdjust: - fixup_internal(s.extrap<std::pair<size_t, ProductionPtr> >()->second, - m, seen); - break; - default: - break; - } -} - -template<typename Handler> -class SimpleParser { - Decoder* decoder_; - Handler& handler_; - std::stack<Symbol> parsingStack; - - static void throwMismatch(Symbol::Kind actual, Symbol::Kind expected) - { - std::ostringstream oss; - oss << "Invalid operation. Schema requires: " << - Symbol::toString(expected) << ", got: " << - Symbol::toString(actual); - throw Exception(oss.str()); - } - - static void assertMatch(Symbol::Kind actual, Symbol::Kind expected) - { - if (expected != actual) { - throwMismatch(actual, expected); - } - - } - - void append(const ProductionPtr& ss) { - for (Production::const_iterator it = ss->begin(); - it != ss->end(); ++it) { - parsingStack.push(*it); - } - } - - size_t popSize() { - const Symbol& s = parsingStack.top(); - assertMatch(Symbol::sSizeCheck, s.kind()); - size_t result = s.extra<size_t>(); - parsingStack.pop(); - return result; - } - - static void assertLessThan(size_t n, size_t s) { - if (n >= s) { - std::ostringstream oss; - oss << "Size max value. Upper bound: " << s << " found " << n; - throw Exception(oss.str()); - } - } - -public: - Symbol::Kind advance(Symbol::Kind k) { - for (; ;) { - Symbol& s = parsingStack.top(); -// std::cout << "advance: " << Symbol::toString(s.kind()) -// << " looking for " << Symbol::toString(k) << '\n'; - if (s.kind() == k) { - parsingStack.pop(); - return k; - } else if (s.isTerminal()) { - throwMismatch(k, s.kind()); - } else { - switch (s.kind()) { - case Symbol::sRoot: - append(boost::tuples::get<0>(*s.extrap<RootInfo>())); - continue; - case Symbol::sIndirect: - { - ProductionPtr pp = - s.extra<ProductionPtr>(); - parsingStack.pop(); - append(pp); - } - continue; - case Symbol::sSymbolic: - { - ProductionPtr pp( - s.extra<std::weak_ptr<Production> >()); - parsingStack.pop(); - append(pp); - } - continue; - case Symbol::sRepeater: - { - RepeaterInfo *p = s.extrap<RepeaterInfo>(); - std::stack<ssize_t>& ns = boost::tuples::get<0>(*p); - if (ns.empty()) { - throw Exception( - "Empty item count stack in repeater advance"); - } - if (ns.top() == 0) { - throw Exception( - "Zero item count in repeater advance"); - } - --ns.top(); - append(boost::tuples::get<2>(*p)); - } - continue; - case Symbol::sError: - throw Exception(s.extra<std::string>()); - case Symbol::sResolve: - { - const std::pair<Symbol::Kind, Symbol::Kind>* p = - s.extrap<std::pair<Symbol::Kind, Symbol::Kind> >(); - assertMatch(p->second, k); - Symbol::Kind result = p->first; - parsingStack.pop(); - return result; - } - case Symbol::sSkipStart: - parsingStack.pop(); - skip(*decoder_); - break; - default: - if (s.isImplicitAction()) { - size_t n = handler_.handle(s); - if (s.kind() == Symbol::sWriterUnion) { - parsingStack.pop(); - selectBranch(n); - } else { - parsingStack.pop(); - } - } else { - std::ostringstream oss; - oss << "Encountered " << Symbol::toString(s.kind()) - << " while looking for " << Symbol::toString(k); - throw Exception(oss.str()); - } - } - } - } - } - - void skip(Decoder& d) { - const size_t sz = parsingStack.size(); - if (sz == 0) { - throw Exception("Nothing to skip!"); - } - while (parsingStack.size() >= sz) { - Symbol& t = parsingStack.top(); - // std::cout << "skip: " << Symbol::toString(t.kind()) << '\n'; - switch (t.kind()) { - case Symbol::sNull: - d.decodeNull(); - break; - case Symbol::sBool: - d.decodeBool(); - break; - case Symbol::sInt: - d.decodeInt(); - break; - case Symbol::sLong: - d.decodeLong(); - break; - case Symbol::sFloat: - d.decodeFloat(); - break; - case Symbol::sDouble: - d.decodeDouble(); - break; - case Symbol::sString: - d.skipString(); - break; - case Symbol::sBytes: - d.skipBytes(); - break; - case Symbol::sArrayStart: - { - parsingStack.pop(); - size_t n = d.skipArray(); - processImplicitActions(); - assertMatch(Symbol::sRepeater, parsingStack.top().kind()); - if (n == 0) { - break; - } - Symbol& t = parsingStack.top(); - RepeaterInfo *p = t.extrap<RepeaterInfo>(); - boost::tuples::get<0>(*p).push(n); - continue; - } - case Symbol::sArrayEnd: - break; - case Symbol::sMapStart: - { - parsingStack.pop(); - size_t n = d.skipMap(); - processImplicitActions(); - assertMatch(Symbol::sRepeater, parsingStack.top().kind()); - if (n == 0) { - break; - } - Symbol& t = parsingStack.top(); - RepeaterInfo *p = t.extrap<RepeaterInfo>(); - boost::tuples::get<0>(*p).push(n); - continue; - } - case Symbol::sMapEnd: - break; - case Symbol::sFixed: - { - parsingStack.pop(); - Symbol& t = parsingStack.top(); - d.decodeFixed(t.extra<size_t>()); - } - break; - case Symbol::sEnum: - parsingStack.pop(); - d.decodeEnum(); - break; - case Symbol::sUnion: - { - parsingStack.pop(); - size_t n = d.decodeUnionIndex(); - selectBranch(n); - continue; - } - case Symbol::sRepeater: - { - RepeaterInfo *p = t.extrap<RepeaterInfo>(); - std::stack<ssize_t>& ns = boost::tuples::get<0>(*p); - if (ns.empty()) { - throw Exception( - "Empty item count stack in repeater skip"); - } - ssize_t& n = ns.top(); - if (n == 0) { - n = boost::tuples::get<1>(*p) ? d.arrayNext() - : d.mapNext(); - } - if (n != 0) { - --n; - append(boost::tuples::get<3>(*p)); - continue; - } else { - ns.pop(); - } - } - break; - case Symbol::sIndirect: - { - ProductionPtr pp = - t.extra<ProductionPtr>(); - parsingStack.pop(); - append(pp); - } - continue; - case Symbol::sSymbolic: - { - ProductionPtr pp( - t.extra<std::weak_ptr<Production> >()); - parsingStack.pop(); - append(pp); - } - continue; - default: - { - std::ostringstream oss; - oss << "Don't know how to skip " - << Symbol::toString(t.kind()); - throw Exception(oss.str()); - } - } - parsingStack.pop(); - } - } - - void assertSize(size_t n) { - size_t s = popSize(); - if (s != n) { - std::ostringstream oss; - oss << "Incorrect size. Expected: " << s << " found " << n; - throw Exception(oss.str()); - } - } - - void assertLessThanSize(size_t n) { - assertLessThan(n, popSize()); - } - - size_t enumAdjust(size_t n) { - const Symbol& s = parsingStack.top(); - assertMatch(Symbol::sEnumAdjust, s.kind()); - const std::pair<std::vector<int>, std::vector<std::string> >* v = - s.extrap<std::pair<std::vector<int>, std::vector<std::string> > >(); - assertLessThan(n, v->first.size()); - - int result = v->first[n]; - if (result < 0) { - std::ostringstream oss; - oss << "Cannot resolve symbol: " << v->second[-result - 1] - << std::endl; - throw Exception(oss.str()); - } - parsingStack.pop(); - return result; - } - - size_t unionAdjust() { - const Symbol& s = parsingStack.top(); - assertMatch(Symbol::sUnionAdjust, s.kind()); - std::pair<size_t, ProductionPtr> p = - s.extra<std::pair<size_t, ProductionPtr> >(); - parsingStack.pop(); - append(p.second); - return p.first; - } - - std::string nameForIndex(size_t e) { - const Symbol& s = parsingStack.top(); - assertMatch(Symbol::sNameList, s.kind()); - const std::vector<std::string> names = - s.extra<std::vector<std::string> >(); - if (e >= names.size()) { - throw Exception("Not that many names"); - } - std::string result = names[e]; - parsingStack.pop(); - return result; - } - - size_t indexForName(const std::string &name) { - const Symbol& s = parsingStack.top(); - assertMatch(Symbol::sNameList, s.kind()); - const std::vector<std::string> names = - s.extra<std::vector<std::string> >(); - std::vector<std::string>::const_iterator it = - std::find(names.begin(), names.end(), name); - if (it == names.end()) { - throw Exception("No such enum symbol"); - } - size_t result = it - names.begin(); - parsingStack.pop(); - return result; - } - - void pushRepeatCount(size_t n) { - processImplicitActions(); - Symbol& s = parsingStack.top(); - assertMatch(Symbol::sRepeater, s.kind()); - RepeaterInfo *p = s.extrap<RepeaterInfo>(); - std::stack<ssize_t> &nn = boost::tuples::get<0>(*p); - nn.push(n); - } - - void nextRepeatCount(size_t n) { - processImplicitActions(); - Symbol& s = parsingStack.top(); - assertMatch(Symbol::sRepeater, s.kind()); - RepeaterInfo *p = s.extrap<RepeaterInfo>(); - std::stack<ssize_t> &nn = boost::tuples::get<0>(*p); - if (nn.empty() || nn.top() != 0) { - throw Exception("Wrong number of items"); - } - nn.top() = n; - } - - void popRepeater() { - processImplicitActions(); - Symbol& s = parsingStack.top(); - assertMatch(Symbol::sRepeater, s.kind()); - RepeaterInfo *p = s.extrap<RepeaterInfo>(); - std::stack<ssize_t> &ns = boost::tuples::get<0>(*p); - if (ns.empty()) { - throw Exception("Incorrect number of items (empty)"); - } - if (ns.top() > 0) { - throw Exception("Incorrect number of items (non-zero)"); - } - ns.pop(); - parsingStack.pop(); - } - - void selectBranch(size_t n) { - const Symbol& s = parsingStack.top(); - assertMatch(Symbol::sAlternative, s.kind()); - std::vector<ProductionPtr> v = - s.extra<std::vector<ProductionPtr> >(); - if (n >= v.size()) { - throw Exception("Not that many branches"); - } - parsingStack.pop(); - append(v[n]); - } - - const std::vector<size_t>& sizeList() { - const Symbol& s = parsingStack.top(); - assertMatch(Symbol::sSizeList, s.kind()); - return *s.extrap<std::vector<size_t> >(); - } - - Symbol::Kind top() const { - return parsingStack.top().kind(); - } - - void pop() { - parsingStack.pop(); - } - - void processImplicitActions() { - for (; ;) { - Symbol& s = parsingStack.top(); - if (s.isImplicitAction()) { - handler_.handle(s); - parsingStack.pop(); - } else if (s.kind() == Symbol::sSkipStart) { - parsingStack.pop(); - skip(*decoder_); - } else { - break; - } - } - } - - SimpleParser(const Symbol& s, Decoder* d, Handler& h) : - decoder_(d), handler_(h) { - parsingStack.push(s); - } - - void reset() { - while (parsingStack.size() > 1) { - parsingStack.pop(); - } - } - -}; - -inline std::ostream& operator<<(std::ostream& os, const Symbol s); - -inline std::ostream& operator<<(std::ostream& os, const Production p) -{ - os << '('; - for (Production::const_iterator it = p.begin(); it != p.end(); ++it) { - os << *it << ", "; - } - os << ')'; - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const Symbol s) -{ - switch (s.kind()) { - case Symbol::sRepeater: - { - const RepeaterInfo& ri = *s.extrap<RepeaterInfo>(); - os << '(' << Symbol::toString(s.kind()) - << ' ' << *boost::tuples::get<2>(ri) - << ' ' << *boost::tuples::get<3>(ri) - << ')'; - } - break; - case Symbol::sIndirect: - { - os << '(' << Symbol::toString(s.kind()) << ' ' - << *s.extra<std::shared_ptr<Production> >() << ')'; - } - break; - case Symbol::sAlternative: - { - os << '(' << Symbol::toString(s.kind()); - for (std::vector<ProductionPtr>::const_iterator it = - s.extrap<std::vector<ProductionPtr> >()->begin(); - it != s.extrap<std::vector<ProductionPtr> >()->end(); - ++it) { - os << ' ' << **it; - } - os << ')'; - } - break; - case Symbol::sSymbolic: - { - os << '(' << Symbol::toString(s.kind()) - << ' ' << s.extra<std::weak_ptr<Production> >().lock() - << ')'; - } - break; - default: - os << Symbol::toString(s.kind()); - break; - } - return os; - } -} // namespace parsing -} // namespace avro - -#endif +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_parsing_Symbol_hh__ +#define avro_parsing_Symbol_hh__ + +#include <vector> +#include <map> +#include <set> +#include <stack> +#include <sstream> + +#include <boost/any.hpp> +#include <boost/tuple/tuple.hpp> + +#include "Node.hh" +#include "Decoder.hh" +#include "Exception.hh" + +namespace avro { +namespace parsing { + +class Symbol; + +typedef std::vector<Symbol> Production; +typedef std::shared_ptr<Production> ProductionPtr; +typedef boost::tuple<std::stack<ssize_t>, bool, ProductionPtr, ProductionPtr> RepeaterInfo; +typedef boost::tuple<ProductionPtr, ProductionPtr> RootInfo; + +class Symbol { +public: + enum Kind { + sTerminalLow, // extra has nothing + sNull, + sBool, + sInt, + sLong, + sFloat, + sDouble, + sString, + sBytes, + sArrayStart, + sArrayEnd, + sMapStart, + sMapEnd, + sFixed, + sEnum, + sUnion, + sTerminalHigh, + sSizeCheck, // Extra has size + sNameList, // Extra has a vector<string> + sRoot, // Root for a schema, extra is Symbol + sRepeater, // Array or Map, extra is symbol + sAlternative, // One of many (union), extra is Union + sPlaceholder, // To be fixed up later. + sIndirect, // extra is shared_ptr<Production> + sSymbolic, // extra is weal_ptr<Production> + sEnumAdjust, + sUnionAdjust, + sSkipStart, + sResolve, + + sImplicitActionLow, + sRecordStart, + sRecordEnd, + sField, // extra is string + sRecord, + sSizeList, + sWriterUnion, + sDefaultStart, // extra has default value in Avro binary encoding + sDefaultEnd, + sImplicitActionHigh, + sError + }; + +private: + Kind kind_; + boost::any extra_; + + + explicit Symbol(Kind k) : kind_(k) { } + template <typename T> Symbol(Kind k, T t) : kind_(k), extra_(t) { } +public: + + Kind kind() const { + return kind_; + } + + template <typename T> T extra() const { + return boost::any_cast<T>(extra_); + } + + template <typename T> T* extrap() { + return boost::any_cast<T>(&extra_); + } + + template <typename T> const T* extrap() const { + return boost::any_cast<T>(&extra_); + } + + template <typename T> void extra(const T& t) { + extra_ = t; + } + + bool isTerminal() const { + return kind_ > sTerminalLow && kind_ < sTerminalHigh; + } + + bool isImplicitAction() const { + return kind_ > sImplicitActionLow && kind_ < sImplicitActionHigh; + } + + static const char* stringValues[]; + static const char* toString(Kind k) { + return stringValues[k]; + } + + static Symbol rootSymbol(ProductionPtr& s) + { + return Symbol(Symbol::sRoot, RootInfo(s, std::make_shared<Production>())); + } + + static Symbol rootSymbol(const ProductionPtr& main, + const ProductionPtr& backup) + { + return Symbol(Symbol::sRoot, RootInfo(main, backup)); + } + + static Symbol nullSymbol() { + return Symbol(sNull); + } + + static Symbol boolSymbol() { + return Symbol(sBool); + } + + static Symbol intSymbol() { + return Symbol(sInt); + } + + static Symbol longSymbol() { + return Symbol(sLong); + } + + static Symbol floatSymbol() { + return Symbol(sFloat); + } + + static Symbol doubleSymbol() { + return Symbol(sDouble); + } + + static Symbol stringSymbol() { + return Symbol(sString); + } + + static Symbol bytesSymbol() { + return Symbol(sBytes); + } + + static Symbol sizeCheckSymbol(size_t s) { + return Symbol(sSizeCheck, s); + } + + static Symbol fixedSymbol() { + return Symbol(sFixed); + } + + static Symbol enumSymbol() { + return Symbol(sEnum); + } + + static Symbol arrayStartSymbol() { + return Symbol(sArrayStart); + } + + static Symbol arrayEndSymbol() { + return Symbol(sArrayEnd); + } + + static Symbol mapStartSymbol() { + return Symbol(sMapStart); + } + + static Symbol mapEndSymbol() { + return Symbol(sMapEnd); + } + + static Symbol repeater(const ProductionPtr& p, + bool isArray) { + return repeater(p, p, isArray); + } + + static Symbol repeater(const ProductionPtr& read, + const ProductionPtr& skip, + bool isArray) { + std::stack<ssize_t> s; + return Symbol(sRepeater, RepeaterInfo(s, isArray, read, skip)); + } + + static Symbol defaultStartAction(std::shared_ptr<std::vector<uint8_t> > bb) + { + return Symbol(sDefaultStart, bb); + } + + static Symbol defaultEndAction() + { + return Symbol(sDefaultEnd); + } + + static Symbol alternative( + const std::vector<ProductionPtr>& branches) + { + return Symbol(Symbol::sAlternative, branches); + } + + static Symbol unionSymbol() { + return Symbol(sUnion); + } + + static Symbol recordStartSymbol() { + return Symbol(sRecordStart); + } + + static Symbol recordEndSymbol() { + return Symbol(sRecordEnd); + } + + static Symbol fieldSymbol(const std::string& name) { + return Symbol(sField, name); + } + + static Symbol writerUnionAction() { + return Symbol(sWriterUnion); + } + + static Symbol nameListSymbol( + const std::vector<std::string>& v) { + return Symbol(sNameList, v); + } + + template <typename T> + static Symbol placeholder(const T& n) { + return Symbol(sPlaceholder, n); + } + + static Symbol indirect(const ProductionPtr& p) { + return Symbol(sIndirect, p); + } + + static Symbol symbolic(const std::weak_ptr<Production>& p) { + return Symbol(sSymbolic, p); + } + + static Symbol enumAdjustSymbol(const NodePtr& writer, + const NodePtr& reader); + + static Symbol unionAdjustSymbol(size_t branch, + const ProductionPtr& p) { + return Symbol(sUnionAdjust, std::make_pair(branch, p)); + } + + static Symbol sizeListAction(std::vector<size_t> order) { + return Symbol(sSizeList, order); + } + + static Symbol recordAction() { + return Symbol(sRecord); + } + + static Symbol error(const NodePtr& writer, const NodePtr& reader); + + static Symbol resolveSymbol(Kind w, Kind r) { + return Symbol(sResolve, std::make_pair(w, r)); + } + + static Symbol skipStart() { + return Symbol(sSkipStart); + } + +}; + +/** + * Recursively replaces all placeholders in the production with the + * corresponding values. + */ +template<typename T> +void fixup(const ProductionPtr& p, + const std::map<T, ProductionPtr> &m) +{ + std::set<ProductionPtr> seen; + for (Production::iterator it = p->begin(); it != p->end(); ++it) { + fixup(*it, m, seen); + } +} + + +/** + * Recursively replaces all placeholders in the symbol with the values with the + * corresponding values. + */ +template<typename T> +void fixup_internal(const ProductionPtr& p, + const std::map<T, ProductionPtr> &m, + std::set<ProductionPtr>& seen) +{ + if (seen.find(p) == seen.end()) { + seen.insert(p); + for (Production::iterator it = p->begin(); it != p->end(); ++it) { + fixup(*it, m, seen); + } + } +} + +template<typename T> +void fixup(Symbol& s, const std::map<T, ProductionPtr> &m, + std::set<ProductionPtr>& seen) +{ + switch (s.kind()) { + case Symbol::sIndirect: + fixup_internal(s.extra<ProductionPtr>(), m, seen); + break; + case Symbol::sAlternative: + { + const std::vector<ProductionPtr> *vv = + s.extrap<std::vector<ProductionPtr> >(); + for (std::vector<ProductionPtr>::const_iterator it = vv->begin(); + it != vv->end(); ++it) { + fixup_internal(*it, m, seen); + } + } + break; + case Symbol::sRepeater: + { + const RepeaterInfo& ri = *s.extrap<RepeaterInfo>(); + fixup_internal(boost::tuples::get<2>(ri), m, seen); + fixup_internal(boost::tuples::get<3>(ri), m, seen); + } + break; + case Symbol::sPlaceholder: + { + typename std::map<T, std::shared_ptr<Production> >::const_iterator it = + m.find(s.extra<T>()); + if (it == m.end()) { + throw Exception("Placeholder symbol cannot be resolved"); + } + s = Symbol::symbolic(std::weak_ptr<Production>(it->second)); + } + break; + case Symbol::sUnionAdjust: + fixup_internal(s.extrap<std::pair<size_t, ProductionPtr> >()->second, + m, seen); + break; + default: + break; + } +} + +template<typename Handler> +class SimpleParser { + Decoder* decoder_; + Handler& handler_; + std::stack<Symbol> parsingStack; + + static void throwMismatch(Symbol::Kind actual, Symbol::Kind expected) + { + std::ostringstream oss; + oss << "Invalid operation. Schema requires: " << + Symbol::toString(expected) << ", got: " << + Symbol::toString(actual); + throw Exception(oss.str()); + } + + static void assertMatch(Symbol::Kind actual, Symbol::Kind expected) + { + if (expected != actual) { + throwMismatch(actual, expected); + } + + } + + void append(const ProductionPtr& ss) { + for (Production::const_iterator it = ss->begin(); + it != ss->end(); ++it) { + parsingStack.push(*it); + } + } + + size_t popSize() { + const Symbol& s = parsingStack.top(); + assertMatch(Symbol::sSizeCheck, s.kind()); + size_t result = s.extra<size_t>(); + parsingStack.pop(); + return result; + } + + static void assertLessThan(size_t n, size_t s) { + if (n >= s) { + std::ostringstream oss; + oss << "Size max value. Upper bound: " << s << " found " << n; + throw Exception(oss.str()); + } + } + +public: + Symbol::Kind advance(Symbol::Kind k) { + for (; ;) { + Symbol& s = parsingStack.top(); +// std::cout << "advance: " << Symbol::toString(s.kind()) +// << " looking for " << Symbol::toString(k) << '\n'; + if (s.kind() == k) { + parsingStack.pop(); + return k; + } else if (s.isTerminal()) { + throwMismatch(k, s.kind()); + } else { + switch (s.kind()) { + case Symbol::sRoot: + append(boost::tuples::get<0>(*s.extrap<RootInfo>())); + continue; + case Symbol::sIndirect: + { + ProductionPtr pp = + s.extra<ProductionPtr>(); + parsingStack.pop(); + append(pp); + } + continue; + case Symbol::sSymbolic: + { + ProductionPtr pp( + s.extra<std::weak_ptr<Production> >()); + parsingStack.pop(); + append(pp); + } + continue; + case Symbol::sRepeater: + { + RepeaterInfo *p = s.extrap<RepeaterInfo>(); + std::stack<ssize_t>& ns = boost::tuples::get<0>(*p); + if (ns.empty()) { + throw Exception( + "Empty item count stack in repeater advance"); + } + if (ns.top() == 0) { + throw Exception( + "Zero item count in repeater advance"); + } + --ns.top(); + append(boost::tuples::get<2>(*p)); + } + continue; + case Symbol::sError: + throw Exception(s.extra<std::string>()); + case Symbol::sResolve: + { + const std::pair<Symbol::Kind, Symbol::Kind>* p = + s.extrap<std::pair<Symbol::Kind, Symbol::Kind> >(); + assertMatch(p->second, k); + Symbol::Kind result = p->first; + parsingStack.pop(); + return result; + } + case Symbol::sSkipStart: + parsingStack.pop(); + skip(*decoder_); + break; + default: + if (s.isImplicitAction()) { + size_t n = handler_.handle(s); + if (s.kind() == Symbol::sWriterUnion) { + parsingStack.pop(); + selectBranch(n); + } else { + parsingStack.pop(); + } + } else { + std::ostringstream oss; + oss << "Encountered " << Symbol::toString(s.kind()) + << " while looking for " << Symbol::toString(k); + throw Exception(oss.str()); + } + } + } + } + } + + void skip(Decoder& d) { + const size_t sz = parsingStack.size(); + if (sz == 0) { + throw Exception("Nothing to skip!"); + } + while (parsingStack.size() >= sz) { + Symbol& t = parsingStack.top(); + // std::cout << "skip: " << Symbol::toString(t.kind()) << '\n'; + switch (t.kind()) { + case Symbol::sNull: + d.decodeNull(); + break; + case Symbol::sBool: + d.decodeBool(); + break; + case Symbol::sInt: + d.decodeInt(); + break; + case Symbol::sLong: + d.decodeLong(); + break; + case Symbol::sFloat: + d.decodeFloat(); + break; + case Symbol::sDouble: + d.decodeDouble(); + break; + case Symbol::sString: + d.skipString(); + break; + case Symbol::sBytes: + d.skipBytes(); + break; + case Symbol::sArrayStart: + { + parsingStack.pop(); + size_t n = d.skipArray(); + processImplicitActions(); + assertMatch(Symbol::sRepeater, parsingStack.top().kind()); + if (n == 0) { + break; + } + Symbol& t = parsingStack.top(); + RepeaterInfo *p = t.extrap<RepeaterInfo>(); + boost::tuples::get<0>(*p).push(n); + continue; + } + case Symbol::sArrayEnd: + break; + case Symbol::sMapStart: + { + parsingStack.pop(); + size_t n = d.skipMap(); + processImplicitActions(); + assertMatch(Symbol::sRepeater, parsingStack.top().kind()); + if (n == 0) { + break; + } + Symbol& t = parsingStack.top(); + RepeaterInfo *p = t.extrap<RepeaterInfo>(); + boost::tuples::get<0>(*p).push(n); + continue; + } + case Symbol::sMapEnd: + break; + case Symbol::sFixed: + { + parsingStack.pop(); + Symbol& t = parsingStack.top(); + d.decodeFixed(t.extra<size_t>()); + } + break; + case Symbol::sEnum: + parsingStack.pop(); + d.decodeEnum(); + break; + case Symbol::sUnion: + { + parsingStack.pop(); + size_t n = d.decodeUnionIndex(); + selectBranch(n); + continue; + } + case Symbol::sRepeater: + { + RepeaterInfo *p = t.extrap<RepeaterInfo>(); + std::stack<ssize_t>& ns = boost::tuples::get<0>(*p); + if (ns.empty()) { + throw Exception( + "Empty item count stack in repeater skip"); + } + ssize_t& n = ns.top(); + if (n == 0) { + n = boost::tuples::get<1>(*p) ? d.arrayNext() + : d.mapNext(); + } + if (n != 0) { + --n; + append(boost::tuples::get<3>(*p)); + continue; + } else { + ns.pop(); + } + } + break; + case Symbol::sIndirect: + { + ProductionPtr pp = + t.extra<ProductionPtr>(); + parsingStack.pop(); + append(pp); + } + continue; + case Symbol::sSymbolic: + { + ProductionPtr pp( + t.extra<std::weak_ptr<Production> >()); + parsingStack.pop(); + append(pp); + } + continue; + default: + { + std::ostringstream oss; + oss << "Don't know how to skip " + << Symbol::toString(t.kind()); + throw Exception(oss.str()); + } + } + parsingStack.pop(); + } + } + + void assertSize(size_t n) { + size_t s = popSize(); + if (s != n) { + std::ostringstream oss; + oss << "Incorrect size. Expected: " << s << " found " << n; + throw Exception(oss.str()); + } + } + + void assertLessThanSize(size_t n) { + assertLessThan(n, popSize()); + } + + size_t enumAdjust(size_t n) { + const Symbol& s = parsingStack.top(); + assertMatch(Symbol::sEnumAdjust, s.kind()); + const std::pair<std::vector<int>, std::vector<std::string> >* v = + s.extrap<std::pair<std::vector<int>, std::vector<std::string> > >(); + assertLessThan(n, v->first.size()); + + int result = v->first[n]; + if (result < 0) { + std::ostringstream oss; + oss << "Cannot resolve symbol: " << v->second[-result - 1] + << std::endl; + throw Exception(oss.str()); + } + parsingStack.pop(); + return result; + } + + size_t unionAdjust() { + const Symbol& s = parsingStack.top(); + assertMatch(Symbol::sUnionAdjust, s.kind()); + std::pair<size_t, ProductionPtr> p = + s.extra<std::pair<size_t, ProductionPtr> >(); + parsingStack.pop(); + append(p.second); + return p.first; + } + + std::string nameForIndex(size_t e) { + const Symbol& s = parsingStack.top(); + assertMatch(Symbol::sNameList, s.kind()); + const std::vector<std::string> names = + s.extra<std::vector<std::string> >(); + if (e >= names.size()) { + throw Exception("Not that many names"); + } + std::string result = names[e]; + parsingStack.pop(); + return result; + } + + size_t indexForName(const std::string &name) { + const Symbol& s = parsingStack.top(); + assertMatch(Symbol::sNameList, s.kind()); + const std::vector<std::string> names = + s.extra<std::vector<std::string> >(); + std::vector<std::string>::const_iterator it = + std::find(names.begin(), names.end(), name); + if (it == names.end()) { + throw Exception("No such enum symbol"); + } + size_t result = it - names.begin(); + parsingStack.pop(); + return result; + } + + void pushRepeatCount(size_t n) { + processImplicitActions(); + Symbol& s = parsingStack.top(); + assertMatch(Symbol::sRepeater, s.kind()); + RepeaterInfo *p = s.extrap<RepeaterInfo>(); + std::stack<ssize_t> &nn = boost::tuples::get<0>(*p); + nn.push(n); + } + + void nextRepeatCount(size_t n) { + processImplicitActions(); + Symbol& s = parsingStack.top(); + assertMatch(Symbol::sRepeater, s.kind()); + RepeaterInfo *p = s.extrap<RepeaterInfo>(); + std::stack<ssize_t> &nn = boost::tuples::get<0>(*p); + if (nn.empty() || nn.top() != 0) { + throw Exception("Wrong number of items"); + } + nn.top() = n; + } + + void popRepeater() { + processImplicitActions(); + Symbol& s = parsingStack.top(); + assertMatch(Symbol::sRepeater, s.kind()); + RepeaterInfo *p = s.extrap<RepeaterInfo>(); + std::stack<ssize_t> &ns = boost::tuples::get<0>(*p); + if (ns.empty()) { + throw Exception("Incorrect number of items (empty)"); + } + if (ns.top() > 0) { + throw Exception("Incorrect number of items (non-zero)"); + } + ns.pop(); + parsingStack.pop(); + } + + void selectBranch(size_t n) { + const Symbol& s = parsingStack.top(); + assertMatch(Symbol::sAlternative, s.kind()); + std::vector<ProductionPtr> v = + s.extra<std::vector<ProductionPtr> >(); + if (n >= v.size()) { + throw Exception("Not that many branches"); + } + parsingStack.pop(); + append(v[n]); + } + + const std::vector<size_t>& sizeList() { + const Symbol& s = parsingStack.top(); + assertMatch(Symbol::sSizeList, s.kind()); + return *s.extrap<std::vector<size_t> >(); + } + + Symbol::Kind top() const { + return parsingStack.top().kind(); + } + + void pop() { + parsingStack.pop(); + } + + void processImplicitActions() { + for (; ;) { + Symbol& s = parsingStack.top(); + if (s.isImplicitAction()) { + handler_.handle(s); + parsingStack.pop(); + } else if (s.kind() == Symbol::sSkipStart) { + parsingStack.pop(); + skip(*decoder_); + } else { + break; + } + } + } + + SimpleParser(const Symbol& s, Decoder* d, Handler& h) : + decoder_(d), handler_(h) { + parsingStack.push(s); + } + + void reset() { + while (parsingStack.size() > 1) { + parsingStack.pop(); + } + } + +}; + +inline std::ostream& operator<<(std::ostream& os, const Symbol s); + +inline std::ostream& operator<<(std::ostream& os, const Production p) +{ + os << '('; + for (Production::const_iterator it = p.begin(); it != p.end(); ++it) { + os << *it << ", "; + } + os << ')'; + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const Symbol s) +{ + switch (s.kind()) { + case Symbol::sRepeater: + { + const RepeaterInfo& ri = *s.extrap<RepeaterInfo>(); + os << '(' << Symbol::toString(s.kind()) + << ' ' << *boost::tuples::get<2>(ri) + << ' ' << *boost::tuples::get<3>(ri) + << ')'; + } + break; + case Symbol::sIndirect: + { + os << '(' << Symbol::toString(s.kind()) << ' ' + << *s.extra<std::shared_ptr<Production> >() << ')'; + } + break; + case Symbol::sAlternative: + { + os << '(' << Symbol::toString(s.kind()); + for (std::vector<ProductionPtr>::const_iterator it = + s.extrap<std::vector<ProductionPtr> >()->begin(); + it != s.extrap<std::vector<ProductionPtr> >()->end(); + ++it) { + os << ' ' << **it; + } + os << ')'; + } + break; + case Symbol::sSymbolic: + { + os << '(' << Symbol::toString(s.kind()) + << ' ' << s.extra<std::weak_ptr<Production> >().lock() + << ')'; + } + break; + default: + os << Symbol::toString(s.kind()); + break; + } + return os; + } +} // namespace parsing +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/impl/parsing/ValidatingCodec.cc b/contrib/libs/apache/avro/impl/parsing/ValidatingCodec.cc index 8a291d4317..fdf6ef898f 100644 --- a/contrib/libs/apache/avro/impl/parsing/ValidatingCodec.cc +++ b/contrib/libs/apache/avro/impl/parsing/ValidatingCodec.cc @@ -1,591 +1,591 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ValidatingCodec.hh" - -#include <string> -#include <map> -#include <algorithm> -#include <memory> -#include <boost/any.hpp> - -#include "ValidSchema.hh" -#include "Decoder.hh" -#include "Encoder.hh" -#include "NodeImpl.hh" - -namespace avro { - -using std::make_shared; - -namespace parsing { - -using std::shared_ptr; -using std::static_pointer_cast; - -using std::map; -using std::vector; -using std::pair; -using std::string; -using std::reverse; -using std::ostringstream; - -/** Follows the design of Avro Parser in Java. */ -ProductionPtr ValidatingGrammarGenerator::generate(const NodePtr& n) -{ - map<NodePtr, ProductionPtr> m; - ProductionPtr result = doGenerate(n, m); - fixup(result, m); - return result; -} - -Symbol ValidatingGrammarGenerator::generate(const ValidSchema& schema) -{ - ProductionPtr r = generate(schema.root()); - return Symbol::rootSymbol(r); -} - -ProductionPtr ValidatingGrammarGenerator::doGenerate(const NodePtr& n, - map<NodePtr, ProductionPtr> &m) { - switch (n->type()) { - case AVRO_NULL: - return make_shared<Production>(1, Symbol::nullSymbol()); - case AVRO_BOOL: - return make_shared<Production>(1, Symbol::boolSymbol()); - case AVRO_INT: - return make_shared<Production>(1, Symbol::intSymbol()); - case AVRO_LONG: - return make_shared<Production>(1, Symbol::longSymbol()); - case AVRO_FLOAT: - return make_shared<Production>(1, Symbol::floatSymbol()); - case AVRO_DOUBLE: - return make_shared<Production>(1, Symbol::doubleSymbol()); - case AVRO_STRING: - return make_shared<Production>(1, Symbol::stringSymbol()); - case AVRO_BYTES: - return make_shared<Production>(1, Symbol::bytesSymbol()); - case AVRO_FIXED: - { - ProductionPtr result = make_shared<Production>(); - result->push_back(Symbol::sizeCheckSymbol(n->fixedSize())); - result->push_back(Symbol::fixedSymbol()); - m[n] = result; - return result; - } - case AVRO_RECORD: - { - ProductionPtr result = make_shared<Production>(); - - m.erase(n); - size_t c = n->leaves(); - for (size_t i = 0; i < c; ++i) { - const NodePtr& leaf = n->leafAt(i); - ProductionPtr v = doGenerate(leaf, m); - copy(v->rbegin(), v->rend(), back_inserter(*result)); - } - reverse(result->begin(), result->end()); - - m[n] = result; - return make_shared<Production>(1, Symbol::indirect(result)); - } - case AVRO_ENUM: - { - ProductionPtr result = make_shared<Production>(); - result->push_back(Symbol::sizeCheckSymbol(n->names())); - result->push_back(Symbol::enumSymbol()); - m[n] = result; - return result; - } - case AVRO_ARRAY: - { - ProductionPtr result = make_shared<Production>(); - result->push_back(Symbol::arrayEndSymbol()); - result->push_back(Symbol::repeater(doGenerate(n->leafAt(0), m), true)); - result->push_back(Symbol::arrayStartSymbol()); - return result; - } - case AVRO_MAP: - { - ProductionPtr pp = doGenerate(n->leafAt(1), m); - ProductionPtr v(new Production(*pp)); - v->push_back(Symbol::stringSymbol()); - ProductionPtr result = make_shared<Production>(); - result->push_back(Symbol::mapEndSymbol()); - result->push_back(Symbol::repeater(v, false)); - result->push_back(Symbol::mapStartSymbol()); - return result; - } - case AVRO_UNION: - { - vector<ProductionPtr> vv; - size_t c = n->leaves(); - vv.reserve(c); - for (size_t i = 0; i < c; ++i) { - vv.push_back(doGenerate(n->leafAt(i), m)); - } - ProductionPtr result = make_shared<Production>(); - result->push_back(Symbol::alternative(vv)); - result->push_back(Symbol::unionSymbol()); - return result; - } - case AVRO_SYMBOLIC: - { - shared_ptr<NodeSymbolic> ns = static_pointer_cast<NodeSymbolic>(n); - NodePtr nn = ns->getNode(); - map<NodePtr, ProductionPtr>::iterator it = - m.find(nn); - if (it != m.end() && it->second) { - return it->second; - } else { - m[nn] = ProductionPtr(); - return make_shared<Production>(1, Symbol::placeholder(nn)); - } - } - default: - throw Exception("Unknown node type"); - } -} - -struct DummyHandler { - size_t handle(const Symbol& s) { - return 0; - } -}; - -template <typename P> -class ValidatingDecoder : public Decoder { - const shared_ptr<Decoder> base; - DummyHandler handler_; - P parser; - - void init(InputStream& is); - void decodeNull(); - bool decodeBool(); - int32_t decodeInt(); - int64_t decodeLong(); - float decodeFloat(); - double decodeDouble(); - void decodeString(string& value); - void skipString(); - void decodeBytes(vector<uint8_t>& value); - void skipBytes(); - void decodeFixed(size_t n, vector<uint8_t>& value); - void skipFixed(size_t n); - size_t decodeEnum(); - size_t arrayStart(); - size_t arrayNext(); - size_t skipArray(); - size_t mapStart(); - size_t mapNext(); - size_t skipMap(); - size_t decodeUnionIndex(); - void drain() { - base->drain(); - } - -public: - - ValidatingDecoder(const ValidSchema& s, const shared_ptr<Decoder> b) : - base(b), - parser(ValidatingGrammarGenerator().generate(s), NULL, handler_) { } - -}; - -template <typename P> -void ValidatingDecoder<P>::init(InputStream& is) -{ - base->init(is); -} - -template <typename P> -void ValidatingDecoder<P>::decodeNull() -{ - parser.advance(Symbol::sNull); - base->decodeNull(); -} - -template <typename P> -bool ValidatingDecoder<P>::decodeBool() -{ - parser.advance(Symbol::sBool); - return base->decodeBool(); -} - -template <typename P> -int32_t ValidatingDecoder<P>::decodeInt() -{ - parser.advance(Symbol::sInt); - return base->decodeInt(); -} - -template <typename P> -int64_t ValidatingDecoder<P>::decodeLong() -{ - parser.advance(Symbol::sLong); - return base->decodeLong(); -} - -template <typename P> -float ValidatingDecoder<P>::decodeFloat() -{ - parser.advance(Symbol::sFloat); - return base->decodeFloat(); -} - -template <typename P> -double ValidatingDecoder<P>::decodeDouble() -{ - parser.advance(Symbol::sDouble); - return base->decodeDouble(); -} - -template <typename P> -void ValidatingDecoder<P>::decodeString(string& value) -{ - parser.advance(Symbol::sString); - base->decodeString(value); -} - -template <typename P> -void ValidatingDecoder<P>::skipString() -{ - parser.advance(Symbol::sString); - base->skipString(); -} - -template <typename P> -void ValidatingDecoder<P>::decodeBytes(vector<uint8_t>& value) -{ - parser.advance(Symbol::sBytes); - base->decodeBytes(value); -} - -template <typename P> -void ValidatingDecoder<P>::skipBytes() -{ - parser.advance(Symbol::sBytes); - base->skipBytes(); -} - -template <typename P> -void ValidatingDecoder<P>::decodeFixed(size_t n, vector<uint8_t>& value) -{ - parser.advance(Symbol::sFixed); - parser.assertSize(n); - base->decodeFixed(n, value); -} - -template <typename P> -void ValidatingDecoder<P>::skipFixed(size_t n) -{ - parser.advance(Symbol::sFixed); - parser.assertSize(n); - base->skipFixed(n); -} - -template <typename P> -size_t ValidatingDecoder<P>::decodeEnum() -{ - parser.advance(Symbol::sEnum); - size_t result = base->decodeEnum(); - parser.assertLessThanSize(result); - return result; -} - -template <typename P> -size_t ValidatingDecoder<P>::arrayStart() -{ - parser.advance(Symbol::sArrayStart); - size_t result = base->arrayStart(); - parser.pushRepeatCount(result); - if (result == 0) { - parser.popRepeater(); - parser.advance(Symbol::sArrayEnd); - } - return result; -} - -template <typename P> -size_t ValidatingDecoder<P>::arrayNext() -{ - size_t result = base->arrayNext(); - parser.nextRepeatCount(result); - if (result == 0) { - parser.popRepeater(); - parser.advance(Symbol::sArrayEnd); - } - return result; -} - -template <typename P> -size_t ValidatingDecoder<P>::skipArray() -{ - parser.advance(Symbol::sArrayStart); - size_t n = base->skipArray(); - if (n == 0) { - parser.pop(); - } else { - parser.pushRepeatCount(n); - parser.skip(*base); - } - parser.advance(Symbol::sArrayEnd); - return 0; -} - -template <typename P> -size_t ValidatingDecoder<P>::mapStart() -{ - parser.advance(Symbol::sMapStart); - size_t result = base->mapStart(); - parser.pushRepeatCount(result); - if (result == 0) { - parser.popRepeater(); - parser.advance(Symbol::sMapEnd); - } - return result; -} - -template <typename P> -size_t ValidatingDecoder<P>::mapNext() -{ - size_t result = base->mapNext(); - parser.nextRepeatCount(result); - if (result == 0) { - parser.popRepeater(); - parser.advance(Symbol::sMapEnd); - } - return result; -} - -template <typename P> -size_t ValidatingDecoder<P>::skipMap() -{ - parser.advance(Symbol::sMapStart); - size_t n = base->skipMap(); - if (n == 0) { - parser.pop(); - } else { - parser.pushRepeatCount(n); - parser.skip(*base); - } - parser.advance(Symbol::sMapEnd); - return 0; -} - -template <typename P> -size_t ValidatingDecoder<P>::decodeUnionIndex() -{ - parser.advance(Symbol::sUnion); - size_t result = base->decodeUnionIndex(); - parser.selectBranch(result); - return result; -} - -template <typename P> -class ValidatingEncoder : public Encoder { - DummyHandler handler_; - P parser_; - EncoderPtr base_; - - void init(OutputStream& os); - void flush(); - int64_t byteCount() const; - void encodeNull(); - void encodeBool(bool b); - void encodeInt(int32_t i); - void encodeLong(int64_t l); - void encodeFloat(float f); - void encodeDouble(double d); - void encodeString(const std::string& s); - void encodeBytes(const uint8_t *bytes, size_t len); - void encodeFixed(const uint8_t *bytes, size_t len); - void encodeEnum(size_t e); - void arrayStart(); - void arrayEnd(); - void mapStart(); - void mapEnd(); - void setItemCount(size_t count); - void startItem(); - void encodeUnionIndex(size_t e); -public: - ValidatingEncoder(const ValidSchema& schema, const EncoderPtr& base) : - parser_(ValidatingGrammarGenerator().generate(schema), NULL, handler_), - base_(base) { } -}; - -template<typename P> -void ValidatingEncoder<P>::init(OutputStream& os) -{ - base_->init(os); -} - -template<typename P> -void ValidatingEncoder<P>::flush() -{ - base_->flush(); -} - -template<typename P> -void ValidatingEncoder<P>::encodeNull() -{ - parser_.advance(Symbol::sNull); - base_->encodeNull(); -} - -template<typename P> -void ValidatingEncoder<P>::encodeBool(bool b) -{ - parser_.advance(Symbol::sBool); - base_->encodeBool(b); -} - -template<typename P> -void ValidatingEncoder<P>::encodeInt(int32_t i) -{ - parser_.advance(Symbol::sInt); - base_->encodeInt(i); -} - -template<typename P> -void ValidatingEncoder<P>::encodeLong(int64_t l) -{ - parser_.advance(Symbol::sLong); - base_->encodeLong(l); -} - -template<typename P> -void ValidatingEncoder<P>::encodeFloat(float f) -{ - parser_.advance(Symbol::sFloat); - base_->encodeFloat(f); -} - -template<typename P> -void ValidatingEncoder<P>::encodeDouble(double d) -{ - parser_.advance(Symbol::sDouble); - base_->encodeDouble(d); -} - -template<typename P> -void ValidatingEncoder<P>::encodeString(const std::string& s) -{ - parser_.advance(Symbol::sString); - base_->encodeString(s); -} - -template<typename P> -void ValidatingEncoder<P>::encodeBytes(const uint8_t *bytes, size_t len) -{ - parser_.advance(Symbol::sBytes); - base_->encodeBytes(bytes, len); -} - -template<typename P> -void ValidatingEncoder<P>::encodeFixed(const uint8_t *bytes, size_t len) -{ - parser_.advance(Symbol::sFixed); - parser_.assertSize(len); - base_->encodeFixed(bytes, len); -} - -template<typename P> -void ValidatingEncoder<P>::encodeEnum(size_t e) -{ - parser_.advance(Symbol::sEnum); - parser_.assertLessThanSize(e); - base_->encodeEnum(e); -} - -template<typename P> -void ValidatingEncoder<P>::arrayStart() -{ - parser_.advance(Symbol::sArrayStart); - parser_.pushRepeatCount(0); - base_->arrayStart(); -} - -template<typename P> -void ValidatingEncoder<P>::arrayEnd() -{ - parser_.popRepeater(); - parser_.advance(Symbol::sArrayEnd); - base_->arrayEnd(); -} - -template<typename P> -void ValidatingEncoder<P>::mapStart() -{ - parser_.advance(Symbol::sMapStart); - parser_.pushRepeatCount(0); - base_->mapStart(); -} - -template<typename P> -void ValidatingEncoder<P>::mapEnd() -{ - parser_.popRepeater(); - parser_.advance(Symbol::sMapEnd); - base_->mapEnd(); -} - -template<typename P> -void ValidatingEncoder<P>::setItemCount(size_t count) -{ - parser_.nextRepeatCount(count); - base_->setItemCount(count); -} - -template<typename P> -void ValidatingEncoder<P>::startItem() -{ - if (parser_.top() != Symbol::sRepeater) { - throw Exception("startItem at not an item boundary"); - } - base_->startItem(); -} - -template<typename P> -void ValidatingEncoder<P>::encodeUnionIndex(size_t e) -{ - parser_.advance(Symbol::sUnion); - parser_.selectBranch(e); - base_->encodeUnionIndex(e); -} - -template<typename P> -int64_t ValidatingEncoder<P>::byteCount() const -{ - return base_->byteCount(); -} - -} // namespace parsing - -DecoderPtr validatingDecoder(const ValidSchema& s, - const DecoderPtr& base) -{ - return make_shared<parsing::ValidatingDecoder<parsing::SimpleParser<parsing::DummyHandler> > >(s, base); -} - -EncoderPtr validatingEncoder(const ValidSchema& schema, const EncoderPtr& base) -{ - return make_shared<parsing::ValidatingEncoder<parsing::SimpleParser<parsing::DummyHandler> > >(schema, base); -} - -} // namespace avro - +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ValidatingCodec.hh" + +#include <string> +#include <map> +#include <algorithm> +#include <memory> +#include <boost/any.hpp> + +#include "ValidSchema.hh" +#include "Decoder.hh" +#include "Encoder.hh" +#include "NodeImpl.hh" + +namespace avro { + +using std::make_shared; + +namespace parsing { + +using std::shared_ptr; +using std::static_pointer_cast; + +using std::map; +using std::vector; +using std::pair; +using std::string; +using std::reverse; +using std::ostringstream; + +/** Follows the design of Avro Parser in Java. */ +ProductionPtr ValidatingGrammarGenerator::generate(const NodePtr& n) +{ + map<NodePtr, ProductionPtr> m; + ProductionPtr result = doGenerate(n, m); + fixup(result, m); + return result; +} + +Symbol ValidatingGrammarGenerator::generate(const ValidSchema& schema) +{ + ProductionPtr r = generate(schema.root()); + return Symbol::rootSymbol(r); +} + +ProductionPtr ValidatingGrammarGenerator::doGenerate(const NodePtr& n, + map<NodePtr, ProductionPtr> &m) { + switch (n->type()) { + case AVRO_NULL: + return make_shared<Production>(1, Symbol::nullSymbol()); + case AVRO_BOOL: + return make_shared<Production>(1, Symbol::boolSymbol()); + case AVRO_INT: + return make_shared<Production>(1, Symbol::intSymbol()); + case AVRO_LONG: + return make_shared<Production>(1, Symbol::longSymbol()); + case AVRO_FLOAT: + return make_shared<Production>(1, Symbol::floatSymbol()); + case AVRO_DOUBLE: + return make_shared<Production>(1, Symbol::doubleSymbol()); + case AVRO_STRING: + return make_shared<Production>(1, Symbol::stringSymbol()); + case AVRO_BYTES: + return make_shared<Production>(1, Symbol::bytesSymbol()); + case AVRO_FIXED: + { + ProductionPtr result = make_shared<Production>(); + result->push_back(Symbol::sizeCheckSymbol(n->fixedSize())); + result->push_back(Symbol::fixedSymbol()); + m[n] = result; + return result; + } + case AVRO_RECORD: + { + ProductionPtr result = make_shared<Production>(); + + m.erase(n); + size_t c = n->leaves(); + for (size_t i = 0; i < c; ++i) { + const NodePtr& leaf = n->leafAt(i); + ProductionPtr v = doGenerate(leaf, m); + copy(v->rbegin(), v->rend(), back_inserter(*result)); + } + reverse(result->begin(), result->end()); + + m[n] = result; + return make_shared<Production>(1, Symbol::indirect(result)); + } + case AVRO_ENUM: + { + ProductionPtr result = make_shared<Production>(); + result->push_back(Symbol::sizeCheckSymbol(n->names())); + result->push_back(Symbol::enumSymbol()); + m[n] = result; + return result; + } + case AVRO_ARRAY: + { + ProductionPtr result = make_shared<Production>(); + result->push_back(Symbol::arrayEndSymbol()); + result->push_back(Symbol::repeater(doGenerate(n->leafAt(0), m), true)); + result->push_back(Symbol::arrayStartSymbol()); + return result; + } + case AVRO_MAP: + { + ProductionPtr pp = doGenerate(n->leafAt(1), m); + ProductionPtr v(new Production(*pp)); + v->push_back(Symbol::stringSymbol()); + ProductionPtr result = make_shared<Production>(); + result->push_back(Symbol::mapEndSymbol()); + result->push_back(Symbol::repeater(v, false)); + result->push_back(Symbol::mapStartSymbol()); + return result; + } + case AVRO_UNION: + { + vector<ProductionPtr> vv; + size_t c = n->leaves(); + vv.reserve(c); + for (size_t i = 0; i < c; ++i) { + vv.push_back(doGenerate(n->leafAt(i), m)); + } + ProductionPtr result = make_shared<Production>(); + result->push_back(Symbol::alternative(vv)); + result->push_back(Symbol::unionSymbol()); + return result; + } + case AVRO_SYMBOLIC: + { + shared_ptr<NodeSymbolic> ns = static_pointer_cast<NodeSymbolic>(n); + NodePtr nn = ns->getNode(); + map<NodePtr, ProductionPtr>::iterator it = + m.find(nn); + if (it != m.end() && it->second) { + return it->second; + } else { + m[nn] = ProductionPtr(); + return make_shared<Production>(1, Symbol::placeholder(nn)); + } + } + default: + throw Exception("Unknown node type"); + } +} + +struct DummyHandler { + size_t handle(const Symbol& s) { + return 0; + } +}; + +template <typename P> +class ValidatingDecoder : public Decoder { + const shared_ptr<Decoder> base; + DummyHandler handler_; + P parser; + + void init(InputStream& is); + void decodeNull(); + bool decodeBool(); + int32_t decodeInt(); + int64_t decodeLong(); + float decodeFloat(); + double decodeDouble(); + void decodeString(string& value); + void skipString(); + void decodeBytes(vector<uint8_t>& value); + void skipBytes(); + void decodeFixed(size_t n, vector<uint8_t>& value); + void skipFixed(size_t n); + size_t decodeEnum(); + size_t arrayStart(); + size_t arrayNext(); + size_t skipArray(); + size_t mapStart(); + size_t mapNext(); + size_t skipMap(); + size_t decodeUnionIndex(); + void drain() { + base->drain(); + } + +public: + + ValidatingDecoder(const ValidSchema& s, const shared_ptr<Decoder> b) : + base(b), + parser(ValidatingGrammarGenerator().generate(s), NULL, handler_) { } + +}; + +template <typename P> +void ValidatingDecoder<P>::init(InputStream& is) +{ + base->init(is); +} + +template <typename P> +void ValidatingDecoder<P>::decodeNull() +{ + parser.advance(Symbol::sNull); + base->decodeNull(); +} + +template <typename P> +bool ValidatingDecoder<P>::decodeBool() +{ + parser.advance(Symbol::sBool); + return base->decodeBool(); +} + +template <typename P> +int32_t ValidatingDecoder<P>::decodeInt() +{ + parser.advance(Symbol::sInt); + return base->decodeInt(); +} + +template <typename P> +int64_t ValidatingDecoder<P>::decodeLong() +{ + parser.advance(Symbol::sLong); + return base->decodeLong(); +} + +template <typename P> +float ValidatingDecoder<P>::decodeFloat() +{ + parser.advance(Symbol::sFloat); + return base->decodeFloat(); +} + +template <typename P> +double ValidatingDecoder<P>::decodeDouble() +{ + parser.advance(Symbol::sDouble); + return base->decodeDouble(); +} + +template <typename P> +void ValidatingDecoder<P>::decodeString(string& value) +{ + parser.advance(Symbol::sString); + base->decodeString(value); +} + +template <typename P> +void ValidatingDecoder<P>::skipString() +{ + parser.advance(Symbol::sString); + base->skipString(); +} + +template <typename P> +void ValidatingDecoder<P>::decodeBytes(vector<uint8_t>& value) +{ + parser.advance(Symbol::sBytes); + base->decodeBytes(value); +} + +template <typename P> +void ValidatingDecoder<P>::skipBytes() +{ + parser.advance(Symbol::sBytes); + base->skipBytes(); +} + +template <typename P> +void ValidatingDecoder<P>::decodeFixed(size_t n, vector<uint8_t>& value) +{ + parser.advance(Symbol::sFixed); + parser.assertSize(n); + base->decodeFixed(n, value); +} + +template <typename P> +void ValidatingDecoder<P>::skipFixed(size_t n) +{ + parser.advance(Symbol::sFixed); + parser.assertSize(n); + base->skipFixed(n); +} + +template <typename P> +size_t ValidatingDecoder<P>::decodeEnum() +{ + parser.advance(Symbol::sEnum); + size_t result = base->decodeEnum(); + parser.assertLessThanSize(result); + return result; +} + +template <typename P> +size_t ValidatingDecoder<P>::arrayStart() +{ + parser.advance(Symbol::sArrayStart); + size_t result = base->arrayStart(); + parser.pushRepeatCount(result); + if (result == 0) { + parser.popRepeater(); + parser.advance(Symbol::sArrayEnd); + } + return result; +} + +template <typename P> +size_t ValidatingDecoder<P>::arrayNext() +{ + size_t result = base->arrayNext(); + parser.nextRepeatCount(result); + if (result == 0) { + parser.popRepeater(); + parser.advance(Symbol::sArrayEnd); + } + return result; +} + +template <typename P> +size_t ValidatingDecoder<P>::skipArray() +{ + parser.advance(Symbol::sArrayStart); + size_t n = base->skipArray(); + if (n == 0) { + parser.pop(); + } else { + parser.pushRepeatCount(n); + parser.skip(*base); + } + parser.advance(Symbol::sArrayEnd); + return 0; +} + +template <typename P> +size_t ValidatingDecoder<P>::mapStart() +{ + parser.advance(Symbol::sMapStart); + size_t result = base->mapStart(); + parser.pushRepeatCount(result); + if (result == 0) { + parser.popRepeater(); + parser.advance(Symbol::sMapEnd); + } + return result; +} + +template <typename P> +size_t ValidatingDecoder<P>::mapNext() +{ + size_t result = base->mapNext(); + parser.nextRepeatCount(result); + if (result == 0) { + parser.popRepeater(); + parser.advance(Symbol::sMapEnd); + } + return result; +} + +template <typename P> +size_t ValidatingDecoder<P>::skipMap() +{ + parser.advance(Symbol::sMapStart); + size_t n = base->skipMap(); + if (n == 0) { + parser.pop(); + } else { + parser.pushRepeatCount(n); + parser.skip(*base); + } + parser.advance(Symbol::sMapEnd); + return 0; +} + +template <typename P> +size_t ValidatingDecoder<P>::decodeUnionIndex() +{ + parser.advance(Symbol::sUnion); + size_t result = base->decodeUnionIndex(); + parser.selectBranch(result); + return result; +} + +template <typename P> +class ValidatingEncoder : public Encoder { + DummyHandler handler_; + P parser_; + EncoderPtr base_; + + void init(OutputStream& os); + void flush(); + int64_t byteCount() const; + void encodeNull(); + void encodeBool(bool b); + void encodeInt(int32_t i); + void encodeLong(int64_t l); + void encodeFloat(float f); + void encodeDouble(double d); + void encodeString(const std::string& s); + void encodeBytes(const uint8_t *bytes, size_t len); + void encodeFixed(const uint8_t *bytes, size_t len); + void encodeEnum(size_t e); + void arrayStart(); + void arrayEnd(); + void mapStart(); + void mapEnd(); + void setItemCount(size_t count); + void startItem(); + void encodeUnionIndex(size_t e); +public: + ValidatingEncoder(const ValidSchema& schema, const EncoderPtr& base) : + parser_(ValidatingGrammarGenerator().generate(schema), NULL, handler_), + base_(base) { } +}; + +template<typename P> +void ValidatingEncoder<P>::init(OutputStream& os) +{ + base_->init(os); +} + +template<typename P> +void ValidatingEncoder<P>::flush() +{ + base_->flush(); +} + +template<typename P> +void ValidatingEncoder<P>::encodeNull() +{ + parser_.advance(Symbol::sNull); + base_->encodeNull(); +} + +template<typename P> +void ValidatingEncoder<P>::encodeBool(bool b) +{ + parser_.advance(Symbol::sBool); + base_->encodeBool(b); +} + +template<typename P> +void ValidatingEncoder<P>::encodeInt(int32_t i) +{ + parser_.advance(Symbol::sInt); + base_->encodeInt(i); +} + +template<typename P> +void ValidatingEncoder<P>::encodeLong(int64_t l) +{ + parser_.advance(Symbol::sLong); + base_->encodeLong(l); +} + +template<typename P> +void ValidatingEncoder<P>::encodeFloat(float f) +{ + parser_.advance(Symbol::sFloat); + base_->encodeFloat(f); +} + +template<typename P> +void ValidatingEncoder<P>::encodeDouble(double d) +{ + parser_.advance(Symbol::sDouble); + base_->encodeDouble(d); +} + +template<typename P> +void ValidatingEncoder<P>::encodeString(const std::string& s) +{ + parser_.advance(Symbol::sString); + base_->encodeString(s); +} + +template<typename P> +void ValidatingEncoder<P>::encodeBytes(const uint8_t *bytes, size_t len) +{ + parser_.advance(Symbol::sBytes); + base_->encodeBytes(bytes, len); +} + +template<typename P> +void ValidatingEncoder<P>::encodeFixed(const uint8_t *bytes, size_t len) +{ + parser_.advance(Symbol::sFixed); + parser_.assertSize(len); + base_->encodeFixed(bytes, len); +} + +template<typename P> +void ValidatingEncoder<P>::encodeEnum(size_t e) +{ + parser_.advance(Symbol::sEnum); + parser_.assertLessThanSize(e); + base_->encodeEnum(e); +} + +template<typename P> +void ValidatingEncoder<P>::arrayStart() +{ + parser_.advance(Symbol::sArrayStart); + parser_.pushRepeatCount(0); + base_->arrayStart(); +} + +template<typename P> +void ValidatingEncoder<P>::arrayEnd() +{ + parser_.popRepeater(); + parser_.advance(Symbol::sArrayEnd); + base_->arrayEnd(); +} + +template<typename P> +void ValidatingEncoder<P>::mapStart() +{ + parser_.advance(Symbol::sMapStart); + parser_.pushRepeatCount(0); + base_->mapStart(); +} + +template<typename P> +void ValidatingEncoder<P>::mapEnd() +{ + parser_.popRepeater(); + parser_.advance(Symbol::sMapEnd); + base_->mapEnd(); +} + +template<typename P> +void ValidatingEncoder<P>::setItemCount(size_t count) +{ + parser_.nextRepeatCount(count); + base_->setItemCount(count); +} + +template<typename P> +void ValidatingEncoder<P>::startItem() +{ + if (parser_.top() != Symbol::sRepeater) { + throw Exception("startItem at not an item boundary"); + } + base_->startItem(); +} + +template<typename P> +void ValidatingEncoder<P>::encodeUnionIndex(size_t e) +{ + parser_.advance(Symbol::sUnion); + parser_.selectBranch(e); + base_->encodeUnionIndex(e); +} + +template<typename P> +int64_t ValidatingEncoder<P>::byteCount() const +{ + return base_->byteCount(); +} + +} // namespace parsing + +DecoderPtr validatingDecoder(const ValidSchema& s, + const DecoderPtr& base) +{ + return make_shared<parsing::ValidatingDecoder<parsing::SimpleParser<parsing::DummyHandler> > >(s, base); +} + +EncoderPtr validatingEncoder(const ValidSchema& schema, const EncoderPtr& base) +{ + return make_shared<parsing::ValidatingEncoder<parsing::SimpleParser<parsing::DummyHandler> > >(schema, base); +} + +} // namespace avro + diff --git a/contrib/libs/apache/avro/impl/parsing/ValidatingCodec.hh b/contrib/libs/apache/avro/impl/parsing/ValidatingCodec.hh index b90b3ea64a..39ceda033e 100644 --- a/contrib/libs/apache/avro/impl/parsing/ValidatingCodec.hh +++ b/contrib/libs/apache/avro/impl/parsing/ValidatingCodec.hh @@ -1,51 +1,51 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef avro_parsing_ValidatingCodec_hh__ -#define avro_parsing_ValidatingCodec_hh__ - -#include <map> -#include <vector> - -#include "Symbol.hh" -#include "ValidSchema.hh" -#include "NodeImpl.hh" - -namespace avro { -namespace parsing { - -class ValidatingGrammarGenerator { -protected: - template<typename T> - static void doFixup(Production& p, const std::map<T, ProductionPtr> &m); - - template<typename T> - static void doFixup(Symbol &s, const std::map<T, ProductionPtr> &m); - virtual ProductionPtr doGenerate(const NodePtr& n, - std::map<NodePtr, ProductionPtr> &m); - - ProductionPtr generate(const NodePtr& schema); -public: - Symbol generate(const ValidSchema& schema); - -}; - -} // namespace parsing -} // namespace avro - -#endif +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_parsing_ValidatingCodec_hh__ +#define avro_parsing_ValidatingCodec_hh__ + +#include <map> +#include <vector> + +#include "Symbol.hh" +#include "ValidSchema.hh" +#include "NodeImpl.hh" + +namespace avro { +namespace parsing { + +class ValidatingGrammarGenerator { +protected: + template<typename T> + static void doFixup(Production& p, const std::map<T, ProductionPtr> &m); + + template<typename T> + static void doFixup(Symbol &s, const std::map<T, ProductionPtr> &m); + virtual ProductionPtr doGenerate(const NodePtr& n, + std::map<NodePtr, ProductionPtr> &m); + + ProductionPtr generate(const NodePtr& schema); +public: + Symbol generate(const ValidSchema& schema); + +}; + +} // namespace parsing +} // namespace avro + +#endif diff --git a/contrib/libs/apache/avro/ya.make b/contrib/libs/apache/avro/ya.make index 11e72e643a..80197fdc47 100644 --- a/contrib/libs/apache/avro/ya.make +++ b/contrib/libs/apache/avro/ya.make @@ -1,14 +1,14 @@ -# Generated by devtools/yamaker from nixpkgs 5701e5bc3bfb317e1f37ff3fb889eae7584a1206. - -LIBRARY() - +# Generated by devtools/yamaker from nixpkgs 5701e5bc3bfb317e1f37ff3fb889eae7584a1206. + +LIBRARY() + OWNER( - g:cpp-contrib + g:cpp-contrib g:yql ) - -VERSION(1.10.2) - + +VERSION(1.10.2) + ORIGINAL_SOURCE(https://github.com/apache/avro/archive/release-1.10.2.tar.gz) LICENSE( @@ -16,53 +16,53 @@ LICENSE( BSL-1.0 AND FSFAP ) - + LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -PEERDIR( - contrib/libs/snappy - contrib/restricted/boost +PEERDIR( + contrib/libs/snappy + contrib/restricted/boost contrib/restricted/boost/libs/iostreams -) - +) + ADDINCL( contrib/libs/apache/avro/api ) - -NO_COMPILER_WARNINGS() - -NO_UTIL() - -CFLAGS( - -DAVRO_SOURCE - -DSNAPPY_CODEC_AVAILABLE -) - -SRCS( - impl/BinaryDecoder.cc - impl/BinaryEncoder.cc - impl/Compiler.cc - impl/DataFile.cc - impl/FileStream.cc - impl/Generic.cc - impl/GenericDatum.cc - impl/LogicalType.cc - impl/Node.cc - impl/NodeImpl.cc - impl/Resolver.cc - impl/ResolverSchema.cc - impl/Schema.cc - impl/Stream.cc - impl/Types.cc - impl/ValidSchema.cc - impl/Validator.cc - impl/Zigzag.cc - impl/json/JsonDom.cc - impl/json/JsonIO.cc - impl/parsing/JsonCodec.cc - impl/parsing/ResolvingDecoder.cc - impl/parsing/Symbol.cc - impl/parsing/ValidatingCodec.cc -) - -END() + +NO_COMPILER_WARNINGS() + +NO_UTIL() + +CFLAGS( + -DAVRO_SOURCE + -DSNAPPY_CODEC_AVAILABLE +) + +SRCS( + impl/BinaryDecoder.cc + impl/BinaryEncoder.cc + impl/Compiler.cc + impl/DataFile.cc + impl/FileStream.cc + impl/Generic.cc + impl/GenericDatum.cc + impl/LogicalType.cc + impl/Node.cc + impl/NodeImpl.cc + impl/Resolver.cc + impl/ResolverSchema.cc + impl/Schema.cc + impl/Stream.cc + impl/Types.cc + impl/ValidSchema.cc + impl/Validator.cc + impl/Zigzag.cc + impl/json/JsonDom.cc + impl/json/JsonIO.cc + impl/parsing/JsonCodec.cc + impl/parsing/ResolvingDecoder.cc + impl/parsing/Symbol.cc + impl/parsing/ValidatingCodec.cc +) + +END() |