diff options
author | Aidar Samerkhanov <aidarsamer@yandex-team.ru> | 2022-02-09 18:18:45 +0300 |
---|---|---|
committer | Daniil Cherednik <dcherednik@yandex-team.ru> | 2022-02-10 15:58:17 +0300 |
commit | e65c50047c24f91dcd6454edcd9b0e4bf9a2ae2a (patch) | |
tree | 379a6851244b5c9ff9c3d7994c26126b9b712942 | |
parent | 542b4cb3a3bbb51b5a1cdedd117ee52a1e8f2032 (diff) | |
download | ydb-e65c50047c24f91dcd6454edcd9b0e4bf9a2ae2a.tar.gz |
KIKIMR-13365. Add workload commands to ydb cli.
KIKIMR-13365. Add workload command to YDB cli.
ref:d1b633b524f135ff2d0f23ee7534c1f67268be75
44 files changed, 5585 insertions, 42 deletions
diff --git a/contrib/libs/hdr_histogram/.yandex_meta/devtools.copyrights.report b/contrib/libs/hdr_histogram/.yandex_meta/devtools.copyrights.report new file mode 100644 index 0000000000..f3726e72e6 --- /dev/null +++ b/contrib/libs/hdr_histogram/.yandex_meta/devtools.copyrights.report @@ -0,0 +1,72 @@ +# File format ($ symbol means the beginning of a line): +# +# $ # this message +# $ # ======================= +# $ # comments (all commentaries should starts with some number of spaces and # symbol) +# ${action} {license id} {license text hash} +# $BELONGS ./ya/make/file/relative/path/1/ya.make ./ya/make/2/ya.make +# ${all_file_action} filename +# $ # user commentaries (many lines) +# $ generated description - files with this license, license text... (some number of lines that starts with some number of spaces, do not modify) +# ${action} {license spdx} {license text hash} +# $BELONGS ./ya/make/file/relative/path/3/ya.make +# ${all_file_action} filename +# $ # user commentaries +# $ generated description +# $ ... +# +# You can modify action, all_file_action and add commentaries +# Available actions: +# keep - keep license in contrib and use in credits +# skip - skip license +# remove - remove all files with this license +# rename - save license text/links into licenses texts file, but not store SPDX into LINCENSE macro. You should store correct license id into devtools.license.spdx.txt file +# +# {all file action} records will be generated when license text contains filename that exists on filesystem (in contrib directory) +# We suppose that that files can contain some license info +# Available all file actions: +# FILE_IGNORE - ignore file (do nothing) +# FILE_INCLUDE - include all file data into licenses text file +# ======================= + +KEEP COPYRIGHT_SERVICE_LABEL 7a0c945c7fd9793c36bc3f1ca40fa797 +BELONGS ya.make + License text: + Copyright (c) 2012, 2013, 2014 Gil Tene + Copyright (c) 2014 Michael Barker + Copyright (c) 2014 Matt Warren + All rights reserved. + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + LICENSE.txt [16:19] + +KEEP COPYRIGHT_SERVICE_LABEL 7c11f46ea34135c1e2037adce3f26a1a +BELONGS ya.make + License text: + Copyright (c) 2012, 2013, 2014 Gil Tene + Copyright (c) 2014 Michael Barker + Copyright (c) 2014 Matt Warren + All rights reserved. + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + LICENSE.txt [16:19] + +KEEP COPYRIGHT_SERVICE_LABEL d958332c87382cbd5c3451af3cfc976d +BELONGS ya.make + License text: + Copyright (c) 2012, 2013, 2014 Gil Tene + Copyright (c) 2014 Michael Barker + Copyright (c) 2014 Matt Warren + All rights reserved. + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + LICENSE.txt [16:19] diff --git a/contrib/libs/hdr_histogram/.yandex_meta/devtools.licenses.report b/contrib/libs/hdr_histogram/.yandex_meta/devtools.licenses.report new file mode 100644 index 0000000000..899e758b22 --- /dev/null +++ b/contrib/libs/hdr_histogram/.yandex_meta/devtools.licenses.report @@ -0,0 +1,141 @@ +# File format ($ symbol means the beginning of a line): +# +# $ # this message +# $ # ======================= +# $ # comments (all commentaries should starts with some number of spaces and # symbol) +# ${action} {license spdx} {license text hash} +# $BELONGS ./ya/make/file/relative/path/1/ya.make ./ya/make/2/ya.make +# ${all_file_action} filename +# $ # user commentaries (many lines) +# $ generated description - files with this license, license text... (some number of lines that starts with some number of spaces, do not modify) +# ${action} {license spdx} {license text hash} +# $BELONGS ./ya/make/file/relative/path/3/ya.make +# ${all_file_action} filename +# $ # user commentaries +# $ generated description +# $ ... +# +# You can modify action, all_file_action and add commentaries +# Available actions: +# keep - keep license in contrib and use in credits +# skip - skip license +# remove - remove all files with this license +# rename - save license text/links into licenses texts file, but not store SPDX into LINCENSE macro. You should store correct license id into devtools.license.spdx.txt file +# +# {all file action} records will be generated when license text contains filename that exists on filesystem (in contrib directory) +# We suppose that that files can contain some license info +# Available all file actions: +# FILE_IGNORE - ignore file (do nothing) +# FILE_INCLUDE - include all file data into licenses text file +# ======================= + +KEEP CC0-1.0 AND BSD-2-Clause 0d3a9d8603290e9ff13aea7ed6b4fb86 +BELONGS ya.make + Note: matched license text is too long. Read it in the source files. + Scancode info: + Original SPDX id: CC0-1.0 + Score : 100.00 + Match type : NOTICE + Links : http://creativecommons.org/publicdomain/zero/1.0/, http://creativecommons.org/publicdomain/zero/1.0/legalcode, https://spdx.org/licenses/CC0-1.0 + Files with this license: + LICENSE.txt [2:14] + Scancode info: + Original SPDX id: BSD-2-Clause + Score : 100.00 + Match type : NOTICE + Links : http://opensource.org/licenses/bsd-license.php, http://www.opensource.org/licenses/BSD-2-Clause, https://spdx.org/licenses/BSD-2-Clause + Files with this license: + LICENSE.txt [2:14] + +KEEP CC0-1.0 27442f52d8b635b92af00d2a85e22393 +BELONGS ya.make + License text: + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + Scancode info: + Original SPDX id: CC0-1.0 + Score : 100.00 + Match type : NOTICE + Links : http://creativecommons.org/publicdomain/zero/1.0/, http://creativecommons.org/publicdomain/zero/1.0/legalcode, https://spdx.org/licenses/CC0-1.0 + Files with this license: + src/hdr_histogram.c [3:4] + src/hdr_histogram.h [3:4] + src/hdr_interval_recorder.c [3:4] + src/hdr_interval_recorder.h [3:4] + src/hdr_time.h [3:4] + src/hdr_writer_reader_phaser.c [3:4] + src/hdr_writer_reader_phaser.h [3:4] + +KEEP CC0-1.0 3853e2a78a247145b4aa16667736f6de +BELONGS ya.make + Note: matched license text is too long. Read it in the source files. + Scancode info: + Original SPDX id: CC0-1.0 + Score : 100.00 + Match type : TEXT + Links : http://creativecommons.org/publicdomain/zero/1.0/, http://creativecommons.org/publicdomain/zero/1.0/legalcode, https://spdx.org/licenses/CC0-1.0 + Files with this license: + COPYING.txt [1:121] + +KEEP CC0-1.0 4bcc1b8e65be1d8f3d8b2ddb2ef94f6a +BELONGS ya.make + License text: + * Written by Philip Orwig and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + Scancode info: + Original SPDX id: CC0-1.0 + Score : 100.00 + Match type : NOTICE + Links : http://creativecommons.org/publicdomain/zero/1.0/, http://creativecommons.org/publicdomain/zero/1.0/legalcode, https://spdx.org/licenses/CC0-1.0 + Files with this license: + src/hdr_atomic.h [3:4] + src/hdr_thread.h [3:4] + +KEEP BSD-2-Clause 652477f3584c57bfd67ad3534ca70d66 +BELONGS ya.make + Note: matched license text is too long. Read it in the source files. + Scancode info: + Original SPDX id: BSD-2-Clause + Score : 100.00 + Match type : TEXT + Links : http://opensource.org/licenses/bsd-license.php, http://www.opensource.org/licenses/BSD-2-Clause, https://spdx.org/licenses/BSD-2-Clause + Files with this license: + LICENSE.txt [21:41] + +KEEP CC0-1.0 8996e50f093524d1bb4458d443c51d3d +BELONGS ya.make + License text: + * Released to the public domain, as explained at http://creativecommons.org/publicdomain/zero/1.0/ + Scancode info: + Original SPDX id: CC0-1.0 + Score : 100.00 + Match type : NOTICE + Links : http://creativecommons.org/publicdomain/zero/1.0/, http://creativecommons.org/publicdomain/zero/1.0/legalcode, https://spdx.org/licenses/CC0-1.0 + Files with this license: + src/hdr_endian.h [3:3] + +KEEP CC0-1.0 af8bc0f246a23fc16cb03d508c868432 +BELONGS ya.make + License text: + * Written by Michael Barker and Philip Orwig and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + Scancode info: + Original SPDX id: CC0-1.0 + Score : 100.00 + Match type : NOTICE + Links : http://creativecommons.org/publicdomain/zero/1.0/, http://creativecommons.org/publicdomain/zero/1.0/legalcode, https://spdx.org/licenses/CC0-1.0 + Files with this license: + src/hdr_time.c [3:4] + +KEEP CC0-1.0 e8fbb0dbcb7d5f1ad1664b4f404ba721 +BELONGS ya.make + License text: + * Written by Philip Orwig and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + Scancode info: + Original SPDX id: CC0-1.0 + Score : 100.00 + Match type : NOTICE + Links : http://creativecommons.org/publicdomain/zero/1.0/, http://creativecommons.org/publicdomain/zero/1.0/legalcode, https://spdx.org/licenses/CC0-1.0 + Files with this license: + src/hdr_thread.c [3:4] diff --git a/contrib/libs/hdr_histogram/.yandex_meta/licenses.list.txt b/contrib/libs/hdr_histogram/.yandex_meta/licenses.list.txt new file mode 100644 index 0000000000..303d814f64 --- /dev/null +++ b/contrib/libs/hdr_histogram/.yandex_meta/licenses.list.txt @@ -0,0 +1,191 @@ +====================BSD-2-Clause==================== + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + +====================CC0-1.0==================== + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + + +====================CC0-1.0==================== + * Written by Philip Orwig and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + + +====================CC0-1.0==================== +* Released to the public domain, as explained at http://creativecommons.org/publicdomain/zero/1.0/ + + +====================CC0-1.0==================== +* Written by Michael Barker and Philip Orwig and released to the public domain, +* as explained at http://creativecommons.org/publicdomain/zero/1.0/ + + +====================CC0-1.0==================== +* Written by Philip Orwig and released to the public domain, +* as explained at http://creativecommons.org/publicdomain/zero/1.0/ + + +====================CC0-1.0==================== +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. + +====================CC0-1.0 AND BSD-2-Clause==================== +and Matt Warren, and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +For users of this code who wish to consume it under the "BSD" license +rather than under the public domain or CC0 contribution text mentioned +above, the code found under this directory is *also* provided under the +following license (commonly referred to as the BSD 2-Clause License). This +license does not detract from the above stated release of the code into +the public domain, and simply represents an additional license granted by +the Author. + +----------------------------------------------------------------------------- +** Beginning of "BSD 2-Clause License" text. ** + + +====================COPYRIGHT==================== + Copyright (c) 2012, 2013, 2014 Gil Tene + Copyright (c) 2014 Michael Barker + Copyright (c) 2014 Matt Warren + All rights reserved. diff --git a/contrib/libs/hdr_histogram/COPYING.txt b/contrib/libs/hdr_histogram/COPYING.txt new file mode 100644 index 0000000000..0e259d42c9 --- /dev/null +++ b/contrib/libs/hdr_histogram/COPYING.txt @@ -0,0 +1,121 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. diff --git a/contrib/libs/hdr_histogram/LICENSE.txt b/contrib/libs/hdr_histogram/LICENSE.txt new file mode 100644 index 0000000000..9b4e66ed7d --- /dev/null +++ b/contrib/libs/hdr_histogram/LICENSE.txt @@ -0,0 +1,41 @@ +The code in this repository code was Written by Gil Tene, Michael Barker, +and Matt Warren, and released to the public domain, as explained at +http://creativecommons.org/publicdomain/zero/1.0/ + +For users of this code who wish to consume it under the "BSD" license +rather than under the public domain or CC0 contribution text mentioned +above, the code found under this directory is *also* provided under the +following license (commonly referred to as the BSD 2-Clause License). This +license does not detract from the above stated release of the code into +the public domain, and simply represents an additional license granted by +the Author. + +----------------------------------------------------------------------------- +** Beginning of "BSD 2-Clause License" text. ** + + Copyright (c) 2012, 2013, 2014 Gil Tene + Copyright (c) 2014 Michael Barker + Copyright (c) 2014 Matt Warren + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. diff --git a/contrib/libs/hdr_histogram/README.md b/contrib/libs/hdr_histogram/README.md new file mode 100644 index 0000000000..b669c3fe95 --- /dev/null +++ b/contrib/libs/hdr_histogram/README.md @@ -0,0 +1,79 @@ +HdrHistogram_c: 'C' port of High Dynamic Range (HDR) Histogram + +HdrHistogram +---------------------------------------------- +[![Gitter](https://badges.gitter.im/Join Chat.svg)](https://gitter.im/HdrHistogram/HdrHistogram?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +Windows Build: ![AppVeyor](https://ci.appveyor.com/api/projects/status/github/HdrHistogram/HdrHistogram_c) + +This port contains a subset of the functionality supported by the Java +implementation. The current supported features are: + +* Standard histogram with 64 bit counts (32/16 bit counts not supported) +* All iterator types (all values, recorded, percentiles, linear, logarithmic) +* Histogram serialisation (encoding version 1.2, decoding 1.0-1.2) +* Reader/writer phaser and interval recorder + +Features not supported, but planned + +* Auto-resizing of histograms + +Features unlikely to be implemented + +* Double histograms +* Atomic/Concurrent histograms +* 16/32 bit histograms + +# Simple Tutorial + +## Recording values + +```C +#include <hdr_histogram.h> + +struct hdr_histogram* histogram; + +// Initialise the histogram +hdr_init( + 1, // Minimum value + INT64_C(3600000000), // Maximum value + 3, // Number of significant figures + &histogram) // Pointer to initialise + +// Record value +hdr_record_value( + histogram, // Histogram to record to + value) // Value to record + +// Record value n times +hdr_record_values( + histogram, // Histogram to record to + value, // Value to record + 10) // Record value 10 times + +// Record value with correction for co-ordinated omission. +hdr_record_corrected_value( + histogram, // Histogram to record to + value, // Value to record + 1000) // Record with expected interval of 1000. + +// Print out the values of the histogram +hdr_percentiles_print( + histogram, + stdout, // File to write to + 5, // Granularity of printed values + 1.0, // Multiplier for results + CLASSIC); // Format CLASSIC/CSV supported. +``` + +## More examples + +For more detailed examples of recording and logging results look at the +[hdr_decoder](examples/hdr_decoder.c) +and [hiccup](examples/hiccup.c) +examples. You can run hiccup and decoder +and pipe the results of one into the other. + +``` +$ ./examples/hiccup | ./examples/hdr_decoder +``` diff --git a/contrib/libs/hdr_histogram/src/hdr_atomic.h b/contrib/libs/hdr_histogram/src/hdr_atomic.h new file mode 100644 index 0000000000..7015d985da --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_atomic.h @@ -0,0 +1,59 @@ +/** + * hdr_atomic.h + * Written by Philip Orwig and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#ifndef HDR_ATOMIC_H__ +#define HDR_ATOMIC_H__ + + +#if defined(_MSC_VER) + +#include <stdint.h> +#include <intrin.h> + +static void __inline * hdr_atomic_load_pointer(void** pointer) +{ + _ReadBarrier(); + return *pointer; +} + +static void hdr_atomic_store_pointer(void** pointer, void* value) +{ + _WriteBarrier(); + *pointer = value; +} + +static int64_t __inline hdr_atomic_load_64(int64_t* field) +{ + _ReadBarrier(); + return *field; +} + +static void __inline hdr_atomic_store_64(int64_t* field, int64_t value) +{ + _WriteBarrier(); + *field = value; +} + +static int64_t __inline hdr_atomic_exchange_64(volatile int64_t* field, int64_t initial) +{ + return _InterlockedExchange64(field, initial); +} + +static int64_t __inline hdr_atomic_add_fetch_64(volatile int64_t* field, int64_t value) +{ + return _InterlockedExchangeAdd64(field, value) + value; +} + +#else +#define hdr_atomic_load_pointer(x) __atomic_load_n(x, __ATOMIC_SEQ_CST) +#define hdr_atomic_store_pointer(f,v) __atomic_store_n(f,v, __ATOMIC_SEQ_CST) +#define hdr_atomic_load_64(x) __atomic_load_n(x, __ATOMIC_SEQ_CST) +#define hdr_atomic_store_64(f,v) __atomic_store_n(f,v, __ATOMIC_SEQ_CST) +#define hdr_atomic_exchange_64(f,i) __atomic_exchange_n(f,i, __ATOMIC_SEQ_CST) +#define hdr_atomic_add_fetch_64(field, value) __atomic_add_fetch(field, value, __ATOMIC_SEQ_CST) +#endif + +#endif /* HDR_ATOMIC_H__ */ diff --git a/contrib/libs/hdr_histogram/src/hdr_encoding.c b/contrib/libs/hdr_histogram/src/hdr_encoding.c new file mode 100644 index 0000000000..758cf68ebb --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_encoding.c @@ -0,0 +1,313 @@ +// +// Created by barkerm on 9/09/15. +// + +#include <errno.h> +#include <stddef.h> +#include <math.h> + +#include "hdr_encoding.h" +#include "hdr_tests.h" + +int zig_zag_encode_i64(uint8_t* buffer, int64_t signed_value) +{ + int64_t value = signed_value; + + value = (value << 1) ^ (value >> 63); + int bytesWritten = 0; + if (value >> 7 == 0) + { + buffer[0] = (uint8_t) value; + bytesWritten = 1; + } + else + { + buffer[0] = (uint8_t) ((value & 0x7F) | 0x80); + if (value >> 14 == 0) + { + buffer[1] = (uint8_t) (value >> 7); + bytesWritten = 2; + } + else + { + buffer[1] = (uint8_t) ((value >> 7 | 0x80)); + if (value >> 21 == 0) + { + buffer[2] = (uint8_t) (value >> 14); + bytesWritten = 3; + } + else + { + buffer[2] = (uint8_t) (value >> 14 | 0x80); + if (value >> 28 == 0) + { + buffer[3] = (uint8_t) (value >> 21); + bytesWritten = 4; + } + else + { + buffer[3] = (uint8_t) (value >> 21 | 0x80); + if (value >> 35 == 0) + { + buffer[4] = (uint8_t) (value >> 28); + bytesWritten = 5; + } + else + { + buffer[4] = (uint8_t) (value >> 28 | 0x80); + if (value >> 42 == 0) + { + buffer[5] = (uint8_t) (value >> 35); + bytesWritten = 6; + } + else + { + buffer[5] = (uint8_t) (value >> 35 | 0x80); + if (value >> 49 == 0) + { + buffer[6] = (uint8_t) (value >> 42); + bytesWritten = 7; + } + else + { + buffer[6] = (uint8_t) (value >> 42 | 0x80); + if (value >> 56 == 0) + { + buffer[7] = (uint8_t) (value >> 49); + bytesWritten = 8; + } + else + { + buffer[7] = (uint8_t) (value >> 49 | 0x80); + buffer[8] = (uint8_t) (value >> 56); + bytesWritten = 9; + } + } + } + } + } + } + } + } + + return bytesWritten; +} + +int zig_zag_decode_i64(const uint8_t* buffer, int64_t* retVal) +{ + uint64_t v = buffer[0]; + uint64_t value = v & 0x7F; + int bytesRead = 1; + if ((v & 0x80) != 0) + { + bytesRead = 2; + v = buffer[1]; + value |= (v & 0x7F) << 7; + if ((v & 0x80) != 0) + { + bytesRead = 3; + v = buffer[2]; + value |= (v & 0x7F) << 14; + if ((v & 0x80) != 0) + { + bytesRead = 4; + v = buffer[3]; + value |= (v & 0x7F) << 21; + if ((v & 0x80) != 0) + { + bytesRead = 5; + v = buffer[4]; + value |= (v & 0x7F) << 28; + if ((v & 0x80) != 0) + { + bytesRead = 6; + v = buffer[5]; + value |= (v & 0x7F) << 35; + if ((v & 0x80) != 0) + { + bytesRead = 7; + v = buffer[6]; + value |= (v & 0x7F) << 42; + if ((v & 0x80) != 0) + { + bytesRead = 8; + v = buffer[7]; + value |= (v & 0x7F) << 49; + if ((v & 0x80) != 0) + { + bytesRead = 9; + v = buffer[8]; + value |= v << 56; + } + } + } + } + } + } + } + } + + value = (value >> 1) ^ (-(value & 1)); + *retVal = (int64_t) value; + + return bytesRead; +} + +static const char base64_table[] = + { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/', '\0' + }; + +static char get_base_64(uint32_t _24_bit_value, int shift) +{ + uint32_t _6_bit_value = 0x3F & (_24_bit_value >> shift); + return base64_table[_6_bit_value]; +} + +static int from_base_64(int c) +{ + if ('A' <= c && c <= 'Z') + { + return c - 'A'; + } + else if ('a' <= c && c <= 'z') + { + return (c - 'a') + 26; + } + else if ('0' <= c && c <= '9') + { + return (c - '0') + 52; + } + else if ('+' == c) + { + return 62; + } + else if ('/' == c) + { + return 63; + } + else if ('=' == c) + { + return 0; + } + + return EINVAL; +} + +size_t hdr_base64_encoded_len(size_t decoded_size) +{ + return (size_t) (ceil(decoded_size / 3.0) * 4.0); +} + +size_t hdr_base64_decoded_len(size_t encoded_size) +{ + return (encoded_size / 4) * 3; +} + +static void hdr_base64_encode_block_pad(const uint8_t* input, char* output, size_t pad) +{ + uint32_t _24_bit_value = 0; + + switch (pad) + { + case 2: + _24_bit_value = (input[0] << 16) + (input[1] << 8); + + output[0] = get_base_64(_24_bit_value, 18); + output[1] = get_base_64(_24_bit_value, 12); + output[2] = get_base_64(_24_bit_value, 6); + output[3] = '='; + + break; + + case 1: + _24_bit_value = (input[0] << 16); + + output[0] = get_base_64(_24_bit_value, 18); + output[1] = get_base_64(_24_bit_value, 12); + output[2] = '='; + output[3] = '='; + + break; + + default: + // No-op + break; + } +} + +/** + * Assumes that there is 3 input bytes and 4 output chars. + */ +void hdr_base64_encode_block(const uint8_t* input, char* output) +{ + uint32_t _24_bit_value = (input[0] << 16) + (input[1] << 8) + (input[2]); + + output[0] = get_base_64(_24_bit_value, 18); + output[1] = get_base_64(_24_bit_value, 12); + output[2] = get_base_64(_24_bit_value, 6); + output[3] = get_base_64(_24_bit_value, 0); +} + +int hdr_base64_encode( + const uint8_t* input, size_t input_len, char* output, size_t output_len) +{ + if (hdr_base64_encoded_len(input_len) != output_len) + { + return EINVAL; + } + + size_t i = 0; + size_t j = 0; + for (; input_len - i >= 3 && j < output_len; i += 3, j += 4) + { + hdr_base64_encode_block(&input[i], &output[j]); + } + + size_t remaining = input_len - i; + + hdr_base64_encode_block_pad(&input[i], &output[j], remaining); + + return 0; +} + +/** + * Assumes that there is 4 input chars available and 3 output chars. + */ +void hdr_base64_decode_block(const char* input, uint8_t* output) +{ + uint32_t _24_bit_value = 0; + + _24_bit_value |= from_base_64(input[0]) << 18; + _24_bit_value |= from_base_64(input[1]) << 12; + _24_bit_value |= from_base_64(input[2]) << 6; + _24_bit_value |= from_base_64(input[3]); + + output[0] = (uint8_t) ((_24_bit_value >> 16) & 0xFF); + output[1] = (uint8_t) ((_24_bit_value >> 8) & 0xFF); + output[2] = (uint8_t) ((_24_bit_value) & 0xFF); +} + +int hdr_base64_decode( + const char* input, size_t input_len, uint8_t* output, size_t output_len) +{ + size_t i, j; + + if (input_len < 4 || + (input_len & 3) != 0 || + (input_len / 4) * 3 != output_len) + { + return EINVAL; + } + + for (i = 0, j = 0; i < input_len; i += 4, j += 3) + { + hdr_base64_decode_block(&input[i], &output[j]); + } + + return 0; +} diff --git a/contrib/libs/hdr_histogram/src/hdr_encoding.h b/contrib/libs/hdr_histogram/src/hdr_encoding.h new file mode 100644 index 0000000000..141093ba3b --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_encoding.h @@ -0,0 +1,77 @@ +// +// Created by barkerm on 9/09/15. +// + +#ifndef HDR_ENCODING_H +#define HDR_ENCODING_H + +#include <stdint.h> + +#define MAX_BYTES_LEB128 9 + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Writes a int64_t value to the given buffer in LEB128 ZigZag encoded format + * + * @param buffer the buffer to write to + * @param signed_value the value to write to the buffer + * @return the number of bytes written to the buffer + */ +int zig_zag_encode_i64(uint8_t* buffer, int64_t signed_value); + +/** + * Read an LEB128 ZigZag encoded long value from the given buffer + * + * @param buffer the buffer to read from + * @param retVal out value to capture the read value + * @return the number of bytes read from the buffer + */ +int zig_zag_decode_i64(const uint8_t* buffer, int64_t* signed_value); + +/** + * Gets the length in bytes of base64 data, given the input size. + * + * @param decoded_size the size of the unencoded values. + * @return the encoded size + */ +size_t hdr_base64_encoded_len(size_t decoded_size); + +/** + * Encode into base64. + * + * @param input the data to encode + * @param input_len the length of the data to encode + * @param output the buffer to write the output to + * @param output_len the number of bytes to write to the output + */ +int hdr_base64_encode( + const uint8_t* input, size_t input_len, char* output, size_t output_len); + +/** + * Gets the length in bytes of decoded base64 data, given the size of the base64 encoded + * data. + * + * @param encoded_size the size of the encoded value. + * @return the decoded size + */ +size_t hdr_base64_decoded_len(size_t encoded_size); + +/** + * Decode from base64. + * + * @param input the base64 encoded data + * @param input_len the size in bytes of the endcoded data + * @param output the buffer to write the decoded data to + * @param output_len the number of bytes to write to the output data + */ +int hdr_base64_decode( + const char* input, size_t input_len, uint8_t* output, size_t output_len); + +#ifdef __cplusplus +} +#endif + +#endif //HDR_HISTOGRAM_HDR_ENCODING_H diff --git a/contrib/libs/hdr_histogram/src/hdr_endian.h b/contrib/libs/hdr_histogram/src/hdr_endian.h new file mode 100644 index 0000000000..839fdb16b1 --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_endian.h @@ -0,0 +1,116 @@ +/** +* hdr_time.h +* Released to the public domain, as explained at http://creativecommons.org/publicdomain/zero/1.0/ +*/ + +#ifndef HDR_ENDIAN_H__ +#define HDR_ENDIAN_H__ + +#if (defined(_WIN16) || defined(_WIN32) || defined(_WIN64)) && !defined(__WINDOWS__) + +# define __WINDOWS__ + +#endif + +#if defined(__linux__) || defined(__CYGWIN__) + +# include <endian.h> + +#elif defined(__APPLE__) + +# include <libkern/OSByteOrder.h> + +# define htobe16(x) OSSwapHostToBigInt16(x) +# define htole16(x) OSSwapHostToLittleInt16(x) +# define be16toh(x) OSSwapBigToHostInt16(x) +# define le16toh(x) OSSwapLittleToHostInt16(x) + +# define htobe32(x) OSSwapHostToBigInt32(x) +# define htole32(x) OSSwapHostToLittleInt32(x) +# define be32toh(x) OSSwapBigToHostInt32(x) +# define le32toh(x) OSSwapLittleToHostInt32(x) + +# define htobe64(x) OSSwapHostToBigInt64(x) +# define htole64(x) OSSwapHostToLittleInt64(x) +# define be64toh(x) OSSwapBigToHostInt64(x) +# define le64toh(x) OSSwapLittleToHostInt64(x) + +# define __BYTE_ORDER BYTE_ORDER +# define __BIG_ENDIAN BIG_ENDIAN +# define __LITTLE_ENDIAN LITTLE_ENDIAN +# define __PDP_ENDIAN PDP_ENDIAN + +#elif defined(__OpenBSD__) + +# include <sys/endian.h> + +#elif defined(__NetBSD__) || defined(__FreeBSD__) || defined(__DragonFly__) + +# include <sys/endian.h> + +# define be16toh(x) betoh16(x) +# define le16toh(x) letoh16(x) + +# define be32toh(x) betoh32(x) +# define le32toh(x) letoh32(x) + +# define be64toh(x) betoh64(x) +# define le64toh(x) letoh64(x) + +#elif defined(__WINDOWS__) + +# include <winsock2.h> + +# if BYTE_ORDER == LITTLE_ENDIAN + +# define htobe16(x) htons(x) +# define htole16(x) (x) +# define be16toh(x) ntohs(x) +# define le16toh(x) (x) + +# define htobe32(x) htonl(x) +# define htole32(x) (x) +# define be32toh(x) ntohl(x) +# define le32toh(x) (x) + +# define htobe64(x) htonll(x) +# define htole64(x) (x) +# define be64toh(x) ntohll(x) +# define le64toh(x) (x) + +# elif BYTE_ORDER == BIG_ENDIAN + + /* that would be xbox 360 */ +# define htobe16(x) (x) +# define htole16(x) __builtin_bswap16(x) +# define be16toh(x) (x) +# define le16toh(x) __builtin_bswap16(x) + +# define htobe32(x) (x) +# define htole32(x) __builtin_bswap32(x) +# define be32toh(x) (x) +# define le32toh(x) __builtin_bswap32(x) + +# define htobe64(x) (x) +# define htole64(x) __builtin_bswap64(x) +# define be64toh(x) (x) +# define le64toh(x) __builtin_bswap64(x) + +# else + +# error byte order not supported + +# endif + +# define __BYTE_ORDER BYTE_ORDER +# define __BIG_ENDIAN BIG_ENDIAN +# define __LITTLE_ENDIAN LITTLE_ENDIAN +# define __PDP_ENDIAN PDP_ENDIAN + +#else + +# error platform not supported + +#endif + +#endif diff --git a/contrib/libs/hdr_histogram/src/hdr_histogram.c b/contrib/libs/hdr_histogram/src/hdr_histogram.c new file mode 100644 index 0000000000..8cad5a0d61 --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_histogram.c @@ -0,0 +1,1010 @@ +/** + * hdr_histogram.c + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#include <stdlib.h> +#include <stdbool.h> +#include <math.h> +#include <assert.h> +#include <stdio.h> +#include <string.h> +#include <stdint.h> +#include <errno.h> +#include <inttypes.h> + +#include "hdr_histogram.h" +#include "hdr_tests.h" + +// ###### ####### ## ## ## ## ######## ###### +// ## ## ## ## ## ## ### ## ## ## ## +// ## ## ## ## ## #### ## ## ## +// ## ## ## ## ## ## ## ## ## ###### +// ## ## ## ## ## ## #### ## ## +// ## ## ## ## ## ## ## ### ## ## ## +// ###### ####### ####### ## ## ## ###### + +static int32_t normalize_index(const struct hdr_histogram* h, int32_t index) +{ + if (h->normalizing_index_offset == 0) + { + return index; + } + + int32_t normalized_index = index - h->normalizing_index_offset; + int32_t adjustment = 0; + + if (normalized_index < 0) + { + adjustment = h->counts_len; + } + else if (normalized_index >= h->counts_len) + { + adjustment = -h->counts_len; + } + + return normalized_index + adjustment; +} + +static int64_t counts_get_direct(const struct hdr_histogram* h, int32_t index) +{ + return h->counts[index]; +} + +static int64_t counts_get_normalised(const struct hdr_histogram* h, int32_t index) +{ + return counts_get_direct(h, normalize_index(h, index)); +} + +static void counts_inc_normalised( + struct hdr_histogram* h, int32_t index, int64_t value) +{ + int32_t normalised_index = normalize_index(h, index); + h->counts[normalised_index] += value; + h->total_count += value; +} + +static void update_min_max(struct hdr_histogram* h, int64_t value) +{ + h->min_value = (value < h->min_value && value != 0) ? value : h->min_value; + h->max_value = (value > h->max_value) ? value : h->max_value; +} + +// ## ## ######## #### ## #### ######## ## ## +// ## ## ## ## ## ## ## ## ## +// ## ## ## ## ## ## ## #### +// ## ## ## ## ## ## ## ## +// ## ## ## ## ## ## ## ## +// ## ## ## ## ## ## ## ## +// ####### ## #### ######## #### ## ## + +static int64_t power(int64_t base, int64_t exp) +{ + int64_t result = 1; + while(exp) + { + result *= base; exp--; + } + return result; +} + +#if defined(_MSC_VER) +#pragma intrinsic(_BitScanReverse64) +#endif + +static int32_t get_bucket_index(const struct hdr_histogram* h, int64_t value) +{ +#if defined(_MSC_VER) + uint32_t leading_zero = 0; + _BitScanReverse64(&leading_zero, value | h->sub_bucket_mask); + int32_t pow2ceiling = 64 - (63 - leading_zero); // smallest power of 2 containing value +#else + int32_t pow2ceiling = 64 - __builtin_clzll(value | h->sub_bucket_mask); // smallest power of 2 containing value +#endif + return pow2ceiling - h->unit_magnitude - (h->sub_bucket_half_count_magnitude + 1); +} + +static int32_t get_sub_bucket_index(int64_t value, int32_t bucket_index, int32_t unit_magnitude) +{ + return (int32_t)(value >> (bucket_index + unit_magnitude)); +} + +static int32_t counts_index(const struct hdr_histogram* h, int32_t bucket_index, int32_t sub_bucket_index) +{ + // Calculate the index for the first entry in the bucket: + // (The following is the equivalent of ((bucket_index + 1) * subBucketHalfCount) ): + int32_t bucket_base_index = (bucket_index + 1) << h->sub_bucket_half_count_magnitude; + // Calculate the offset in the bucket: + int32_t offset_in_bucket = sub_bucket_index - h->sub_bucket_half_count; + // The following is the equivalent of ((sub_bucket_index - subBucketHalfCount) + bucketBaseIndex; + return bucket_base_index + offset_in_bucket; +} + +static int64_t value_from_index(int32_t bucket_index, int32_t sub_bucket_index, int32_t unit_magnitude) +{ + return ((int64_t) sub_bucket_index) << (bucket_index + unit_magnitude); +} + +int32_t counts_index_for(const struct hdr_histogram* h, int64_t value) +{ + int32_t bucket_index = get_bucket_index(h, value); + int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude); + + return counts_index(h, bucket_index, sub_bucket_index); +} + +int64_t hdr_value_at_index(const struct hdr_histogram *h, int32_t index) +{ + int32_t bucket_index = (index >> h->sub_bucket_half_count_magnitude) - 1; + int32_t sub_bucket_index = (index & (h->sub_bucket_half_count - 1)) + h->sub_bucket_half_count; + + if (bucket_index < 0) + { + sub_bucket_index -= h->sub_bucket_half_count; + bucket_index = 0; + } + + return value_from_index(bucket_index, sub_bucket_index, h->unit_magnitude); +} + +int64_t hdr_size_of_equivalent_value_range(const struct hdr_histogram* h, int64_t value) +{ + int32_t bucket_index = get_bucket_index(h, value); + int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude); + int32_t adjusted_bucket = (sub_bucket_index >= h->sub_bucket_count) ? (bucket_index + 1) : bucket_index; + return INT64_C(1) << (h->unit_magnitude + adjusted_bucket); +} + +static int64_t lowest_equivalent_value(const struct hdr_histogram* h, int64_t value) +{ + int32_t bucket_index = get_bucket_index(h, value); + int32_t sub_bucket_index = get_sub_bucket_index(value, bucket_index, h->unit_magnitude); + return value_from_index(bucket_index, sub_bucket_index, h->unit_magnitude); +} + +int64_t hdr_next_non_equivalent_value(const struct hdr_histogram *h, int64_t value) +{ + return lowest_equivalent_value(h, value) + hdr_size_of_equivalent_value_range(h, value); +} + +static int64_t highest_equivalent_value(const struct hdr_histogram* h, int64_t value) +{ + return hdr_next_non_equivalent_value(h, value) - 1; +} + +int64_t hdr_median_equivalent_value(const struct hdr_histogram *h, int64_t value) +{ + return lowest_equivalent_value(h, value) + (hdr_size_of_equivalent_value_range(h, value) >> 1); +} + +static int64_t non_zero_min(const struct hdr_histogram* h) +{ + if (INT64_MAX == h->min_value) + { + return INT64_MAX; + } + + return lowest_equivalent_value(h, h->min_value); +} + +void hdr_reset_internal_counters(struct hdr_histogram* h) +{ + int min_non_zero_index = -1; + int max_index = -1; + int64_t observed_total_count = 0; + int i; + + for (i = 0; i < h->counts_len; i++) + { + int64_t count_at_index; + + if ((count_at_index = counts_get_direct(h, i)) > 0) + { + observed_total_count += count_at_index; + max_index = i; + if (min_non_zero_index == -1 && i != 0) + { + min_non_zero_index = i; + } + } + } + + if (max_index == -1) + { + h->max_value = 0; + } + else + { + int64_t max_value = hdr_value_at_index(h, max_index); + h->max_value = highest_equivalent_value(h, max_value); + } + + if (min_non_zero_index == -1) + { + h->min_value = INT64_MAX; + } + else + { + h->min_value = hdr_value_at_index(h, min_non_zero_index); + } + + h->total_count = observed_total_count; +} + +static int32_t buckets_needed_to_cover_value(int64_t value, int32_t sub_bucket_count, int32_t unit_magnitude) +{ + int64_t smallest_untrackable_value = ((int64_t) sub_bucket_count) << unit_magnitude; + int32_t buckets_needed = 1; + while (smallest_untrackable_value <= value) + { + if (smallest_untrackable_value > INT64_MAX / 2) + { + return buckets_needed + 1; + } + smallest_untrackable_value <<= 1; + buckets_needed++; + } + + return buckets_needed; +} + +// ## ## ######## ## ## ####### ######## ## ## +// ### ### ## ### ### ## ## ## ## ## ## +// #### #### ## #### #### ## ## ## ## #### +// ## ### ## ###### ## ### ## ## ## ######## ## +// ## ## ## ## ## ## ## ## ## ## +// ## ## ## ## ## ## ## ## ## ## +// ## ## ######## ## ## ####### ## ## ## + +int hdr_calculate_bucket_config( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram_bucket_config* cfg) +{ + if (lowest_trackable_value < 1 || + significant_figures < 1 || 5 < significant_figures) + { + return EINVAL; + } + else if (lowest_trackable_value * 2 > highest_trackable_value) + { + return EINVAL; + } + + cfg->lowest_trackable_value = lowest_trackable_value; + cfg->significant_figures = significant_figures; + cfg->highest_trackable_value = highest_trackable_value; + + int64_t largest_value_with_single_unit_resolution = 2 * power(10, significant_figures); + int32_t sub_bucket_count_magnitude = (int32_t) ceil(log((double)largest_value_with_single_unit_resolution) / log(2)); + cfg->sub_bucket_half_count_magnitude = ((sub_bucket_count_magnitude > 1) ? sub_bucket_count_magnitude : 1) - 1; + + cfg->unit_magnitude = (int32_t) floor(log((double)lowest_trackable_value) / log(2)); + + cfg->sub_bucket_count = (int32_t) pow(2, (cfg->sub_bucket_half_count_magnitude + 1)); + cfg->sub_bucket_half_count = cfg->sub_bucket_count / 2; + cfg->sub_bucket_mask = ((int64_t) cfg->sub_bucket_count - 1) << cfg->unit_magnitude; + + // determine exponent range needed to support the trackable value with no overflow: + cfg->bucket_count = buckets_needed_to_cover_value(highest_trackable_value, cfg->sub_bucket_count, (int32_t)cfg->unit_magnitude); + cfg->counts_len = (cfg->bucket_count + 1) * (cfg->sub_bucket_count / 2); + + return 0; +} + +void hdr_init_preallocated(struct hdr_histogram* h, struct hdr_histogram_bucket_config* cfg) +{ + h->lowest_trackable_value = cfg->lowest_trackable_value; + h->highest_trackable_value = cfg->highest_trackable_value; + h->unit_magnitude = (int32_t)cfg->unit_magnitude; + h->significant_figures = (int32_t)cfg->significant_figures; + h->sub_bucket_half_count_magnitude = cfg->sub_bucket_half_count_magnitude; + h->sub_bucket_half_count = cfg->sub_bucket_half_count; + h->sub_bucket_mask = cfg->sub_bucket_mask; + h->sub_bucket_count = cfg->sub_bucket_count; + h->min_value = INT64_MAX; + h->max_value = 0; + h->normalizing_index_offset = 0; + h->conversion_ratio = 1.0; + h->bucket_count = cfg->bucket_count; + h->counts_len = cfg->counts_len; + h->total_count = 0; +} + +int hdr_init( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram** result) +{ + struct hdr_histogram_bucket_config cfg; + + int r = hdr_calculate_bucket_config(lowest_trackable_value, highest_trackable_value, significant_figures, &cfg); + if (r) + { + return r; + } + + size_t histogram_size = sizeof(struct hdr_histogram) + cfg.counts_len * sizeof(int64_t); + struct hdr_histogram* histogram = malloc(histogram_size); + + if (!histogram) + { + return ENOMEM; + } + + // memset will ensure that all of the function pointers are null. + memset((void*) histogram, 0, histogram_size); + + hdr_init_preallocated(histogram, &cfg); + *result = histogram; + + return 0; +} + + +int hdr_alloc(int64_t highest_trackable_value, int significant_figures, struct hdr_histogram** result) +{ + return hdr_init(1, highest_trackable_value, significant_figures, result); +} + +// reset a histogram to zero. +void hdr_reset(struct hdr_histogram *h) +{ + h->total_count=0; + h->min_value = INT64_MAX; + h->max_value = 0; + memset((void *) &h->counts, 0, (sizeof(int64_t) * h->counts_len)); + return; +} + +size_t hdr_get_memory_size(struct hdr_histogram *h) +{ + return sizeof(struct hdr_histogram) + h->counts_len * sizeof(int64_t); +} + +// ## ## ######## ######## ### ######## ######## ###### +// ## ## ## ## ## ## ## ## ## ## ## ## +// ## ## ## ## ## ## ## ## ## ## ## +// ## ## ######## ## ## ## ## ## ###### ###### +// ## ## ## ## ## ######### ## ## ## +// ## ## ## ## ## ## ## ## ## ## ## +// ####### ## ######## ## ## ## ######## ###### + + +bool hdr_record_value(struct hdr_histogram* h, int64_t value) +{ + return hdr_record_values(h, value, 1); +} + +bool hdr_record_values(struct hdr_histogram* h, int64_t value, int64_t count) +{ + if (value < 0) + { + return false; + } + + int32_t counts_index = counts_index_for(h, value); + + if (counts_index < 0 || h->counts_len <= counts_index) + { + return false; + } + + counts_inc_normalised(h, counts_index, count); + update_min_max(h, value); + + return true; +} + +bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t expected_interval) +{ + return hdr_record_corrected_values(h, value, 1, expected_interval); +} + + +bool hdr_record_corrected_values(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval) +{ + if (!hdr_record_values(h, value, count)) + { + return false; + } + + if (expected_interval <= 0 || value <= expected_interval) + { + return true; + } + + int64_t missing_value = value - expected_interval; + for (; missing_value >= expected_interval; missing_value -= expected_interval) + { + if (!hdr_record_values(h, missing_value, count)) + { + return false; + } + } + + return true; +} + +int64_t hdr_add(struct hdr_histogram* h, const struct hdr_histogram* from) +{ + struct hdr_iter iter; + hdr_iter_recorded_init(&iter, from); + int64_t dropped = 0; + + while (hdr_iter_next(&iter)) + { + int64_t value = iter.value; + int64_t count = iter.count; + + if (!hdr_record_values(h, value, count)) + { + dropped += count; + } + } + + return dropped; +} + +int64_t hdr_add_while_correcting_for_coordinated_omission( + struct hdr_histogram* h, struct hdr_histogram* from, int64_t expected_interval) +{ + struct hdr_iter iter; + hdr_iter_recorded_init(&iter, from); + int64_t dropped = 0; + + while (hdr_iter_next(&iter)) + { + int64_t value = iter.value; + int64_t count = iter.count; + + if (!hdr_record_corrected_values(h, value, count, expected_interval)) + { + dropped += count; + } + } + + return dropped; +} + + + +// ## ## ### ## ## ## ######## ###### +// ## ## ## ## ## ## ## ## ## ## +// ## ## ## ## ## ## ## ## ## +// ## ## ## ## ## ## ## ###### ###### +// ## ## ######### ## ## ## ## ## +// ## ## ## ## ## ## ## ## ## ## +// ### ## ## ######## ####### ######## ###### + + +int64_t hdr_max(const struct hdr_histogram* h) +{ + if (0 == h->max_value) + { + return 0; + } + + return highest_equivalent_value(h, h->max_value); +} + +int64_t hdr_min(const struct hdr_histogram* h) +{ + if (0 < hdr_count_at_index(h, 0)) + { + return 0; + } + + return non_zero_min(h); +} + +int64_t hdr_value_at_percentile(const struct hdr_histogram* h, double percentile) +{ + struct hdr_iter iter; + hdr_iter_init(&iter, h); + + double requested_percentile = percentile < 100.0 ? percentile : 100.0; + int64_t count_at_percentile = + (int64_t) (((requested_percentile / 100) * h->total_count) + 0.5); + count_at_percentile = count_at_percentile > 1 ? count_at_percentile : 1; + int64_t total = 0; + + while (hdr_iter_next(&iter)) + { + total += iter.count; + + if (total >= count_at_percentile) + { + int64_t value_from_index = iter.value; + return highest_equivalent_value(h, value_from_index); + } + } + + return 0; +} + +double hdr_mean(const struct hdr_histogram* h) +{ + struct hdr_iter iter; + int64_t total = 0; + + hdr_iter_init(&iter, h); + + while (hdr_iter_next(&iter)) + { + if (0 != iter.count) + { + total += iter.count * hdr_median_equivalent_value(h, iter.value); + } + } + + return (total * 1.0) / h->total_count; +} + +double hdr_stddev(const struct hdr_histogram* h) +{ + double mean = hdr_mean(h); + double geometric_dev_total = 0.0; + + struct hdr_iter iter; + hdr_iter_init(&iter, h); + + while (hdr_iter_next(&iter)) + { + if (0 != iter.count) + { + double dev = (hdr_median_equivalent_value(h, iter.value) * 1.0) - mean; + geometric_dev_total += (dev * dev) * iter.count; + } + } + + return sqrt(geometric_dev_total / h->total_count); +} + +bool hdr_values_are_equivalent(const struct hdr_histogram* h, int64_t a, int64_t b) +{ + return lowest_equivalent_value(h, a) == lowest_equivalent_value(h, b); +} + +int64_t hdr_lowest_equivalent_value(const struct hdr_histogram* h, int64_t value) +{ + return lowest_equivalent_value(h, value); +} + +int64_t hdr_count_at_value(const struct hdr_histogram* h, int64_t value) +{ + return counts_get_normalised(h, counts_index_for(h, value)); +} + +int64_t hdr_count_at_index(const struct hdr_histogram* h, int32_t index) +{ + return counts_get_normalised(h, index); +} + + +// #### ######## ######## ######## ### ######## ####### ######## ###### +// ## ## ## ## ## ## ## ## ## ## ## ## ## ## +// ## ## ## ## ## ## ## ## ## ## ## ## ## +// ## ## ###### ######## ## ## ## ## ## ######## ###### +// ## ## ## ## ## ######### ## ## ## ## ## ## +// ## ## ## ## ## ## ## ## ## ## ## ## ## ## +// #### ## ######## ## ## ## ## ## ####### ## ## ###### + + +static bool has_buckets(struct hdr_iter* iter) +{ + return iter->counts_index < iter->h->counts_len; +} + +static bool has_next(struct hdr_iter* iter) +{ + return iter->cumulative_count < iter->total_count; +} + +static bool move_next(struct hdr_iter* iter) +{ + iter->counts_index++; + + if (!has_buckets(iter)) + { + return false; + } + + iter->count = counts_get_normalised(iter->h, iter->counts_index); + iter->cumulative_count += iter->count; + + iter->value = hdr_value_at_index(iter->h, iter->counts_index); + iter->highest_equivalent_value = highest_equivalent_value(iter->h, iter->value); + iter->lowest_equivalent_value = lowest_equivalent_value(iter->h, iter->value); + iter->median_equivalent_value = hdr_median_equivalent_value(iter->h, iter->value); + + return true; +} + +static int64_t peek_next_value_from_index(struct hdr_iter* iter) +{ + return hdr_value_at_index(iter->h, iter->counts_index + 1); +} + +static bool next_value_greater_than_reporting_level_upper_bound( + struct hdr_iter *iter, int64_t reporting_level_upper_bound) +{ + if (iter->counts_index >= iter->h->counts_len) + { + return false; + } + + return peek_next_value_from_index(iter) > reporting_level_upper_bound; +} + +static bool _basic_iter_next(struct hdr_iter *iter) +{ + if (!has_next(iter)) + { + return false; + } + + move_next(iter); + + return true; +} + +static void _update_iterated_values(struct hdr_iter* iter, int64_t new_value_iterated_to) +{ + iter->value_iterated_from = iter->value_iterated_to; + iter->value_iterated_to = new_value_iterated_to; +} + +static bool _all_values_iter_next(struct hdr_iter* iter) +{ + bool result = move_next(iter); + + if (result) + { + _update_iterated_values(iter, iter->value); + } + + return result; +} + +void hdr_iter_init(struct hdr_iter* iter, const struct hdr_histogram* h) +{ + iter->h = h; + + iter->counts_index = -1; + iter->total_count = h->total_count; + iter->count = 0; + iter->cumulative_count = 0; + iter->value = 0; + iter->highest_equivalent_value = 0; + iter->value_iterated_from = 0; + iter->value_iterated_to = 0; + + iter->_next_fp = _all_values_iter_next; +} + +bool hdr_iter_next(struct hdr_iter* iter) +{ + return iter->_next_fp(iter); +} + +// ######## ######## ######## ###### ######## ## ## ######## #### ## ######## ###### +// ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## ## +// ## ## ## ## ## ## ## #### ## ## ## ## ## ## +// ######## ###### ######## ## ###### ## ## ## ## ## ## ###### ###### +// ## ## ## ## ## ## ## #### ## ## ## ## ## +// ## ## ## ## ## ## ## ## ### ## ## ## ## ## ## +// ## ######## ## ## ###### ######## ## ## ## #### ######## ######## ###### + +static bool _percentile_iter_next(struct hdr_iter* iter) +{ + struct hdr_iter_percentiles* percentiles = &iter->specifics.percentiles; + + if (!has_next(iter)) + { + if (percentiles->seen_last_value) + { + return false; + } + + percentiles->seen_last_value = true; + percentiles->percentile = 100.0; + + return true; + } + + if (iter->counts_index == -1 && !_basic_iter_next(iter)) + { + return false; + } + + do + { + double current_percentile = (100.0 * (double) iter->cumulative_count) / iter->h->total_count; + if (iter->count != 0 && + percentiles->percentile_to_iterate_to <= current_percentile) + { + _update_iterated_values(iter, highest_equivalent_value(iter->h, iter->value)); + + percentiles->percentile = percentiles->percentile_to_iterate_to; + int64_t temp = (int64_t)(log(100 / (100.0 - (percentiles->percentile_to_iterate_to))) / log(2)) + 1; + int64_t half_distance = (int64_t) pow(2, (double) temp); + int64_t percentile_reporting_ticks = percentiles->ticks_per_half_distance * half_distance; + percentiles->percentile_to_iterate_to += 100.0 / percentile_reporting_ticks; + + return true; + } + } + while (_basic_iter_next(iter)); + + return true; +} + +void hdr_iter_percentile_init(struct hdr_iter* iter, const struct hdr_histogram* h, int32_t ticks_per_half_distance) +{ + iter->h = h; + + hdr_iter_init(iter, h); + + iter->specifics.percentiles.seen_last_value = false; + iter->specifics.percentiles.ticks_per_half_distance = ticks_per_half_distance; + iter->specifics.percentiles.percentile_to_iterate_to = 0.0; + iter->specifics.percentiles.percentile = 0.0; + + iter->_next_fp = _percentile_iter_next; +} + +static void format_line_string(char* str, size_t len, int significant_figures, format_type format) +{ +#if defined(_MSC_VER) +#define snprintf _snprintf +#pragma warning(push) +#pragma warning(disable: 4996) +#endif + const char* format_str = "%s%d%s"; + + switch (format) + { + case CSV: + snprintf(str, len, format_str, "%.", significant_figures, "f,%f,%d,%.2f\n"); + break; + case CLASSIC: + snprintf(str, len, format_str, "%12.", significant_figures, "f %12f %12d %12.2f\n"); + break; + default: + snprintf(str, len, format_str, "%12.", significant_figures, "f %12f %12d %12.2f\n"); + } +#if defined(_MSC_VER) +#undef snprintf +#pragma warning(pop) +#endif +} + + +// ######## ######## ###### ####### ######## ######## ######## ######## +// ## ## ## ## ## ## ## ## ## ## ## ## ## ## +// ## ## ## ## ## ## ## ## ## ## ## ## ## +// ######## ###### ## ## ## ######## ## ## ###### ## ## +// ## ## ## ## ## ## ## ## ## ## ## ## ## +// ## ## ## ## ## ## ## ## ## ## ## ## ## ## +// ## ## ######## ###### ####### ## ## ######## ######## ######## + + +static bool _recorded_iter_next(struct hdr_iter* iter) +{ + while (_basic_iter_next(iter)) + { + if (iter->count != 0) + { + _update_iterated_values(iter, iter->value); + + iter->specifics.recorded.count_added_in_this_iteration_step = iter->count; + return true; + } + } + + return false; +} + +void hdr_iter_recorded_init(struct hdr_iter* iter, const struct hdr_histogram* h) +{ + hdr_iter_init(iter, h); + + iter->specifics.recorded.count_added_in_this_iteration_step = 0; + + iter->_next_fp = _recorded_iter_next; +} + +// ## #### ## ## ######## ### ######## +// ## ## ### ## ## ## ## ## ## +// ## ## #### ## ## ## ## ## ## +// ## ## ## ## ## ###### ## ## ######## +// ## ## ## #### ## ######### ## ## +// ## ## ## ### ## ## ## ## ## +// ######## #### ## ## ######## ## ## ## ## + + +static bool _iter_linear_next(struct hdr_iter* iter) +{ + struct hdr_iter_linear* linear = &iter->specifics.linear; + + linear->count_added_in_this_iteration_step = 0; + + if (has_next(iter) || + next_value_greater_than_reporting_level_upper_bound( + iter, linear->next_value_reporting_level_lowest_equivalent)) + { + do + { + if (iter->value >= linear->next_value_reporting_level_lowest_equivalent) + { + _update_iterated_values(iter, linear->next_value_reporting_level); + + linear->next_value_reporting_level += linear->value_units_per_bucket; + linear->next_value_reporting_level_lowest_equivalent = + lowest_equivalent_value(iter->h, linear->next_value_reporting_level); + + return true; + } + + if (!move_next(iter)) + { + return true; + } + + linear->count_added_in_this_iteration_step += iter->count; + } + while (true); + } + + return false; +} + + +void hdr_iter_linear_init(struct hdr_iter* iter, const struct hdr_histogram* h, int64_t value_units_per_bucket) +{ + hdr_iter_init(iter, h); + + iter->specifics.linear.count_added_in_this_iteration_step = 0; + iter->specifics.linear.value_units_per_bucket = value_units_per_bucket; + iter->specifics.linear.next_value_reporting_level = value_units_per_bucket; + iter->specifics.linear.next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(h, value_units_per_bucket); + + iter->_next_fp = _iter_linear_next; +} + +// ## ####### ###### ### ######## #### ######## ## ## ## ## #### ###### +// ## ## ## ## ## ## ## ## ## ## ## ## ## ### ### ## ## ## +// ## ## ## ## ## ## ## ## ## ## ## ## #### #### ## ## +// ## ## ## ## #### ## ## ######## ## ## ######### ## ### ## ## ## +// ## ## ## ## ## ######### ## ## ## ## ## ## ## ## ## ## +// ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## ## +// ######## ####### ###### ## ## ## ## #### ## ## ## ## ## #### ###### + +static bool _log_iter_next(struct hdr_iter *iter) +{ + struct hdr_iter_log* logarithmic = &iter->specifics.log; + + logarithmic->count_added_in_this_iteration_step = 0; + + if (has_next(iter) || + next_value_greater_than_reporting_level_upper_bound( + iter, logarithmic->next_value_reporting_level_lowest_equivalent)) + { + do + { + if (iter->value >= logarithmic->next_value_reporting_level_lowest_equivalent) + { + _update_iterated_values(iter, logarithmic->next_value_reporting_level); + + logarithmic->next_value_reporting_level *= (int64_t)logarithmic->log_base; + logarithmic->next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(iter->h, logarithmic->next_value_reporting_level); + + return true; + } + + if (!move_next(iter)) + { + return true; + } + + logarithmic->count_added_in_this_iteration_step += iter->count; + } + while (true); + } + + return false; +} + +void hdr_iter_log_init( + struct hdr_iter* iter, + const struct hdr_histogram* h, + int64_t value_units_first_bucket, + double log_base) +{ + hdr_iter_init(iter, h); + iter->specifics.log.count_added_in_this_iteration_step = 0; + iter->specifics.log.log_base = log_base; + iter->specifics.log.next_value_reporting_level = value_units_first_bucket; + iter->specifics.log.next_value_reporting_level_lowest_equivalent = lowest_equivalent_value(h, value_units_first_bucket); + + iter->_next_fp = _log_iter_next; +} + +// Printing. + +static const char* format_head_string(format_type format) +{ + switch (format) + { + case CSV: + return "%s,%s,%s,%s\n"; + case CLASSIC: + return "%12s %12s %12s %12s\n\n"; + default: + return "%12s %12s %12s %12s\n\n"; + } +} + +static const char CLASSIC_FOOTER[] = + "#[Mean = %12.3f, StdDeviation = %12.3f]\n" + "#[Max = %12.3f, Total count = %12" PRIu64 "]\n" + "#[Buckets = %12d, SubBuckets = %12d]\n"; + +int hdr_percentiles_print( + struct hdr_histogram* h, FILE* stream, int32_t ticks_per_half_distance, + double value_scale, format_type format) +{ + char line_format[25]; + format_line_string(line_format, 25, h->significant_figures, format); + const char* head_format = format_head_string(format); + int rc = 0; + + struct hdr_iter iter; + hdr_iter_percentile_init(&iter, h, ticks_per_half_distance); + + if (fprintf( + stream, head_format, + "Value", "Percentile", "TotalCount", "1/(1-Percentile)") < 0) + { + rc = EIO; + goto cleanup; + } + + struct hdr_iter_percentiles * percentiles = &iter.specifics.percentiles; + while (hdr_iter_next(&iter)) + { + double value = iter.highest_equivalent_value / value_scale; + double percentile = percentiles->percentile / 100.0; + int64_t total_count = iter.cumulative_count; + double inverted_percentile = (1.0 / (1.0 - percentile)); + + if (fprintf( + stream, line_format, value, percentile, total_count, inverted_percentile) < 0) + { + rc = EIO; + goto cleanup; + } + } + + if (CLASSIC == format) + { + double mean = hdr_mean(h) / value_scale; + double stddev = hdr_stddev(h) / value_scale; + double max = hdr_max(h) / value_scale; + + if (fprintf( + stream, CLASSIC_FOOTER, mean, stddev, max, + h->total_count, h->bucket_count, h->sub_bucket_count) < 0) + { + rc = EIO; + goto cleanup; + } + } + + cleanup: + return rc; +} diff --git a/contrib/libs/hdr_histogram/src/hdr_histogram.h b/contrib/libs/hdr_histogram/src/hdr_histogram.h new file mode 100644 index 0000000000..c4ba1a43f9 --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_histogram.h @@ -0,0 +1,428 @@ +/** + * hdr_histogram.h + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + * + * The source for the hdr_histogram utilises a few C99 constructs, specifically + * the use of stdint/stdbool and inline variable declaration. + */ + +#ifndef HDR_HISTOGRAM_H +#define HDR_HISTOGRAM_H 1 + +#include <stdint.h> +#include <stdbool.h> +#include <stdio.h> + +struct hdr_histogram +{ + int64_t lowest_trackable_value; + int64_t highest_trackable_value; + int32_t unit_magnitude; + int32_t significant_figures; + int32_t sub_bucket_half_count_magnitude; + int32_t sub_bucket_half_count; + int64_t sub_bucket_mask; + int32_t sub_bucket_count; + int32_t bucket_count; + int64_t min_value; + int64_t max_value; + int32_t normalizing_index_offset; + double conversion_ratio; + int32_t counts_len; + int64_t total_count; + int64_t counts[0]; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Allocate the memory and initialise the hdr_histogram. + * + * Due to the size of the histogram being the result of some reasonably + * involved math on the input parameters this function it is tricky to stack allocate. + * The histogram is allocated in a single contigious block so can be delete via free, + * without any structure specific destructor. + * + * @param lowest_trackable_value The smallest possible value to be put into the + * histogram. + * @param highest_trackable_value The largest possible value to be put into the + * histogram. + * @param significant_figures The level of precision for this histogram, i.e. the number + * of figures in a decimal number that will be maintained. E.g. a value of 3 will mean + * the results from the histogram will be accurate up to the first three digits. Must + * be a value between 1 and 5 (inclusive). + * @param result Output parameter to capture allocated histogram. + * @return 0 on success, EINVAL if lowest_trackable_value is < 1 or the + * significant_figure value is outside of the allowed range, ENOMEM if malloc + * failed. + */ +int hdr_init( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram** result); + +/** + * Allocate the memory and initialise the hdr_histogram. This is the equivalent of calling + * hdr_init(1, highest_trackable_value, significant_figures, result); + * + * @deprecated use hdr_init. + */ +int hdr_alloc(int64_t highest_trackable_value, int significant_figures, struct hdr_histogram** result); + + +/** + * Reset a histogram to zero - empty out a histogram and re-initialise it + * + * If you want to re-use an existing histogram, but reset everything back to zero, this + * is the routine to use. + * + * @param h The histogram you want to reset to empty. + * + */ +void hdr_reset(struct hdr_histogram* h); + +/** + * Get the memory size of the hdr_histogram. + * + * @param h "This" pointer + * @return The amount of memory used by the hdr_histogram in bytes + */ +size_t hdr_get_memory_size(struct hdr_histogram* h); + +/** + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at construction time. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_value(struct hdr_histogram* h, int64_t value); + +/** + * Records count values in the histogram, will round this value of to a + * precision at or better than the significant_figure specified at construction + * time. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @return false if any value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_values(struct hdr_histogram* h, int64_t value, int64_t count); + + +/** + * Record a value in the histogram and backfill based on an expected interval. + * + * Records a value in the histogram, will round this value of to a precision at or better + * than the significant_figure specified at contruction time. This is specifically used + * for recording latency. If the value is larger than the expected_interval then the + * latency recording system has experienced co-ordinated omission. This method fills in the + * values that would have occured had the client providing the load not been blocked. + + * @param h "This" pointer + * @param value Value to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_value(struct hdr_histogram* h, int64_t value, int64_t expexcted_interval); +/** + * Record a value in the histogram 'count' times. Applies the same correcting logic + * as 'hdr_record_corrected_value'. + * + * @param h "This" pointer + * @param value Value to add to the histogram + * @param count Number of 'value's to add to the histogram + * @param expected_interval The delay between recording values. + * @return false if the value is larger than the highest_trackable_value and can't be recorded, + * true otherwise. + */ +bool hdr_record_corrected_values(struct hdr_histogram* h, int64_t value, int64_t count, int64_t expected_interval); + +/** + * Adds all of the values from 'from' to 'this' histogram. Will return the + * number of values that are dropped when copying. Values will be dropped + * if they around outside of h.lowest_trackable_value and + * h.highest_trackable_value. + * + * @param h "This" pointer + * @param from Histogram to copy values from. + * @return The number of values dropped when copying. + */ +int64_t hdr_add(struct hdr_histogram* h, const struct hdr_histogram* from); + +/** + * Adds all of the values from 'from' to 'this' histogram. Will return the + * number of values that are dropped when copying. Values will be dropped + * if they around outside of h.lowest_trackable_value and + * h.highest_trackable_value. + * + * @param h "This" pointer + * @param from Histogram to copy values from. + * @return The number of values dropped when copying. + */ +int64_t hdr_add_while_correcting_for_coordinated_omission( + struct hdr_histogram* h, struct hdr_histogram* from, int64_t expected_interval); + +/** + * Get minimum value from the histogram. Will return 2^63-1 if the histogram + * is empty. + * + * @param h "This" pointer + */ +int64_t hdr_min(const struct hdr_histogram* h); + +/** + * Get maximum value from the histogram. Will return 0 if the histogram + * is empty. + * + * @param h "This" pointer + */ +int64_t hdr_max(const struct hdr_histogram* h); + +/** + * Get the value at a specific percentile. + * + * @param h "This" pointer. + * @param percentile The percentile to get the value for + */ +int64_t hdr_value_at_percentile(const struct hdr_histogram* h, double percentile); + +/** + * Gets the standard deviation for the values in the histogram. + * + * @param h "This" pointer + * @return The standard deviation + */ +double hdr_stddev(const struct hdr_histogram* h); + +/** + * Gets the mean for the values in the histogram. + * + * @param h "This" pointer + * @return The mean + */ +double hdr_mean(const struct hdr_histogram* h); + +/** + * Determine if two values are equivalent with the histogram's resolution. + * Where "equivalent" means that value samples recorded for any two + * equivalent values are counted in a common total count. + * + * @param h "This" pointer + * @param a first value to compare + * @param b second value to compare + * @return 'true' if values are equivalent with the histogram's resolution. + */ +bool hdr_values_are_equivalent(const struct hdr_histogram* h, int64_t a, int64_t b); + +/** + * Get the lowest value that is equivalent to the given value within the histogram's resolution. + * Where "equivalent" means that value samples recorded for any two + * equivalent values are counted in a common total count. + * + * @param h "This" pointer + * @param value The given value + * @return The lowest value that is equivalent to the given value within the histogram's resolution. + */ +int64_t hdr_lowest_equivalent_value(const struct hdr_histogram* h, int64_t value); + +/** + * Get the count of recorded values at a specific value + * (to within the histogram resolution at the value level). + * + * @param h "This" pointer + * @param value The value for which to provide the recorded count + * @return The total count of values recorded in the histogram within the value range that is + * {@literal >=} lowestEquivalentValue(<i>value</i>) and {@literal <=} highestEquivalentValue(<i>value</i>) + */ +int64_t hdr_count_at_value(const struct hdr_histogram* h, int64_t value); + +int64_t hdr_count_at_index(const struct hdr_histogram* h, int32_t index); + +int64_t hdr_value_at_index(const struct hdr_histogram* h, int32_t index); + +struct hdr_iter_percentiles +{ + bool seen_last_value; + int32_t ticks_per_half_distance; + double percentile_to_iterate_to; + double percentile; +}; + +struct hdr_iter_recorded +{ + int64_t count_added_in_this_iteration_step; +}; + +struct hdr_iter_linear +{ + int64_t value_units_per_bucket; + int64_t count_added_in_this_iteration_step; + int64_t next_value_reporting_level; + int64_t next_value_reporting_level_lowest_equivalent; +}; + +struct hdr_iter_log +{ + double log_base; + int64_t count_added_in_this_iteration_step; + int64_t next_value_reporting_level; + int64_t next_value_reporting_level_lowest_equivalent; +}; + +/** + * The basic iterator. This is a generic structure + * that supports all of the types of iteration. Use + * the appropriate initialiser to get the desired + * iteration. + * + * @ + */ +struct hdr_iter +{ + const struct hdr_histogram* h; + /** raw index into the counts array */ + int32_t counts_index; + /** snapshot of the length at the time the iterator is created */ + int32_t total_count; + /** value directly from array for the current counts_index */ + int64_t count; + /** sum of all of the counts up to and including the count at this index */ + int64_t cumulative_count; + /** The current value based on counts_index */ + int64_t value; + int64_t highest_equivalent_value; + int64_t lowest_equivalent_value; + int64_t median_equivalent_value; + int64_t value_iterated_from; + int64_t value_iterated_to; + + union + { + struct hdr_iter_percentiles percentiles; + struct hdr_iter_recorded recorded; + struct hdr_iter_linear linear; + struct hdr_iter_log log; + } specifics; + + bool (* _next_fp)(struct hdr_iter* iter); + +}; + +/** + * Initalises the basic iterator. + * + * @param itr 'This' pointer + * @param h The histogram to iterate over + */ +void hdr_iter_init(struct hdr_iter* iter, const struct hdr_histogram* h); + +/** + * Initialise the iterator for use with percentiles. + */ +void hdr_iter_percentile_init(struct hdr_iter* iter, const struct hdr_histogram* h, int32_t ticks_per_half_distance); + +/** + * Initialise the iterator for use with recorded values. + */ +void hdr_iter_recorded_init(struct hdr_iter* iter, const struct hdr_histogram* h); + +/** + * Initialise the iterator for use with linear values. + */ +void hdr_iter_linear_init( + struct hdr_iter* iter, + const struct hdr_histogram* h, + int64_t value_units_per_bucket); + +/** + * Initialise the iterator for use with logarithmic values + */ +void hdr_iter_log_init( + struct hdr_iter* iter, + const struct hdr_histogram* h, + int64_t value_units_first_bucket, + double log_base); + +/** + * Iterate to the next value for the iterator. If there are no more values + * available return faluse. + * + * @param itr 'This' pointer + * @return 'false' if there are no values remaining for this iterator. + */ +bool hdr_iter_next(struct hdr_iter* iter); + +typedef enum +{ + CLASSIC, + CSV +} format_type; + +/** + * Print out a percentile based histogram to the supplied stream. Note that + * this call will not flush the FILE, this is left up to the user. + * + * @param h 'This' pointer + * @param stream The FILE to write the output to + * @param ticks_per_half_distance The number of iteration steps per half-distance to 100% + * @param value_scale Scale the output values by this amount + * @param format_type Format to use, e.g. CSV. + * @return 0 on success, error code on failure. EIO if an error occurs writing + * the output. + */ +int hdr_percentiles_print( + struct hdr_histogram* h, FILE* stream, int32_t ticks_per_half_distance, + double value_scale, format_type format); + +/** +* Internal allocation methods, used by hdr_dbl_histogram. +*/ +struct hdr_histogram_bucket_config +{ + int64_t lowest_trackable_value; + int64_t highest_trackable_value; + int64_t unit_magnitude; + int64_t significant_figures; + int32_t sub_bucket_half_count_magnitude; + int32_t sub_bucket_half_count; + int64_t sub_bucket_mask; + int32_t sub_bucket_count; + int32_t bucket_count; + int32_t counts_len; +}; + +int hdr_calculate_bucket_config( + int64_t lowest_trackable_value, + int64_t highest_trackable_value, + int significant_figures, + struct hdr_histogram_bucket_config* cfg); + +void hdr_init_preallocated(struct hdr_histogram* h, struct hdr_histogram_bucket_config* cfg); + +int64_t hdr_size_of_equivalent_value_range(const struct hdr_histogram* h, int64_t value); + +int64_t hdr_next_non_equivalent_value(const struct hdr_histogram* h, int64_t value); + +int64_t hdr_median_equivalent_value(const struct hdr_histogram* h, int64_t value); + +/** + * Used to reset counters after importing data manuallying into the histogram, used by the logging code + * and other custom serialisation tools. + */ +void hdr_reset_internal_counters(struct hdr_histogram* h); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/libs/hdr_histogram/src/hdr_interval_recorder.c b/contrib/libs/hdr_histogram/src/hdr_interval_recorder.c new file mode 100644 index 0000000000..2b0ba87cae --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_interval_recorder.c @@ -0,0 +1,53 @@ +/** + * hdr_interval_recorder.h + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#include "hdr_atomic.h" +#include "hdr_interval_recorder.h" + +int hdr_interval_recorder_init(struct hdr_interval_recorder* r) +{ + return hdr_writer_reader_phaser_init(&r->phaser); +} + +void hdr_interval_recorder_destroy(struct hdr_interval_recorder* r) +{ + hdr_writer_reader_phaser_destory(&r->phaser); +} + +void hdr_interval_recorder_update( + struct hdr_interval_recorder* r, + void(*update_action)(void*, void*), + void* arg) +{ + int64_t val = hdr_phaser_writer_enter(&r->phaser); + + void* active = hdr_atomic_load_pointer(&r->active); + + update_action(active, arg); + + hdr_phaser_writer_exit(&r->phaser, val); +} + +void* hdr_interval_recorder_sample(struct hdr_interval_recorder* r) +{ + void* temp; + + hdr_phaser_reader_lock(&r->phaser); + + temp = r->inactive; + + // volatile read + r->inactive = hdr_atomic_load_pointer(&r->active); + + // volatile write + hdr_atomic_store_pointer(&r->active, temp); + + hdr_phaser_flip_phase(&r->phaser, 0); + + hdr_phaser_reader_unlock(&r->phaser); + + return r->inactive; +} diff --git a/contrib/libs/hdr_histogram/src/hdr_interval_recorder.h b/contrib/libs/hdr_histogram/src/hdr_interval_recorder.h new file mode 100644 index 0000000000..e363c23c66 --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_interval_recorder.h @@ -0,0 +1,40 @@ +/** + * hdr_interval_recorder.h + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#ifndef HDR_INTERVAL_RECORDER_H +#define HDR_INTERVAL_RECORDER_H 1 + +#include "hdr_writer_reader_phaser.h" + +HDR_ALIGN_PREFIX(8) +struct hdr_interval_recorder +{ + void* active; + void* inactive; + struct hdr_writer_reader_phaser phaser; +} +HDR_ALIGN_SUFFIX(8); + +#ifdef __cplusplus +extern "C" { +#endif + +int hdr_interval_recorder_init(struct hdr_interval_recorder* r); + +void hdr_interval_recorder_destroy(struct hdr_interval_recorder* r); + +void hdr_interval_recorder_update( + struct hdr_interval_recorder* r, + void(*update_action)(void*, void*), + void* arg); + +void* hdr_interval_recorder_sample(struct hdr_interval_recorder* r); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/libs/hdr_histogram/src/hdr_tests.h b/contrib/libs/hdr_histogram/src/hdr_tests.h new file mode 100644 index 0000000000..c016d3a6de --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_tests.h @@ -0,0 +1,22 @@ +#ifndef HDR_TESTS_H +#define HDR_TESTS_H + +/* These are functions used in tests and are not intended for normal usage. */ + +#include "hdr_histogram.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t counts_index_for(const struct hdr_histogram* h, int64_t value); +int hdr_encode_compressed(struct hdr_histogram* h, uint8_t** compressed_histogram, size_t* compressed_len); +int hdr_decode_compressed(uint8_t* buffer, size_t length, struct hdr_histogram** histogram); +void hdr_base64_decode_block(const char* input, uint8_t* output); +void hdr_base64_encode_block(const uint8_t* input, char* output); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/libs/hdr_histogram/src/hdr_thread.c b/contrib/libs/hdr_histogram/src/hdr_thread.c new file mode 100644 index 0000000000..721780edb9 --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_thread.c @@ -0,0 +1,102 @@ +/** +* hdr_thread.c +* Written by Philip Orwig and released to the public domain, +* as explained at http://creativecommons.org/publicdomain/zero/1.0/ +*/ + +#include <stdlib.h> +#include "hdr_thread.h" + +struct hdr_mutex* hdr_mutex_alloc(void) +{ + return malloc(sizeof(hdr_mutex)); +} + +void hdr_mutex_free(struct hdr_mutex* mutex) +{ + free(mutex); +} + +#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) + +#if !defined(WIN32_LEAN_AND_MEAN) +#define WIN32_LEAN_AND_MEAN +#endif + +#include <windows.h> +#include <WinSock2.h> + +int hdr_mutex_init(struct hdr_mutex* mutex) +{ + InitializeCriticalSection((CRITICAL_SECTION*)(mutex->_critical_section)); + return 0; +} + +void hdr_mutex_destroy(struct hdr_mutex* mutex) +{ + DeleteCriticalSection((CRITICAL_SECTION*)(mutex->_critical_section)); +} + +void hdr_mutex_lock(struct hdr_mutex* mutex) +{ + EnterCriticalSection((CRITICAL_SECTION*)(mutex->_critical_section)); +} + +void hdr_mutex_unlock(struct hdr_mutex* mutex) +{ + LeaveCriticalSection((CRITICAL_SECTION*)(mutex->_critical_section)); +} + +void hdr_yield() +{ + Sleep(0); +} + +int hdr_usleep(unsigned int useconds) +{ + struct timeval tv; + + tv.tv_sec = (long)useconds / 1000000; + tv.tv_usec = useconds % 1000000; + select(0, NULL, NULL, NULL, &tv); + + return 0; +} + + +#else +#include <pthread.h> +#include <unistd.h> + +int hdr_mutex_init(struct hdr_mutex* mutex) +{ + return pthread_mutex_init(&mutex->_mutex, NULL); +} + +void hdr_mutex_destroy(struct hdr_mutex* mutex) +{ + pthread_mutex_destroy(&mutex->_mutex); +} + +void hdr_mutex_lock(struct hdr_mutex* mutex) +{ + pthread_mutex_lock(&mutex->_mutex); +} + +void hdr_mutex_unlock(struct hdr_mutex* mutex) +{ + pthread_mutex_unlock(&mutex->_mutex); +} + +void hdr_yield() +{ + sched_yield(); +} + +int hdr_usleep(unsigned int useconds) +{ + return usleep(useconds); +} + + +#endif diff --git a/contrib/libs/hdr_histogram/src/hdr_thread.h b/contrib/libs/hdr_histogram/src/hdr_thread.h new file mode 100644 index 0000000000..a7d77c02b7 --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_thread.h @@ -0,0 +1,55 @@ +/** + * hdr_thread.h + * Written by Philip Orwig and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#ifndef HDR_THREAD_H__ +#define HDR_THREAD_H__ + +#include <stdint.h> + +#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) + + +#define HDR_ALIGN_PREFIX(alignment) __declspec( align(alignment) ) +#define HDR_ALIGN_SUFFIX(alignment) + +typedef struct hdr_mutex +{ + uint8_t _critical_section[40]; +} hdr_mutex; + +#else + +#include <pthread.h> + +#define HDR_ALIGN_PREFIX(alignment) +#define HDR_ALIGN_SUFFIX(alignment) __attribute__((aligned(alignment))) + +typedef struct hdr_mutex +{ + pthread_mutex_t _mutex; +} hdr_mutex; +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +struct hdr_mutex* hdr_mutex_alloc(void); +void hdr_mutex_free(struct hdr_mutex*); + +int hdr_mutex_init(struct hdr_mutex* mutex); +void hdr_mutex_destroy(struct hdr_mutex* mutex); + +void hdr_mutex_lock(struct hdr_mutex* mutex); +void hdr_mutex_unlock(struct hdr_mutex* mutex); + +void hdr_yield(); +int hdr_usleep(unsigned int useconds); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/contrib/libs/hdr_histogram/src/hdr_time.c b/contrib/libs/hdr_histogram/src/hdr_time.c new file mode 100644 index 0000000000..44742076ad --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_time.c @@ -0,0 +1,70 @@ +/** +* hdr_time.h +* Written by Michael Barker and Philip Orwig and released to the public domain, +* as explained at http://creativecommons.org/publicdomain/zero/1.0/ +*/ + +#include "hdr_time.h" + +#if defined(_WIN32) || defined(_WIN64) + +#if !defined(WIN32_LEAN_AND_MEAN) +#define WIN32_LEAN_AND_MEAN +#endif + +#include <windows.h> + +static int s_clockPeriodSet = 0; +static double s_clockPeriod = 1.0; + +void hdr_gettime(hdr_timespec* t) +{ + LARGE_INTEGER num; + /* if this is distasteful, we can add in an hdr_time_init() */ + if (!s_clockPeriodSet) + { + QueryPerformanceFrequency(&num); + s_clockPeriod = 1.0 / (double) num.QuadPart; + s_clockPeriodSet = 1; + } + + QueryPerformanceCounter(&num); + double seconds = num.QuadPart * s_clockPeriod; + double integral; + double remainder = modf(seconds, &integral); + + t->tv_sec = (long) integral; + t->tv_nsec = (long) (remainder * 1000000000); +} + +#elif defined(__APPLE__) + +#include <mach/clock.h> +#include <mach/mach.h> + + +void hdr_gettime(hdr_timespec* ts) +{ + clock_serv_t cclock; + mach_timespec_t mts; + host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); + clock_get_time(cclock, &mts); + mach_port_deallocate(mach_task_self(), cclock); + ts->tv_sec = mts.tv_sec; + ts->tv_nsec = mts.tv_nsec; +} + +#elif defined(__linux__) || defined(__CYGWIN__) + + +void hdr_gettime(hdr_timespec* t) +{ + clock_gettime(CLOCK_MONOTONIC, (struct timespec*)t); +} + +#else + +#warning "Platform not supported\n" + +#endif + diff --git a/contrib/libs/hdr_histogram/src/hdr_time.h b/contrib/libs/hdr_histogram/src/hdr_time.h new file mode 100644 index 0000000000..8b1a71ec8a --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_time.h @@ -0,0 +1,42 @@ +/** + * hdr_time.h + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#ifndef HDR_TIME_H__ +#define HDR_TIME_H__ + +#include <math.h> +#include <time.h> + +#if defined(_WIN32) || defined(_WIN64) || defined(__CYGWIN__) + +typedef struct hdr_timespec +{ + long tv_sec; + long tv_nsec; +} hdr_timespec; + +#else + +typedef struct timespec hdr_timespec; + +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(_MSC_VER) +void hdr_gettime(hdr_timespec* t); +#else +void hdr_gettime(hdr_timespec* t); +#endif + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/contrib/libs/hdr_histogram/src/hdr_writer_reader_phaser.c b/contrib/libs/hdr_histogram/src/hdr_writer_reader_phaser.c new file mode 100644 index 0000000000..21d3adeeaa --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_writer_reader_phaser.c @@ -0,0 +1,135 @@ +/** + * hdr_writer_reader_phaser.h + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#include <stdint.h> +#include <stdbool.h> +#include <errno.h> + +#include "hdr_atomic.h" +#include "hdr_thread.h" + +#include "hdr_writer_reader_phaser.h" + +static int64_t _hdr_phaser_get_epoch(int64_t* field) +{ + return hdr_atomic_load_64(field); +} + +static void _hdr_phaser_set_epoch(int64_t* field, int64_t val) +{ + hdr_atomic_store_64(field, val); +} + +static int64_t _hdr_phaser_reset_epoch(int64_t* field, int64_t initial_value) +{ + return hdr_atomic_exchange_64(field, initial_value); +} + +int hdr_writer_reader_phaser_init(struct hdr_writer_reader_phaser* p) +{ + if (NULL == p) + { + return EINVAL; + } + + p->start_epoch = 0; + p->even_end_epoch = 0; + p->odd_end_epoch = INT64_MIN; + p->reader_mutex = hdr_mutex_alloc(); + + if (!p->reader_mutex) + { + return ENOMEM; + } + + int rc = hdr_mutex_init(p->reader_mutex); + if (0 != rc) + { + return rc; + } + + // TODO: Should I fence here. + + return 0; +} + +void hdr_writer_reader_phaser_destory(struct hdr_writer_reader_phaser* p) +{ + hdr_mutex_destroy(p->reader_mutex); +} + +int64_t hdr_phaser_writer_enter(struct hdr_writer_reader_phaser* p) +{ + return hdr_atomic_add_fetch_64(&p->start_epoch, 1); +} + +void hdr_phaser_writer_exit( + struct hdr_writer_reader_phaser* p, int64_t critical_value_at_enter) +{ + int64_t* end_epoch = + (critical_value_at_enter < 0) ? &p->odd_end_epoch : &p->even_end_epoch; + hdr_atomic_add_fetch_64(end_epoch, 1); +} + +void hdr_phaser_reader_lock(struct hdr_writer_reader_phaser* p) +{ + hdr_mutex_lock(p->reader_mutex); +} + +void hdr_phaser_reader_unlock(struct hdr_writer_reader_phaser* p) +{ + hdr_mutex_unlock(p->reader_mutex); +} + +void hdr_phaser_flip_phase( + struct hdr_writer_reader_phaser* p, int64_t sleep_time_ns) +{ + // TODO: is_held_by_current_thread + unsigned int sleep_time_us = sleep_time_ns < 1000000000 ? (unsigned int) (sleep_time_ns / 1000) : 1000000; + + int64_t start_epoch = _hdr_phaser_get_epoch(&p->start_epoch); + + bool next_phase_is_even = (start_epoch < 0); + + // Clear currently used phase end epoch. + int64_t initial_start_value; + if (next_phase_is_even) + { + initial_start_value = 0; + _hdr_phaser_set_epoch(&p->even_end_epoch, initial_start_value); + } + else + { + initial_start_value = INT64_MIN; + _hdr_phaser_set_epoch(&p->odd_end_epoch, initial_start_value); + } + + // Reset start value, indicating new phase. + int64_t start_value_at_flip = + _hdr_phaser_reset_epoch(&p->start_epoch, initial_start_value); + + bool caught_up = false; + do + { + int64_t* end_epoch = + next_phase_is_even ? &p->odd_end_epoch : &p->even_end_epoch; + + caught_up = _hdr_phaser_get_epoch(end_epoch) == start_value_at_flip; + + if (!caught_up) + { + if (sleep_time_us <= 0) + { + hdr_yield(); + } + else + { + hdr_usleep(sleep_time_us); + } + } + } + while (!caught_up); +} diff --git a/contrib/libs/hdr_histogram/src/hdr_writer_reader_phaser.h b/contrib/libs/hdr_histogram/src/hdr_writer_reader_phaser.h new file mode 100644 index 0000000000..f14a7db6e7 --- /dev/null +++ b/contrib/libs/hdr_histogram/src/hdr_writer_reader_phaser.h @@ -0,0 +1,51 @@ +/** + * hdr_writer_reader_phaser.h + * Written by Michael Barker and released to the public domain, + * as explained at http://creativecommons.org/publicdomain/zero/1.0/ + */ + +#ifndef HDR_WRITER_READER_PHASER_H +#define HDR_WRITER_READER_PHASER_H 1 + +#include <stdlib.h> +#include <stdbool.h> +#include <stdlib.h> +#include <errno.h> + +#include "hdr_thread.h" + +HDR_ALIGN_PREFIX(8) +struct hdr_writer_reader_phaser +{ + int64_t start_epoch; + int64_t even_end_epoch; + int64_t odd_end_epoch; + hdr_mutex* reader_mutex; +} +HDR_ALIGN_SUFFIX(8); + +#ifdef __cplusplus +extern "C" { +#endif + + int hdr_writer_reader_phaser_init(struct hdr_writer_reader_phaser* p); + + void hdr_writer_reader_phaser_destory(struct hdr_writer_reader_phaser* p); + + int64_t hdr_phaser_writer_enter(struct hdr_writer_reader_phaser* p); + + void hdr_phaser_writer_exit( + struct hdr_writer_reader_phaser* p, int64_t critical_value_at_enter); + + void hdr_phaser_reader_lock(struct hdr_writer_reader_phaser* p); + + void hdr_phaser_reader_unlock(struct hdr_writer_reader_phaser* p); + + void hdr_phaser_flip_phase( + struct hdr_writer_reader_phaser* p, int64_t sleep_time_ns); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/contrib/libs/hdr_histogram/ya.make b/contrib/libs/hdr_histogram/ya.make new file mode 100644 index 0000000000..b20d977b57 --- /dev/null +++ b/contrib/libs/hdr_histogram/ya.make @@ -0,0 +1,39 @@ +LIBRARY() + +LICENSE( + BSD-2-Clause AND + CC0-1.0 +) + +LICENSE_TEXTS(.yandex_meta/licenses.list.txt) + +VERSION(0.9.5) + +OWNER( + jamel + g:contrib + g:cpp-contrib +) + +NO_UTIL() + +NO_COMPILER_WARNINGS() + +ADDINCL( + contrib/libs/hdr_histogram/src +) + +SRCS( + src/hdr_encoding.c + src/hdr_interval_recorder.c + src/hdr_histogram.c + src/hdr_writer_reader_phaser.c + src/hdr_time.c + src/hdr_thread.c +) + +PEERDIR( + contrib/libs/zlib +) + +END() diff --git a/library/cpp/histogram/hdr/histogram.cpp b/library/cpp/histogram/hdr/histogram.cpp new file mode 100644 index 0000000000..a213d5d8fd --- /dev/null +++ b/library/cpp/histogram/hdr/histogram.cpp @@ -0,0 +1,155 @@ +#include "histogram.h" + +#include <util/generic/cast.h> +#include <util/generic/yexception.h> + +#include <contrib/libs/hdr_histogram/src/hdr_histogram.h> + +namespace NHdr { + namespace { + struct hdr_histogram* CreateHistogram( + i64 lowestDiscernibleValue, i64 highestTrackableValue, + i32 numberOfSignificantValueDigits, IAllocator* allocator) { + struct hdr_histogram_bucket_config cfg; + + int r = hdr_calculate_bucket_config( + lowestDiscernibleValue, highestTrackableValue, + numberOfSignificantValueDigits, &cfg); + if (r) { + ythrow yexception() << "illegal arguments values"; + } + + size_t histogramSize = sizeof(struct hdr_histogram) + + cfg.counts_len * sizeof(i64); + + IAllocator::TBlock mem = allocator->Allocate(histogramSize); + struct hdr_histogram* histogram = + reinterpret_cast<struct hdr_histogram*>(mem.Data); + + // memset will ensure that all of the function pointers are null + memset(histogram, 0, histogramSize); + + hdr_init_preallocated(histogram, &cfg); + return histogram; + } + + } + + THistogram::THistogram(i64 lowestDiscernibleValue, i64 highestTrackableValue, + i32 numberOfSignificantValueDigits, IAllocator* allocator) + : Data_(CreateHistogram( + lowestDiscernibleValue, highestTrackableValue, + numberOfSignificantValueDigits, allocator)) + , Allocator_(allocator) + { + } + + THistogram::~THistogram() { + if (Data_) { + size_t size = GetMemorySize(); + Allocator_->Release({Data_.Release(), size}); + } + } + + // Histogram structure querying support ----------------------------------- + + i64 THistogram::GetLowestDiscernibleValue() const { + return Data_->lowest_trackable_value; + } + + i64 THistogram::GetHighestTrackableValue() const { + return Data_->highest_trackable_value; + } + + i32 THistogram::GetNumberOfSignificantValueDigits() const { + return Data_->significant_figures; + } + + size_t THistogram::GetMemorySize() const { + return hdr_get_memory_size(Data_.Get()); + } + + i32 THistogram::GetCountsLen() const { + return Data_->counts_len; + } + + i64 THistogram::GetTotalCount() const { + return Data_->total_count; + } + + // Value recording support ------------------------------------------------ + + bool THistogram::RecordValue(i64 value) { + return hdr_record_value(Data_.Get(), value); + } + + bool THistogram::RecordValues(i64 value, i64 count) { + return hdr_record_values(Data_.Get(), value, count); + } + + bool THistogram::RecordValueWithExpectedInterval(i64 value, i64 expectedInterval) { + return hdr_record_corrected_value(Data_.Get(), value, expectedInterval); + } + + bool THistogram::RecordValuesWithExpectedInterval( + i64 value, i64 count, i64 expectedInterval) { + return hdr_record_corrected_values( + Data_.Get(), value, count, expectedInterval); + } + + i64 THistogram::Add(const THistogram& rhs) { + return hdr_add(Data_.Get(), rhs.Data_.Get()); + } + + i64 THistogram::AddWithExpectedInterval(const THistogram& rhs, i64 expectedInterval) { + return hdr_add_while_correcting_for_coordinated_omission( + Data_.Get(), rhs.Data_.Get(), expectedInterval); + } + + // Histogram Data access support ------------------------------------------ + + i64 THistogram::GetMin() const { + return hdr_min(Data_.Get()); + } + + i64 THistogram::GetMax() const { + return hdr_max(Data_.Get()); + } + + double THistogram::GetMean() const { + return hdr_mean(Data_.Get()); + } + + double THistogram::GetStdDeviation() const { + return hdr_stddev(Data_.Get()); + } + + i64 THistogram::GetValueAtPercentile(double percentile) const { + return hdr_value_at_percentile(Data_.Get(), percentile); + } + + i64 THistogram::GetCountAtValue(i64 value) const { + return hdr_count_at_value(Data_.Get(), value); + } + + bool THistogram::ValuesAreEqual(i64 v1, i64 v2) const { + return hdr_values_are_equivalent(Data_.Get(), v1, v2); + } + + i64 THistogram::GetLowestEquivalentValue(i64 value) const { + return hdr_lowest_equivalent_value(Data_.Get(), value); + } + + i64 THistogram::GetHighestEquivalentValue(i64 value) const { + return hdr_next_non_equivalent_value(Data_.Get(), value) - 1; + } + + i64 THistogram::GetMedianEquivalentValue(i64 value) const { + return hdr_median_equivalent_value(Data_.Get(), value); + } + + void THistogram::Reset() { + hdr_reset(Data_.Get()); + } + +} diff --git a/library/cpp/histogram/hdr/histogram.h b/library/cpp/histogram/hdr/histogram.h new file mode 100644 index 0000000000..5f1cebbd9f --- /dev/null +++ b/library/cpp/histogram/hdr/histogram.h @@ -0,0 +1,303 @@ +#pragma once + +#include <util/generic/ptr.h> +#include <util/generic/noncopyable.h> +#include <util/memory/alloc.h> + +struct hdr_histogram; + +namespace NHdr { + /** + * A High Dynamic Range (HDR) Histogram + * + * THdrHistogram supports the recording and analyzing sampled data value counts + * across a configurable integer value range with configurable value precision + * within the range. Value precision is expressed as the number of significant + * digits in the value recording, and provides control over value quantization + * behavior across the value range and the subsequent value resolution at any + * given level. + */ + class THistogram: public TMoveOnly { + public: + /** + * Construct a histogram given the Highest value to be tracked and a number + * of significant decimal digits. The histogram will be constructed to + * implicitly track (distinguish from 0) values as low as 1. Default + * allocator will be used to allocate underlying memory. + * + * @param highestTrackableValue The highest value to be tracked by the + * histogram. Must be a positive integer that is literal >= 2. + * + * @param numberOfSignificantValueDigits Specifies the precision to use. + * This is the number of significant decimal digits to which the + * histogram will maintain value resolution and separation. Must be + * a non-negative integer between 0 and 5. + */ + THistogram(i64 highestTrackableValue, i32 numberOfSignificantValueDigits) + : THistogram(1, highestTrackableValue, numberOfSignificantValueDigits) + { + } + + /** + * Construct a histogram given the Lowest and Highest values to be tracked + * and a number of significant decimal digits. Providing a + * lowestDiscernibleValue is useful in situations where the units used for + * the histogram's values are much smaller that the minimal accuracy + * required. E.g. when tracking time values stated in nanosecond units, + * where the minimal accuracy required is a microsecond, the proper value + * for lowestDiscernibleValue would be 1000. + * + * @param lowestDiscernibleValue The lowest value that can be discerned + * (distinguished from 0) by the histogram. Must be a positive + * integer that is >= 1. May be internally rounded down to nearest + * power of 2. + * + * @param highestTrackableValue The highest value to be tracked by the + * histogram. Must be a positive integer that is + * >= (2 * lowestDiscernibleValue). + * + * @param numberOfSignificantValueDigits Specifies the precision to use. + * This is the number of significant decimal digits to which the + * histogram will maintain value resolution and separation. Must be + * a non-negative integer between 0 and 5. + * + * @param allocator Specifies allocator which will be used to allocate + * memory for histogram. + */ + THistogram(i64 lowestDiscernibleValue, i64 highestTrackableValue, + i32 numberOfSignificantValueDigits, + IAllocator* allocator = TDefaultAllocator::Instance()); + + ~THistogram(); + + // Histogram structure querying support ----------------------------------- + + /** + * @return The configured lowestDiscernibleValue + */ + i64 GetLowestDiscernibleValue() const; + + /** + * @return The configured highestTrackableValue + */ + i64 GetHighestTrackableValue() const; + + /** + * @return The configured numberOfSignificantValueDigits + */ + i32 GetNumberOfSignificantValueDigits() const; + + /** + * @return The size of allocated memory for histogram + */ + size_t GetMemorySize() const; + + /** + * @return The number of created counters + */ + i32 GetCountsLen() const; + + /** + * @return The total count of all recorded values in the histogram + */ + i64 GetTotalCount() const; + + // Value recording support ------------------------------------------------ + + /** + * Records a value in the histogram, will round this value of to a + * precision at or better than the NumberOfSignificantValueDigits specified + * at construction time. + * + * @param value Value to add to the histogram + * @return false if the value is larger than the HighestTrackableValue + * and can't be recorded, true otherwise. + */ + bool RecordValue(i64 value); + + /** + * Records count values in the histogram, will round this value of to a + * precision at or better than the NumberOfSignificantValueDigits specified + * at construction time. + * + * @param value Value to add to the histogram + * @param count Number of values to add to the histogram + * @return false if the value is larger than the HighestTrackableValue + * and can't be recorded, true otherwise. + */ + bool RecordValues(i64 value, i64 count); + + /** + * Records a value in the histogram and backfill based on an expected + * interval. Value will be rounded this to a precision at or better + * than the NumberOfSignificantValueDigits specified at contruction time. + * This is specifically used for recording latency. If the value is larger + * than the expectedInterval then the latency recording system has + * experienced co-ordinated omission. This method fills in the values that + * would have occured had the client providing the load not been blocked. + * + * @param value Value to add to the histogram + * @param expectedInterval The delay between recording values + * @return false if the value is larger than the HighestTrackableValue + * and can't be recorded, true otherwise. + */ + bool RecordValueWithExpectedInterval(i64 value, i64 expectedInterval); + + /** + * Record a value in the histogram count times. Applies the same correcting + * logic as {@link THistogram::RecordValueWithExpectedInterval}. + * + * @param value Value to add to the histogram + * @param count Number of values to add to the histogram + * @param expectedInterval The delay between recording values. + * @return false if the value is larger than the HighestTrackableValue + * and can't be recorded, true otherwise. + */ + bool RecordValuesWithExpectedInterval( + i64 value, i64 count, i64 expectedInterval); + + /** + * Adds all of the values from rhs to this histogram. Will return the + * number of values that are dropped when copying. Values will be dropped + * if they around outside of [LowestDiscernibleValue, GetHighestTrackableValue]. + * + * @param rhs Histogram to copy values from. + * @return The number of values dropped when copying. + */ + i64 Add(const THistogram& rhs); + + /** + * Adds all of the values from rhs to this histogram. Will return the + * number of values that are dropped when copying. Values will be dropped + * if they around outside of [LowestDiscernibleValue, GetHighestTrackableValue]. + * Applies the same correcting logic as + * {@link THistogram::RecordValueWithExpectedInterval}. + * + * @param rhs Histogram to copy values from. + * @return The number of values dropped when copying. + */ + i64 AddWithExpectedInterval(const THistogram& rhs, i64 expectedInterval); + + // Histogram Data access support ------------------------------------------ + + /** + * Get the lowest recorded value level in the histogram. If the histogram + * has no recorded values, the value returned is undefined. + * + * @return the Min value recorded in the histogram + */ + i64 GetMin() const; + + /** + * Get the highest recorded value level in the histogram. If the histogram + * has no recorded values, the value returned is undefined. + * + * @return the Max value recorded in the histogram + */ + i64 GetMax() const; + + /** + * Get the computed mean value of all recorded values in the histogram + * + * @return the mean value (in value units) of the histogram data + */ + double GetMean() const; + + /** + * Get the computed standard deviation of all recorded values in the histogram + * + * @return the standard deviation (in value units) of the histogram data + */ + double GetStdDeviation() const; + + /** + * Get the value at a given percentile. + * Note that two values are "equivalent" in this statement if + * {@link THistogram::ValuesAreEquivalent} would return true. + * + * @param percentile The percentile for which to return the associated + * value + * @return The value that the given percentage of the overall recorded + * value entries in the histogram are either smaller than or + * equivalent to. When the percentile is 0.0, returns the value + * that all value entries in the histogram are either larger than + * or equivalent to. + */ + i64 GetValueAtPercentile(double percentile) const; + + /** + * Get the count of recorded values at a specific value (to within the + * histogram resolution at the value level). + * + * @param value The value for which to provide the recorded count + * @return The total count of values recorded in the histogram within the + * value range that is >= GetLowestEquivalentValue(value) and + * <= GetHighestEquivalentValue(value) + */ + i64 GetCountAtValue(i64 value) const; + + /** + * Determine if two values are equivalent with the histogram's resolution. + * Where "equivalent" means that value samples recorded for any two + * equivalent values are counted in a common total count. + * + * @param v1 first value to compare + * @param v2 second value to compare + * @return True if values are equivalent with the histogram's resolution. + */ + bool ValuesAreEqual(i64 v1, i64 v2) const; + + /** + * Get the lowest value that is equivalent to the given value within the + * histogram's resolution. Where "equivalent" means that value samples + * recorded for any two equivalent values are counted in a common total + * count. + * + * @param value The given value + * @return The lowest value that is equivalent to the given value within + * the histogram's resolution. + */ + i64 GetLowestEquivalentValue(i64 value) const; + + /** + * Get the highest value that is equivalent to the given value within the + * histogram's resolution. Where "equivalent" means that value samples + * recorded for any two equivalent values are counted in a common total + * count. + * + * @param value The given value + * @return The highest value that is equivalent to the given value within + * the histogram's resolution. + */ + i64 GetHighestEquivalentValue(i64 value) const; + + /** + * Get a value that lies in the middle (rounded up) of the range of values + * equivalent the given value. Where "equivalent" means that value samples + * recorded for any two equivalent values are counted in a common total + * count. + * + * @param value The given value + * @return The value lies in the middle (rounded up) of the range of values + * equivalent the given value. + */ + i64 GetMedianEquivalentValue(i64 value) const; + + // misc functions --------------------------------------------------------- + + /** + * Reset a histogram to zero - empty out a histogram and re-initialise it. + * If you want to re-use an existing histogram, but reset everything back + * to zero, this is the routine to use. + */ + void Reset(); + + const hdr_histogram* GetHdrHistogramImpl() const { + return Data_.Get(); + } + + private: + THolder<hdr_histogram> Data_; + IAllocator* Allocator_; + }; +} diff --git a/library/cpp/histogram/hdr/histogram_iter.cpp b/library/cpp/histogram/hdr/histogram_iter.cpp new file mode 100644 index 0000000000..d251fd5dd9 --- /dev/null +++ b/library/cpp/histogram/hdr/histogram_iter.cpp @@ -0,0 +1,146 @@ +#include "histogram_iter.h" + +#include <contrib/libs/hdr_histogram/src/hdr_histogram.h> + +namespace NHdr { + // TBaseHistogramIterator ----------------------------------------------------- + TBaseHistogramIterator::TBaseHistogramIterator() + : Iter_(new hdr_iter) + { + } + + TBaseHistogramIterator::~TBaseHistogramIterator() { + } + + bool TBaseHistogramIterator::Next() { + return hdr_iter_next(Iter_.Get()); + } + + i32 TBaseHistogramIterator::GetCountsIndex() const { + return Iter_->counts_index; + } + + i32 TBaseHistogramIterator::GetTotalCount() const { + return Iter_->total_count; + } + + i64 TBaseHistogramIterator::GetCount() const { + return Iter_->count; + } + + i64 TBaseHistogramIterator::GetCumulativeCount() const { + return Iter_->cumulative_count; + } + + i64 TBaseHistogramIterator::GetValue() const { + return Iter_->value; + } + + i64 TBaseHistogramIterator::GetHighestEquivalentValue() const { + return Iter_->highest_equivalent_value; + } + + i64 TBaseHistogramIterator::GetLowestEquivalentValue() const { + return Iter_->lowest_equivalent_value; + } + + i64 TBaseHistogramIterator::GetMedianEquivalentValue() const { + return Iter_->median_equivalent_value; + } + + i64 TBaseHistogramIterator::GetValueIteratedFrom() const { + return Iter_->value_iterated_from; + } + + i64 TBaseHistogramIterator::GetValueIteratedTo() const { + return Iter_->value_iterated_to; + } + + // TAllValuesIterator --------------------------------------------------------- + + TAllValuesIterator::TAllValuesIterator(const THistogram& histogram) { + hdr_iter_init(Iter_.Get(), histogram.GetHdrHistogramImpl()); + } + + // TRecordedValuesIterator ---------------------------------------------------- + + TRecordedValuesIterator::TRecordedValuesIterator(const THistogram& histogram) { + hdr_iter_recorded_init(Iter_.Get(), histogram.GetHdrHistogramImpl()); + } + + i64 TRecordedValuesIterator::GetCountAddedInThisIterationStep() const { + return Iter_->specifics.recorded.count_added_in_this_iteration_step; + } + + // TPercentileIterator -------------------------------------------------------- + + TPercentileIterator::TPercentileIterator( + const THistogram& histogram, ui32 ticksPerHalfDistance) { + hdr_iter_percentile_init( + Iter_.Get(), histogram.GetHdrHistogramImpl(), + ticksPerHalfDistance); + } + + i32 TPercentileIterator::GetTicketsPerHalfDistance() const { + return Iter_->specifics.percentiles.ticks_per_half_distance; + } + + double TPercentileIterator::GetPercentileToIterateTo() const { + return Iter_->specifics.percentiles.percentile_to_iterate_to; + } + + double TPercentileIterator::GetPercentile() const { + return Iter_->specifics.percentiles.percentile; + } + + // TLinearIterator ------------------------------------------------------------ + + TLinearIterator::TLinearIterator( + const THistogram& histogram, i64 valueUnitsPerBucket) { + hdr_iter_linear_init( + Iter_.Get(), histogram.GetHdrHistogramImpl(), valueUnitsPerBucket); + } + + i64 TLinearIterator::GetValueUnitsPerBucket() const { + return Iter_->specifics.linear.value_units_per_bucket; + } + + i64 TLinearIterator::GetCountAddedInThisIterationStep() const { + return Iter_->specifics.linear.count_added_in_this_iteration_step; + } + + i64 TLinearIterator::GetNextValueReportingLevel() const { + return Iter_->specifics.linear.next_value_reporting_level; + } + + i64 TLinearIterator::GetNextValueReportingLevelLowestEquivalent() const { + return Iter_->specifics.linear.next_value_reporting_level_lowest_equivalent; + } + + // TLogarithmicIterator ------------------------------------------------------- + + TLogarithmicIterator::TLogarithmicIterator( + const THistogram& histogram, i64 valueUnitsInFirstBucket, + double logBase) { + hdr_iter_log_init( + Iter_.Get(), histogram.GetHdrHistogramImpl(), + valueUnitsInFirstBucket, logBase); + } + + double TLogarithmicIterator::GetLogBase() const { + return Iter_->specifics.log.log_base; + } + + i64 TLogarithmicIterator::GetCountAddedInThisIterationStep() const { + return Iter_->specifics.log.count_added_in_this_iteration_step; + } + + i64 TLogarithmicIterator::GetNextValueReportingLevel() const { + return Iter_->specifics.log.next_value_reporting_level; + } + + i64 TLogarithmicIterator::GetNextValueReportingLevelLowestEquivalent() const { + return Iter_->specifics.log.next_value_reporting_level_lowest_equivalent; + } + +} diff --git a/library/cpp/histogram/hdr/histogram_iter.h b/library/cpp/histogram/hdr/histogram_iter.h new file mode 100644 index 0000000000..adfc1616e3 --- /dev/null +++ b/library/cpp/histogram/hdr/histogram_iter.h @@ -0,0 +1,231 @@ +#pragma once + +#include "histogram.h" + +struct hdr_iter; + +namespace NHdr { + /** + * Used for iterating through histogram values. + */ + class TBaseHistogramIterator { + public: + /** + * Iterate to the next value for the iterator. If there are no more values + * available return false. + * + * @return 'false' if there are no values remaining for this iterator. + */ + bool Next(); + + /** + * @return Raw index into the counts array. + */ + i32 GetCountsIndex() const; + + /** + * @return Snapshot of the length at the time the iterator is created. + */ + i32 GetTotalCount() const; + + /** + * @return Value directly from array for the current countsIndex. + */ + i64 GetCount() const; + + /** + * @return Sum of all of the counts up to and including the count at + * this index. + */ + i64 GetCumulativeCount() const; + + /** + * @return The current value based on countsIndex. + */ + i64 GetValue() const; + + /** + * @return The highest value that is equivalent to the current value + * within the histogram's resolution. + */ + i64 GetHighestEquivalentValue() const; + + /** + * @return The lowest value that is equivalent to the current value + * within the histogram's resolution. + */ + i64 GetLowestEquivalentValue() const; + + /** + * @return The value lies in the middle (rounded up) of the range of + * values equivalent the current value. + */ + i64 GetMedianEquivalentValue() const; + + /** + * @return The actual value level that was iterated from by the iterator. + */ + i64 GetValueIteratedFrom() const; + + /** + * @return The actual value level that was iterated to by the iterator. + */ + i64 GetValueIteratedTo() const; + + protected: + // must not be instantiated directly + TBaseHistogramIterator(); + ~TBaseHistogramIterator(); + + protected: + THolder<hdr_iter> Iter_; + }; + + /** + * Used for iterating through histogram values using the finest granularity + * steps supported by the underlying representation. The iteration steps + * through all possible unit value levels, regardless of whether or not there + * were recorded values for that value level, and terminates when all recorded + * histogram values are exhausted. + */ + class TAllValuesIterator: public TBaseHistogramIterator { + public: + /** + * @param histogram The histogram this iterator will operate on + */ + explicit TAllValuesIterator(const THistogram& histogram); + }; + + /** + * Used for iterating through all recorded histogram values using the finest + * granularity steps supported by the underlying representation. The iteration + * steps through all non-zero recorded value counts, and terminates when all + * recorded histogram values are exhausted. + */ + class TRecordedValuesIterator: public TBaseHistogramIterator { + public: + /** + * @param histogram The histogram this iterator will operate on + */ + explicit TRecordedValuesIterator(const THistogram& histogram); + + /** + * @return The count of recorded values in the histogram that were added + * to the totalCount as a result on this iteration step. Since + * multiple iteration steps may occur with overlapping equivalent + * value ranges, the count may be lower than the count found at + * the value (e.g. multiple linear steps or percentile levels can + * occur within a single equivalent value range). + */ + i64 GetCountAddedInThisIterationStep() const; + }; + + /** + * Used for iterating through histogram values according to percentile levels. + * The iteration is performed in steps that start at 0% and reduce their + * distance to 100% according to the <i>percentileTicksPerHalfDistance</i> + * parameter, ultimately reaching 100% when all recorded histogram + * values are exhausted. + */ + class TPercentileIterator: public TBaseHistogramIterator { + public: + /** + * @param histogram The histogram this iterator will operate on + * @param ticksPerHalfDistance The number of equal-sized iteration steps + * per half-distance to 100%. + */ + TPercentileIterator(const THistogram& histogram, ui32 ticksPerHalfDistance); + + /** + * @return The number of equal-sized iteration steps per half-distance + * to 100%. + */ + i32 GetTicketsPerHalfDistance() const; + + double GetPercentileToIterateTo() const; + + /** + * @return The percentile of recorded values in the histogram at values + * equal or smaller than valueIteratedTo. + * + */ + double GetPercentile() const; + }; + + /** + * Used for iterating through histogram values in linear steps. The iteration + * is performed in steps of <i>valueUnitsPerBucket</i> in size, terminating + * when all recorded histogram values are exhausted. Note that each iteration + * "bucket" includes values up to and including the next bucket boundary value. + */ + class TLinearIterator: public TBaseHistogramIterator { + public: + /** + * @param histogram The histogram this iterator will operate on + * @param valueUnitsPerBucket The size (in value units) of each bucket + * iteration. + */ + TLinearIterator(const THistogram& histogram, i64 valueUnitsPerBucket); + + /** + * @return The size (in value units) of each bucket iteration. + */ + i64 GetValueUnitsPerBucket() const; + + /** + * @return The count of recorded values in the histogram that were added + * to the totalCount as a result on this iteration step. Since + * multiple iteration steps may occur with overlapping equivalent + * value ranges, the count may be lower than the count found at + * the value (e.g. multiple linear steps or percentile levels can + * occur within a single equivalent value range). + */ + i64 GetCountAddedInThisIterationStep() const; + + i64 GetNextValueReportingLevel() const; + + i64 GetNextValueReportingLevelLowestEquivalent() const; + }; + + /** + * Used for iterating through histogram values in logarithmically increasing + * levels. The iteration is performed in steps that start at + * <i>valueUnitsInFirstBucket</i> and increase exponentially according to + * <i>logBase</i>, terminating when all recorded histogram values are + * exhausted. Note that each iteration "bucket" includes values up to and + * including the next bucket boundary value. + */ + class TLogarithmicIterator: public TBaseHistogramIterator { + public: + /** + * @param histogram The histogram this iterator will operate on + * @param valueUnitsInFirstBucket the size (in value units) of the first + * value bucket step + * @param logBase the multiplier by which the bucket size is expanded in + * each iteration step. + */ + TLogarithmicIterator( + const THistogram& histogram, i64 valueUnitsInFirstBucket, + double logBase); + + /** + * @return The multiplier by which the bucket size is expanded in each + * iteration step. + */ + double GetLogBase() const; + + /** + * @return The count of recorded values in the histogram that were added + * to the totalCount as a result on this iteration step. Since + * multiple iteration steps may occur with overlapping equivalent + * value ranges, the count may be lower than the count found at + * the value (e.g. multiple linear steps or percentile levels can + * occur within a single equivalent value range). + */ + i64 GetCountAddedInThisIterationStep() const; + + i64 GetNextValueReportingLevel() const; + + i64 GetNextValueReportingLevelLowestEquivalent() const; + }; +} diff --git a/library/cpp/histogram/hdr/histogram_iter_ut.cpp b/library/cpp/histogram/hdr/histogram_iter_ut.cpp new file mode 100644 index 0000000000..9c291a2547 --- /dev/null +++ b/library/cpp/histogram/hdr/histogram_iter_ut.cpp @@ -0,0 +1,210 @@ +#include "histogram_iter.h" + +#include <library/cpp/testing/unittest/registar.h> + +using namespace NHdr; + +Y_UNIT_TEST_SUITE(THistogramIterTest) { + Y_UNIT_TEST(RecordedValues) { + THistogram h(TDuration::Hours(1).MicroSeconds(), 3); + UNIT_ASSERT(h.RecordValues(1000, 1000)); + UNIT_ASSERT(h.RecordValue(1000 * 1000)); + + int index = 0; + TRecordedValuesIterator it(h); + + while (it.Next()) { + i64 countInBucket = it.GetCount(); + i64 countInStep = it.GetCountAddedInThisIterationStep(); + if (index == 0) { + UNIT_ASSERT_EQUAL(countInBucket, 1000); + UNIT_ASSERT_EQUAL(countInStep, 1000); + } else if (index == 1) { + UNIT_ASSERT_EQUAL(countInBucket, 1); + UNIT_ASSERT_EQUAL(countInStep, 1); + } else { + UNIT_FAIL("unexpected index value: " << index); + } + + index++; + } + + UNIT_ASSERT_EQUAL(index, 2); + } + + Y_UNIT_TEST(CorrectedRecordedValues) { + THistogram h(TDuration::Hours(1).MicroSeconds(), 3); + UNIT_ASSERT(h.RecordValuesWithExpectedInterval(1000, 1000, 1000)); + UNIT_ASSERT(h.RecordValueWithExpectedInterval(1000 * 1000, 1000)); + + int index = 0; + i64 totalCount = 0; + TRecordedValuesIterator it(h); + + while (it.Next()) { + i64 countInBucket = it.GetCount(); + i64 countInStep = it.GetCountAddedInThisIterationStep(); + if (index == 0) { + UNIT_ASSERT_EQUAL(countInBucket, 1001); + UNIT_ASSERT_EQUAL(countInStep, 1001); + } else { + UNIT_ASSERT(countInBucket >= 1); + UNIT_ASSERT(countInStep >= 1); + } + index++; + totalCount += countInStep; + } + + UNIT_ASSERT_EQUAL(index, 1000); + UNIT_ASSERT_EQUAL(totalCount, 2000); + } + + Y_UNIT_TEST(LinearValues) { + THistogram h(TDuration::Hours(1).MicroSeconds(), 3); + UNIT_ASSERT(h.RecordValues(1000, 1000)); + UNIT_ASSERT(h.RecordValue(1000 * 1000)); + + int index = 0; + TLinearIterator it(h, 1000); + + while (it.Next()) { + i64 countInBucket = it.GetCount(); + i64 countInStep = it.GetCountAddedInThisIterationStep(); + if (index == 0) { + UNIT_ASSERT_EQUAL(countInBucket, 1000); + UNIT_ASSERT_EQUAL(countInStep, 1000); + } else if (index == 999) { + UNIT_ASSERT_EQUAL(countInBucket, 1); + UNIT_ASSERT_EQUAL(countInStep, 1); + } else { + UNIT_ASSERT_EQUAL(countInBucket, 0); + UNIT_ASSERT_EQUAL(countInStep, 0); + } + + index++; + } + + UNIT_ASSERT_EQUAL(index, 1000); + } + + Y_UNIT_TEST(CorrectLinearValues) { + THistogram h(TDuration::Hours(1).MicroSeconds(), 3); + UNIT_ASSERT(h.RecordValuesWithExpectedInterval(1000, 1000, 1000)); + UNIT_ASSERT(h.RecordValueWithExpectedInterval(1000 * 1000, 1000)); + + int index = 0; + i64 totalCount = 0; + TLinearIterator it(h, 1000); + + while (it.Next()) { + i64 countInBucket = it.GetCount(); + i64 countInStep = it.GetCountAddedInThisIterationStep(); + + if (index == 0) { + UNIT_ASSERT_EQUAL(countInBucket, 1001); + UNIT_ASSERT_EQUAL(countInStep, 1001); + } else { + UNIT_ASSERT_EQUAL(countInBucket, 1); + UNIT_ASSERT_EQUAL(countInStep, 1); + } + + index++; + totalCount += countInStep; + } + + UNIT_ASSERT_EQUAL(index, 1000); + UNIT_ASSERT_EQUAL(totalCount, 2000); + } + + Y_UNIT_TEST(LogarithmicValues) { + THistogram h(TDuration::Hours(1).MicroSeconds(), 3); + UNIT_ASSERT(h.RecordValues(1000, 1000)); + UNIT_ASSERT(h.RecordValue(1000 * 1000)); + + int index = 0; + i64 expectedValue = 1000; + TLogarithmicIterator it(h, 1000, 2.0); + + while (it.Next()) { + i64 value = it.GetValue(); + i64 countInBucket = it.GetCount(); + i64 countInStep = it.GetCountAddedInThisIterationStep(); + + UNIT_ASSERT_EQUAL(value, expectedValue); + + if (index == 0) { + UNIT_ASSERT_EQUAL(countInBucket, 1000); + UNIT_ASSERT_EQUAL(countInStep, 1000); + } else if (index == 10) { + UNIT_ASSERT_EQUAL(countInBucket, 0); + UNIT_ASSERT_EQUAL(countInStep, 1); + } else { + UNIT_ASSERT_EQUAL(countInBucket, 0); + UNIT_ASSERT_EQUAL(countInStep, 0); + } + + index++; + expectedValue *= 2; + } + + UNIT_ASSERT_EQUAL(index, 11); + } + + Y_UNIT_TEST(CorrectedLogarithmicValues) { + THistogram h(TDuration::Hours(1).MicroSeconds(), 3); + UNIT_ASSERT(h.RecordValuesWithExpectedInterval(1000, 1000, 1000)); + UNIT_ASSERT(h.RecordValueWithExpectedInterval(1000 * 1000, 1000)); + + int index = 0; + i64 totalCount = 0; + i64 expectedValue = 1000; + TLogarithmicIterator it(h, 1000, 2.0); + + while (it.Next()) { + i64 value = it.GetValue(); + i64 countInBucket = it.GetCount(); + i64 countInStep = it.GetCountAddedInThisIterationStep(); + + UNIT_ASSERT_EQUAL(value, expectedValue); + + if (index == 0) { + UNIT_ASSERT_EQUAL(countInBucket, 1001); + UNIT_ASSERT_EQUAL(countInStep, 1001); + } + + index++; + totalCount += countInStep; + expectedValue *= 2; + } + + UNIT_ASSERT_EQUAL(index, 11); + UNIT_ASSERT_EQUAL(totalCount, 2000); + } + + Y_UNIT_TEST(LinearIterBucketsCorrectly) { + THistogram h(255, 2); + UNIT_ASSERT(h.RecordValue(193)); + UNIT_ASSERT(h.RecordValue(255)); + UNIT_ASSERT(h.RecordValue(0)); + UNIT_ASSERT(h.RecordValue(1)); + UNIT_ASSERT(h.RecordValue(64)); + UNIT_ASSERT(h.RecordValue(128)); + + int index = 0; + i64 totalCount = 0; + TLinearIterator it(h, 64); + + while (it.Next()) { + if (index == 0) { + // change after iterator was created + UNIT_ASSERT(h.RecordValue(2)); + } + + index++; + totalCount += it.GetCountAddedInThisIterationStep(); + } + + UNIT_ASSERT_EQUAL(index, 4); + UNIT_ASSERT_EQUAL(totalCount, 6); + } +} diff --git a/library/cpp/histogram/hdr/histogram_ut.cpp b/library/cpp/histogram/hdr/histogram_ut.cpp new file mode 100644 index 0000000000..4841b76e71 --- /dev/null +++ b/library/cpp/histogram/hdr/histogram_ut.cpp @@ -0,0 +1,178 @@ +#include "histogram.h" + +#include <library/cpp/testing/unittest/registar.h> + +#include <util/generic/cast.h> + +using namespace NHdr; + +void LoadData(THistogram* h1, THistogram* h2) { + for (int i = 0; i < 1000; i++) { + UNIT_ASSERT(h1->RecordValue(1000)); + UNIT_ASSERT(h2->RecordValueWithExpectedInterval(1000, 1000)); + } + + UNIT_ASSERT(h1->RecordValue(1000 * 1000)); + UNIT_ASSERT(h2->RecordValueWithExpectedInterval(1000 * 1000, 1000)); +} + +Y_UNIT_TEST_SUITE(THistogramTest) { + Y_UNIT_TEST(Creation) { + THistogram h(TDuration::Hours(1).MicroSeconds(), 3); + UNIT_ASSERT_EQUAL(h.GetMemorySize(), 188512); + UNIT_ASSERT_EQUAL(h.GetCountsLen(), 23552); + } + + Y_UNIT_TEST(CreateWithLargeValues) { + THistogram h(20L * 1000 * 1000, 100L * 1000 * 1000, 5); + UNIT_ASSERT(h.RecordValue(100L * 1000 * 1000)); + UNIT_ASSERT(h.RecordValue(20L * 1000 * 1000)); + UNIT_ASSERT(h.RecordValue(30L * 1000 * 1000)); + + i64 v50 = h.GetValueAtPercentile(50.0); + i64 v8333 = h.GetValueAtPercentile(83.33); + i64 v8334 = h.GetValueAtPercentile(83.34); + i64 v99 = h.GetValueAtPercentile(99.0); + + UNIT_ASSERT_EQUAL(v50, 33554431); + UNIT_ASSERT_EQUAL(v8333, 33554431); + UNIT_ASSERT_EQUAL(v8334, 100663295); + UNIT_ASSERT_EQUAL(v99, 100663295); + + UNIT_ASSERT(h.ValuesAreEqual(v50, 20L * 1000 * 1000)); + UNIT_ASSERT(h.ValuesAreEqual(v8333, 30L * 1000 * 1000)); + UNIT_ASSERT(h.ValuesAreEqual(v8334, 100L * 1000 * 1000)); + UNIT_ASSERT(h.ValuesAreEqual(v99, 100L * 1000 * 1000)); + } + + Y_UNIT_TEST(InvalidSignificantValueDigits) { + UNIT_ASSERT_EXCEPTION(THistogram(1000, -1), yexception); + UNIT_ASSERT_EXCEPTION(THistogram(1000, 0), yexception); + UNIT_ASSERT_EXCEPTION(THistogram(1000, 6), yexception); + } + + Y_UNIT_TEST(InvalidLowestDiscernibleValue) { + UNIT_ASSERT_EXCEPTION(THistogram(0, 100, 3), yexception); + UNIT_ASSERT_EXCEPTION(THistogram(110, 100, 3), yexception); + } + + Y_UNIT_TEST(TotalCount) { + i64 oneHour = SafeIntegerCast<i64>(TDuration::Hours(1).MicroSeconds()); + THistogram h1(oneHour, 3); + THistogram h2(oneHour, 3); + LoadData(&h1, &h2); + + UNIT_ASSERT_EQUAL(h1.GetTotalCount(), 1001); + UNIT_ASSERT_EQUAL(h2.GetTotalCount(), 2000); + } + + Y_UNIT_TEST(StatsValues) { + i64 oneHour = SafeIntegerCast<i64>(TDuration::Hours(1).MicroSeconds()); + THistogram h1(oneHour, 3); + THistogram h2(oneHour, 3); + LoadData(&h1, &h2); + + // h1 - histogram without correction + { + UNIT_ASSERT_EQUAL(h1.GetMin(), 1000); + UNIT_ASSERT_EQUAL(h1.GetMax(), 1000447); + UNIT_ASSERT(h1.ValuesAreEqual(h1.GetMax(), 1000 * 1000)); + + // >>> numpy.mean([1000 for i in range(1000)] + [1000000]) + // 1998.0019980019979 + UNIT_ASSERT_DOUBLES_EQUAL(h1.GetMean(), 1998, 0.5); + + // >>> numpy.std([1000 for i in range(1000)] + [1000000]) + // 31559.59423085126 + UNIT_ASSERT_DOUBLES_EQUAL(h1.GetStdDeviation(), 31559, 10); + } + + // h2 - histogram with correction of co-ordinated omission + { + UNIT_ASSERT_EQUAL(h2.GetMin(), 1000); + UNIT_ASSERT_EQUAL(h2.GetMax(), 1000447); + UNIT_ASSERT(h2.ValuesAreEqual(h1.GetMax(), 1000 * 1000)); + UNIT_ASSERT_DOUBLES_EQUAL(h2.GetMean(), 250752, 0.5); + UNIT_ASSERT_DOUBLES_EQUAL(h2.GetStdDeviation(), 322557, 0.5); + } + } + + Y_UNIT_TEST(Percentiles) { + i64 oneHour = SafeIntegerCast<i64>(TDuration::Hours(1).MicroSeconds()); + THistogram h1(oneHour, 3); + THistogram h2(oneHour, 3); + LoadData(&h1, &h2); + + // >>> a = [1000 for i in range(1000)] + [1000000] + // >>> [(p, numpy.percentile(a, p)) for p in [50, 90, 99, 99.99, 99.999, 100]] + // [(50, 1000.0), (90, 1000.0), (99, 1000.0), (99.99, 900099.99999986368), (99.999, 990009.99999989558), (100, 1000000.0)] + + // h1 - histogram without correction + { + i64 v50 = h1.GetValueAtPercentile(50); + i64 v90 = h1.GetValueAtPercentile(90); + i64 v99 = h1.GetValueAtPercentile(99); + i64 v9999 = h1.GetValueAtPercentile(99.99); + i64 v99999 = h1.GetValueAtPercentile(99.999); + i64 v100 = h1.GetValueAtPercentile(100); + + UNIT_ASSERT_EQUAL(v50, 1000); + UNIT_ASSERT_EQUAL(v90, 1000); + UNIT_ASSERT_EQUAL(v99, 1000); + UNIT_ASSERT_EQUAL(v9999, 1000447); + UNIT_ASSERT_EQUAL(v99999, 1000447); + UNIT_ASSERT_EQUAL(v100, 1000447); + + UNIT_ASSERT(h1.ValuesAreEqual(v50, 1000)); + UNIT_ASSERT(h1.ValuesAreEqual(v90, 1000)); + UNIT_ASSERT(h1.ValuesAreEqual(v99, 1000)); + UNIT_ASSERT(h1.ValuesAreEqual(v9999, 1000 * 1000)); + UNIT_ASSERT(h1.ValuesAreEqual(v99999, 1000 * 1000)); + UNIT_ASSERT(h1.ValuesAreEqual(v100, 1000 * 1000)); + } + + // h2 - histogram with correction of co-ordinated omission + { + i64 v50 = h2.GetValueAtPercentile(50); + i64 v90 = h2.GetValueAtPercentile(90); + i64 v99 = h2.GetValueAtPercentile(99); + i64 v9999 = h2.GetValueAtPercentile(99.99); + i64 v99999 = h2.GetValueAtPercentile(99.999); + i64 v100 = h2.GetValueAtPercentile(100); + + UNIT_ASSERT_EQUAL(v50, 1000); + UNIT_ASSERT_EQUAL(v90, 800255); + UNIT_ASSERT_EQUAL(v99, 980479); + UNIT_ASSERT_EQUAL(v9999, 1000447); + UNIT_ASSERT_EQUAL(v99999, 1000447); + UNIT_ASSERT_EQUAL(v100, 1000447); + + UNIT_ASSERT(h2.ValuesAreEqual(v50, 1000)); + UNIT_ASSERT(h2.ValuesAreEqual(v90, 800 * 1000)); + UNIT_ASSERT(h2.ValuesAreEqual(v99, 980 * 1000)); + UNIT_ASSERT(h2.ValuesAreEqual(v9999, 1000 * 1000)); + UNIT_ASSERT(h2.ValuesAreEqual(v99999, 1000 * 1000)); + UNIT_ASSERT(h2.ValuesAreEqual(v100, 1000 * 1000)); + } + } + + Y_UNIT_TEST(OutOfRangeValues) { + THistogram h(1000, 4); + UNIT_ASSERT(h.RecordValue(32767)); + UNIT_ASSERT(!h.RecordValue(32768)); + } + + Y_UNIT_TEST(Reset) { + THistogram h(TDuration::Hours(1).MicroSeconds(), 3); + UNIT_ASSERT(h.RecordValues(1000, 1000)); + UNIT_ASSERT(h.RecordValue(1000 * 1000)); + UNIT_ASSERT_EQUAL(h.GetTotalCount(), 1001); + + h.Reset(); + + UNIT_ASSERT_EQUAL(h.GetTotalCount(), 0); + UNIT_ASSERT_EQUAL(h.GetMin(), Max<i64>()); + UNIT_ASSERT_EQUAL(h.GetMax(), 0); + UNIT_ASSERT_EQUAL(h.GetValueAtPercentile(99.0), 0); + } +} diff --git a/library/cpp/histogram/hdr/ut/ya.make b/library/cpp/histogram/hdr/ut/ya.make new file mode 100644 index 0000000000..13ceb143c8 --- /dev/null +++ b/library/cpp/histogram/hdr/ut/ya.make @@ -0,0 +1,13 @@ +OWNER( + g:util + jamel +) + +UNITTEST_FOR(library/cpp/histogram/hdr) + +SRCS( + histogram_ut.cpp + histogram_iter_ut.cpp +) + +END() diff --git a/library/cpp/histogram/hdr/ya.make b/library/cpp/histogram/hdr/ya.make new file mode 100644 index 0000000000..885099608d --- /dev/null +++ b/library/cpp/histogram/hdr/ya.make @@ -0,0 +1,17 @@ +LIBRARY() + +OWNER( + g:util + jamel +) + +SRCS( + histogram.cpp + histogram_iter.cpp +) + +PEERDIR( + contrib/libs/hdr_histogram +) + +END() diff --git a/ydb/core/blobstorage/testload/test_load_kqp.cpp b/ydb/core/blobstorage/testload/test_load_kqp.cpp index 6a59ae572f..fec4fd7b2c 100644 --- a/ydb/core/blobstorage/testload/test_load_kqp.cpp +++ b/ydb/core/blobstorage/testload/test_load_kqp.cpp @@ -14,6 +14,7 @@ #include <ydb/core/ydb_convert/ydb_convert.h> #include <ydb/library/workload/workload_factory.h> +#include <ydb/library/workload/stock_workload.h> #include <ydb/public/lib/operation_id/operation_id.h> #include <ydb/public/sdk/cpp/client/ydb_params/params.h> @@ -64,9 +65,11 @@ class TKqpWriterTestLoadActor : public TActorBootstrapped<TKqpWriterTestLoadActo bool SequentialWrite; TString StringValue; TString WorkingDir; + size_t ProductCount; + size_t Quantity; bool DeleteTableOnFinish; std::vector<TString> preparedQuery; - std::unique_ptr<IWorkloadQueryGenerator> WorkloadQueryGen; + std::shared_ptr<NYdbWorkload::IWorkloadQueryGenerator> WorkloadQueryGen; TReallyFastRng32 Rng; @@ -109,14 +112,20 @@ public: StringValue = TString(StringValueSize, 'a'); WorkingDir = cmd.GetWorkingDir(); - TWorkloadParams* workloadParams = nullptr; + if (cmd.Workload_case() == NKikimrBlobStorage::TEvTestLoadRequest_TKqpLoadStart::WorkloadCase::kStock) { + ProductCount = cmd.GetStock().GetProductCount(); + Quantity = cmd.GetStock().GetQuantity(); + } + + NYdbWorkload::TWorkloadParams* workloadParams = nullptr; if (cmd.GetWorkloadName() == "stock") { - TStockWorkloadParams params; + NYdbWorkload::TStockWorkloadParams params; params.DbPath = WorkingDir; params.PartitionsByLoad = true; workloadParams = ¶ms; } - WorkloadQueryGen = TWorkloadFactory::GetWorkloadQueryGenerator(cmd.GetWorkloadName(), workloadParams); + NYdbWorkload::TWorkloadFactory factory; + WorkloadQueryGen = factory.GetWorkloadQueryGenerator(cmd.GetWorkloadName(), workloadParams); Y_ASSERT(WorkloadQueryGen.get() != nullptr); Y_ASSERT(DurationSeconds > DelayBeforeMeasurements.Seconds()); diff --git a/ydb/core/protos/blobstorage.proto b/ydb/core/protos/blobstorage.proto index 4304cd1633..a08ccbf865 100644 --- a/ydb/core/protos/blobstorage.proto +++ b/ydb/core/protos/blobstorage.proto @@ -1449,6 +1449,10 @@ message TEvTestLoadRequest { } message TKqpLoadStart { + message TStockWorkload { + optional uint64 ProductCount = 1; + optional uint64 Quantity = 2; + } optional uint64 Tag = 1; optional uint32 DurationSeconds = 2; optional uint32 NumOfSessions = 3; @@ -1461,6 +1465,9 @@ message TEvTestLoadRequest { optional uint64 TotalRowsToUpsert = 10; optional string WorkingDir = 11; optional string WorkloadName = 12; + oneof Workload { + TStockWorkload Stock = 13; + } } message TMemoryLoadStart { diff --git a/ydb/library/workload/stock_workload.cpp b/ydb/library/workload/stock_workload.cpp new file mode 100644 index 0000000000..1b8a9ffad7 --- /dev/null +++ b/ydb/library/workload/stock_workload.cpp @@ -0,0 +1,282 @@ +#include "stock_workload.h" + +#include <util/datetime/base.h> + +#include <cmath> +#include <iomanip> +#include <string> +#include <thread> +#include <random> + +namespace { +uint64_t getOrderId() { + static thread_local std::mt19937_64 generator; + generator.seed(Now().MicroSeconds() + std::hash<std::thread::id>{}(std::this_thread::get_id())); + std::uniform_int_distribution<uint64_t> distribution(1, UINT64_MAX); + return distribution(generator); +} +} + +namespace NYdbWorkload { + +TStockWorkloadGenerator::TStockWorkloadGenerator(const TStockWorkloadParams* params) + : DbPath(params->DbPath) + , Params(*params) + , Rd() + , Gen(Rd()) + , RandExpDistrib(1.6) + , CustomerIdGenerator(1, MAX_CUSTOMERS) + , ProductIdGenerator(1, params->ProductCount) +{ + Gen.seed(Now().MicroSeconds()); +} + +std::string TStockWorkloadGenerator::GetDDLQueries() const { + std::string StockPartitionsDdl = ""; + std::string OrdersPartitionsDdl = "WITH (READ_REPLICAS_SETTINGS = \"per_az:1\")"; + std::string OrderLinesPartitionsDdl = ""; + + if (Params.PartitionsByLoad) { + std::string partsNum = std::to_string(Params.MinPartitions); + + StockPartitionsDdl = "WITH (AUTO_PARTITIONING_BY_LOAD = ENABLED, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = " + partsNum + ")"; + OrdersPartitionsDdl = "WITH (READ_REPLICAS_SETTINGS = \"per_az:1\", AUTO_PARTITIONING_BY_LOAD = ENABLED, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = " + + partsNum + ", UNIFORM_PARTITIONS = " + partsNum + ")"; + OrderLinesPartitionsDdl = "WITH (AUTO_PARTITIONING_BY_LOAD = ENABLED, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = " + + partsNum + ", UNIFORM_PARTITIONS = " + partsNum + ")"; + } + + static const char TablesDdl[] = R"(--!syntax_v1 + CREATE TABLE `%s/stock`(product Utf8, quantity Int64, PRIMARY KEY(product)) %s; + CREATE TABLE `%s/orders`(id Uint64, customer Utf8, created Datetime, processed Datetime, PRIMARY KEY(id), INDEX ix_cust GLOBAL ON (customer, created) COVER (processed)) %s; + CREATE TABLE `%s/orderLines`(id_order Uint64, product Utf8, quantity Int64, PRIMARY KEY(id_order, product)) %s; + )"; + char buf[sizeof(TablesDdl) + sizeof(OrdersPartitionsDdl) + 8192*3]; // 32*256 for DbPath + + int res = std::sprintf(buf, TablesDdl, + DbPath.c_str(), StockPartitionsDdl.c_str(), + DbPath.c_str(), OrdersPartitionsDdl.c_str(), + DbPath.c_str(), OrderLinesPartitionsDdl.c_str() + ); + if (res < 0) { + return ""; + } + return buf; +} + +TQueryInfoList TStockWorkloadGenerator::GetInitialData() { + std::list<TQueryInfo> res; + res.push_back(FillStockData()); + for (size_t i = 0; i < Params.OrderCount; ++i) { + auto queryInfos = InsertRandomOrder(); + res.insert(res.end(), queryInfos.begin(), queryInfos.end()); + } + return res; +} + +TQueryInfo TStockWorkloadGenerator::FillStockData() const { + std::string query = R"(--!syntax_v1 + DECLARE $stocks AS List<Struct<product:Utf8,quantity:Int64>>; + INSERT INTO `stock`(product, quantity) SELECT product, quantity from AS_TABLE( $stocks ); + )"; + + char productName[8] = ""; + NYdb::TValueBuilder rows; + rows.BeginList(); + for (size_t i = 0; i < Params.ProductCount; ++i) { + std::sprintf(productName, "p%.6zu", i); + rows.AddListItem() + .BeginStruct() + .AddMember("product").Utf8(productName) + .AddMember("quantity").Int64(Params.Quantity) + .EndStruct(); + } + rows.EndList(); + + NYdb::TParamsBuilder paramsBuilder; + paramsBuilder.AddParam("$stocks", rows.Build()); + + return TQueryInfo(query, paramsBuilder.Build()); +} + +TQueryInfoList TStockWorkloadGenerator::GetWorkload(int type) { + switch (static_cast<EType>(type)) { + case EType::InsertRandomOrder: + return InsertRandomOrder(); + case EType::SubmitRandomOrder: + return SubmitRandomOrder(); + case EType::SubmitSameOrder: + return SubmitSameOrder(); + case EType::GetRandomCustomerHistory: + return GetRandomCustomerHistory(); + case EType::GetCustomerHistory: + return GetCustomerHistory(); + default: + return TQueryInfoList(); + } +} + +TQueryInfo TStockWorkloadGenerator::InsertOrder(const uint64_t orderID, const std::string& customer, const TProductsQuantity& products) { + std::string query = R"(--!syntax_v1 + DECLARE $ido AS UInt64; + DECLARE $cust as Utf8; + DECLARE $lines AS List<Struct<product:Utf8,quantity:Int64>>; + DECLARE $time AS DateTime; + INSERT INTO `orders`(id, customer, created) values ($ido, $cust, $time); + UPSERT INTO `orderLines`(id_order, product, quantity) SELECT $ido, product, quantity from AS_TABLE( $lines ); + )"; + + NYdb::TValueBuilder rows; + rows.BeginList(); + for (auto const& [product, quantity] : products) { + rows.AddListItem() + .BeginStruct() + .AddMember("product").Utf8(product.c_str()) + .AddMember("quantity").Int64(quantity) + .EndStruct(); + } + rows.EndList(); + + NYdb::TParamsBuilder paramsBuilder; + paramsBuilder + .AddParam("$ido") + .Uint64(orderID) + .Build() + .AddParam("$cust") + .Utf8(customer.c_str()) + .Build() + .AddParam("$time") + .Datetime(Now()) + .Build() + .AddParam("$lines", rows.Build()); + + return TQueryInfo(query, paramsBuilder.Build()); +} + +TQueryInfo TStockWorkloadGenerator::ExecuteOrder(const uint64_t orderID) { + std::string query = R"(--!syntax_v1 + DECLARE $ido AS UINT64; + DECLARE $time AS DateTime; + $prods = SELECT * FROM orderLines as p WHERE p.id_order = $ido; + $cnt = SELECT count(*) FROM $prods; + $newq = SELECT p.product AS product, COALESCE(s.quantity,0)-p.quantity AS quantity + FROM $prods as p LEFT JOIN stock AS s on s.product = p.product; + $check = SELECT count(*) as cntd FROM $newq as q where q.quantity >= 0; + UPSERT INTO stock SELECT product, quantity FROM $newq where $check=$cnt; + $upo = SELECT id, $time as tm FROM orders WHERE id = $ido and $check = $cnt; + UPSERT INTO orders SELECT id, tm as processed FROM $upo; + SELECT * from $newq as q where q.quantity < 0 + )"; + + NYdb::TParamsBuilder paramsBuilder; + paramsBuilder + .AddParam("$ido") + .Uint64(orderID) + .Build() + .AddParam("$time") + .Datetime(Now()) + .Build(); + + return TQueryInfo(query, paramsBuilder.Build()); +} + +TQueryInfo TStockWorkloadGenerator::SelectCustomerHistory(const std::string& customerId, const unsigned int limit) { + std::string query = R"(--!syntax_v1 + DECLARE $cust as Utf8; + DECLARE $limit as UInt32; + select id, customer, created + from orders view ix_cust + where customer = $cust + order by customer desc, created desc + limit $limit; + )"; + + NYdb::TParamsBuilder paramsBuilder; + paramsBuilder + .AddParam("$cust") + .Utf8(customerId.c_str()) + .Build() + .AddParam("$limit") + .Uint32(limit) + .Build(); + + return TQueryInfo(query, paramsBuilder.Build()); +} + +std::string TStockWorkloadGenerator::GetCustomerId() { + return "Name" + std::to_string(CustomerIdGenerator(Gen)); +} + +unsigned int TStockWorkloadGenerator::GetProductCountInOrder() { + unsigned int productCountInOrder = 0; + while (productCountInOrder == 0) { + productCountInOrder = std::abs(std::round(RandExpDistrib(Gen) * 2)); + } + return productCountInOrder; +} + +TStockWorkloadGenerator::TProductsQuantity TStockWorkloadGenerator::GenerateOrder(unsigned int productCountInOrder, int quantity) { + char productName[8] = ""; + TProductsQuantity products; + for (unsigned i = 0; i < productCountInOrder; ++i) { + std::sprintf(productName, "p%.6i", ProductIdGenerator(Gen)); + products.emplace(productName, quantity); + } + return products; +} + +TQueryInfoList TStockWorkloadGenerator::InsertRandomOrder() { + uint64_t orderID = getOrderId(); + auto customer = GetCustomerId(); + auto productCountInOrder = GetProductCountInOrder(); + auto products = GenerateOrder(productCountInOrder, 1); + return TQueryInfoList(1, InsertOrder(orderID, customer, products)); +} + +TQueryInfoList TStockWorkloadGenerator::SubmitRandomOrder() { + TQueryInfoList res; + + uint64_t orderID = getOrderId(); + auto customer = GetCustomerId(); + auto productCountInOrder = GetProductCountInOrder(); + auto products = GenerateOrder(productCountInOrder, 1); + + res.push_back(InsertOrder(orderID, customer, products)); + res.push_back(ExecuteOrder(orderID)); + return res; +} + +TQueryInfoList TStockWorkloadGenerator::SubmitSameOrder() { + TQueryInfoList res; + + uint64_t orderID = getOrderId(); + auto customer = GetCustomerId(); + + char productName[8] = ""; + TProductsQuantity products; + for (unsigned i = 0; i < Params.ProductCount; ++i) { + std::sprintf(productName, "p%.6i", i); + products.emplace(productName, 1); + } + res.push_back(InsertOrder(orderID, customer, products)); + res.push_back(ExecuteOrder(orderID)); + return res; +} + +TQueryInfoList TStockWorkloadGenerator::GetRandomCustomerHistory() { + TQueryInfoList res; + + auto customer = GetCustomerId(); + res.push_back(SelectCustomerHistory(customer, Params.Limit)); + return res; +} + +TQueryInfoList TStockWorkloadGenerator::GetCustomerHistory() { + TQueryInfoList res; + + auto customer = "Name" + std::to_string(MAX_CUSTOMERS); + res.push_back(SelectCustomerHistory(customer, Params.Limit)); + return res; +} + +}
\ No newline at end of file diff --git a/ydb/library/workload/stock_workload.h b/ydb/library/workload/stock_workload.h index 99aa5e27c0..3725cfe139 100644 --- a/ydb/library/workload/stock_workload.h +++ b/ydb/library/workload/stock_workload.h @@ -1,11 +1,20 @@ #pragma once -#include "workload_query_gen.h" +#include "workload_query_generator.h" #include <cctype> +#include <random> + +namespace NYdbWorkload { + struct TStockWorkloadParams : public TWorkloadParams { - bool PartitionsByLoad; + size_t ProductCount = 0; + size_t Quantity = 0; + size_t OrderCount = 0; + unsigned int MinPartitions = 0; + unsigned int Limit = 0; + bool PartitionsByLoad = true; }; class TStockWorkloadGenerator : public IWorkloadQueryGenerator { @@ -20,28 +29,39 @@ public: virtual ~TStockWorkloadGenerator() {} - std::string GetDDLQueries() override { - static const char TablesDdl[] = R"(--!syntax_v1 - CREATE TABLE `%s/stock`(product Utf8, quantity Int64, PRIMARY KEY(product)) %s; - CREATE TABLE `%s/orders`(id Uint64, customer Utf8, created Datetime, processed Datetime, PRIMARY KEY(id)); - CREATE TABLE `%s/orderLines`(id_order Uint64, product Utf8, quantity Int64, PRIMARY KEY(id_order, product)); - )"; - static const char PartitionsDdl[] = R"(WITH (AUTO_PARTITIONING_BY_LOAD = ENABLED, AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = 3))"; - char buf[sizeof(TablesDdl) + sizeof(PartitionsDdl) + 8192*3]; // 32*256 for DbPath - - int res = std::sprintf(buf, TablesDdl, DbPath.c_str(), (PartitionsByLoad ? PartitionsDdl : ""), DbPath.c_str(), DbPath.c_str()); - if (res < 0) { - return ""; - } - return buf; - } + std::string GetDDLQueries() const override; + + TQueryInfoList GetInitialData() override; + + TQueryInfoList GetWorkload(int type) override; + + enum class EType { + InsertRandomOrder, + SubmitRandomOrder, + SubmitSameOrder, + GetRandomCustomerHistory, + GetCustomerHistory + }; private: + static const unsigned int MAX_CUSTOMERS = 10000; // We will have just 10k customers - TStockWorkloadGenerator(const TStockWorkloadParams* params) { - DbPath = params->DbPath; - PartitionsByLoad = params->PartitionsByLoad; - } + TQueryInfoList InsertRandomOrder(); + TQueryInfoList SubmitRandomOrder(); + TQueryInfoList SubmitSameOrder(); + TQueryInfoList GetRandomCustomerHistory(); + TQueryInfoList GetCustomerHistory(); + + using TProductsQuantity = std::map<std::string, int64_t>; + TQueryInfo InsertOrder(const uint64_t orderID, const std::string& customer, const TProductsQuantity& products); + TQueryInfo ExecuteOrder(const uint64_t orderID); + TQueryInfo SelectCustomerHistory(const std::string& customerId, const unsigned int limit); + + std::string GetCustomerId(); + unsigned int GetProductCountInOrder(); + TProductsQuantity GenerateOrder(unsigned int productCountInOrder, int quantity); + + TStockWorkloadGenerator(const TStockWorkloadParams* params); static bool validateDbPath(const std::string& path) { for (size_t i = 0; i < path.size(); ++i) { @@ -53,6 +73,16 @@ private: return true; } + TQueryInfo FillStockData() const; + std::string DbPath; - bool PartitionsByLoad; + TStockWorkloadParams Params; + + std::random_device Rd; + std::mt19937_64 Gen; + std::exponential_distribution<> RandExpDistrib; + std::uniform_int_distribution<unsigned int> CustomerIdGenerator; + std::uniform_int_distribution<unsigned int> ProductIdGenerator; }; + +} // namespace NYdbWorkload
\ No newline at end of file diff --git a/ydb/library/workload/workload_factory.cpp b/ydb/library/workload/workload_factory.cpp new file mode 100644 index 0000000000..f1a34ea78c --- /dev/null +++ b/ydb/library/workload/workload_factory.cpp @@ -0,0 +1,21 @@ +#include "workload_factory.h" + +#include "stock_workload.h" + +namespace NYdbWorkload { + + std::shared_ptr<IWorkloadQueryGenerator> TWorkloadFactory::GetWorkloadQueryGenerator(const std::string& workloadName, + const TWorkloadParams* params) + { + if (!params) { + return nullptr; + } + + if (workloadName == "stock") { + return std::shared_ptr<TStockWorkloadGenerator>(TStockWorkloadGenerator::New(static_cast<const TStockWorkloadParams*>(params))); + } else { + return nullptr; + } + } + +}
\ No newline at end of file diff --git a/ydb/library/workload/workload_factory.h b/ydb/library/workload/workload_factory.h index 8173adc4bb..fbfaaa6546 100644 --- a/ydb/library/workload/workload_factory.h +++ b/ydb/library/workload/workload_factory.h @@ -1,21 +1,14 @@ #pragma once -#include "workload_query_gen.h" -#include "stock_workload.h" +#include "workload_query_generator.h" #include <memory> +namespace NYdbWorkload { + class TWorkloadFactory { public: - static std::unique_ptr<IWorkloadQueryGenerator> GetWorkloadQueryGenerator(const std::string& workloadName, const TWorkloadParams* params) { - if (!params) { - return nullptr; - } - - if (workloadName == "stock") { - return std::unique_ptr<TStockWorkloadGenerator>(TStockWorkloadGenerator::New(static_cast<const TStockWorkloadParams*>(params))); - } else { - return nullptr; - } - } + std::shared_ptr<IWorkloadQueryGenerator> GetWorkloadQueryGenerator(const std::string& workloadName, const TWorkloadParams* params); }; + +} // namespace NYdbWorkload
\ No newline at end of file diff --git a/ydb/library/workload/workload_query_generator.h b/ydb/library/workload/workload_query_generator.h new file mode 100644 index 0000000000..4eb02efb54 --- /dev/null +++ b/ydb/library/workload/workload_query_generator.h @@ -0,0 +1,43 @@ +#pragma once + +#include <ydb/public/sdk/cpp/client/ydb_params/params.h> + +#include <list> +#include <map> +#include <string> + +namespace NYdbWorkload { + +struct TQueryInfo { + TQueryInfo() + : Query("") + , Params(NYdb::TParamsBuilder().Build()) + {} + + TQueryInfo(const std::string& query, const NYdb::TParams&& params) + : Query(query) + , Params(std::move(params)) + {} + + std::string Query; + NYdb::TParams Params; +}; + +using TQueryInfoList = std::list<TQueryInfo>; + +class IWorkloadQueryGenerator { +public: + virtual ~IWorkloadQueryGenerator() {} + + virtual std::string GetDDLQueries() const = 0; + virtual TQueryInfoList GetInitialData() = 0; + + virtual TQueryInfoList GetWorkload(int type) = 0; +}; + +struct TWorkloadParams { + std::string DbPath; +}; + +} // namespace NYdbWorkload + diff --git a/ydb/library/workload/ya.make b/ydb/library/workload/ya.make index 80fb4b9baa..530f140cd3 100644 --- a/ydb/library/workload/ya.make +++ b/ydb/library/workload/ya.make @@ -3,9 +3,12 @@ LIBRARY() OWNER(g:kikimr) SRCS( - stock_workload.h - workload_factory.h - workload_query_gen.h + stock_workload.cpp + workload_factory.cpp +) + +PEERDIR( + ydb/public/api/protos ) END() diff --git a/ydb/public/lib/ydb_cli/commands/stock_workload.cpp b/ydb/public/lib/ydb_cli/commands/stock_workload.cpp new file mode 100644 index 0000000000..de53dd9202 --- /dev/null +++ b/ydb/public/lib/ydb_cli/commands/stock_workload.cpp @@ -0,0 +1,278 @@ +#include "stock_workload.h" + +#include <ydb/library/workload/stock_workload.h> +#include <ydb/library/workload/workload_factory.h> +#include <ydb/public/lib/ydb_cli/commands/ydb_common.h> + +namespace NYdb::NConsoleClient { + +NTable::TSession TWorkloadCommand::GetSession() { + NTable::TCreateSessionResult result = TableClient->GetSession(NTable::TCreateSessionSettings()).GetValueSync(); + ThrowOnError(result); + return result.GetSession(); +} + +TCommandStock::TCommandStock() + : TClientCommandTree("stock", {}, "YDB stock workload") +{ + AddCommand(std::make_unique<TCommandStockInit>()); + AddCommand(std::make_unique<TCommandStockRun>()); +} + +TCommandStockInit::TCommandStockInit() + : TWorkloadCommand("init", {}, "Create and initialize tables for workload") + , ProductCount(0) + , Quantity(0) + , MinPartitions(0) + , PartitionsByLoad(true) +{} + +void TCommandStockInit::Config(TConfig& config) { + TYdbCommand::Config(config); + + config.SetFreeArgsNum(0); + + config.Opts->AddLongOption('p', "products", "Product count. Value in 1..500 000.") + .DefaultValue(100).StoreResult(&ProductCount); + config.Opts->AddLongOption('q', "quantity", "Quantity of each product in stock.") + .DefaultValue(1000).StoreResult(&Quantity); + config.Opts->AddLongOption('o', "orders", "Initial orders count.") + .DefaultValue(100).StoreResult(&OrderCount); + config.Opts->AddLongOption("min-partitions", "Minimum partitions for tables.") + .DefaultValue(40).StoreResult(&MinPartitions); + config.Opts->AddLongOption("auto-partition", "Enable auto partitioning by load.") + .DefaultValue(true).StoreResult(&PartitionsByLoad); +} + +void TCommandStockInit::Parse(TConfig& config) { + TClientCommand::Parse(config); +} + +int TCommandStockInit::Run(TConfig& config) { + if (ProductCount > 500'000) { + throw TMissUseException() << "Product count must be in range 1..500 000." << Endl; + } + + Driver = std::make_unique<NYdb::TDriver>(CreateDriver(config)); + TableClient = std::make_unique<NTable::TTableClient>(*Driver); + NYdbWorkload::TStockWorkloadParams params; + params.DbPath = config.Database; + params.ProductCount = ProductCount; + params.Quantity = Quantity; + params.OrderCount = OrderCount; + params.MinPartitions = MinPartitions; + params.PartitionsByLoad = PartitionsByLoad; + + NYdbWorkload::TWorkloadFactory factory; + auto workloadGen = factory.GetWorkloadQueryGenerator("stock", ¶ms); + if (workloadGen.get() == nullptr) { + throw TMissUseException() << "Invalid path to database." << Endl; + } + + auto session = GetSession(); + auto result = session.ExecuteSchemeQuery(workloadGen->GetDDLQueries()).GetValueSync(); + ThrowOnError(result); + + auto queryInfoList = workloadGen->GetInitialData(); + for (auto queryInfo : queryInfoList) { + auto prepareResult = session.PrepareDataQuery(queryInfo.Query.c_str()).GetValueSync(); + if (!prepareResult.IsSuccess()) { + Cerr << "Prepare failed: " << prepareResult.GetIssues().ToString() << Endl + << "Query:\n" << queryInfo.Query << Endl; + return EXIT_FAILURE; + } + + auto dataQuery = prepareResult.GetQuery(); + auto result = dataQuery.Execute(NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx(), + std::move(queryInfo.Params)).GetValueSync(); + if (!result.IsSuccess()) { + Cerr << "Query execution failed: " << result.GetIssues().ToString() << Endl + << "Query:\n" << queryInfo.Query << Endl; + return EXIT_FAILURE; + } + } + + return EXIT_SUCCESS; +} + +TCommandStockRun::TCommandStockRun() + : TClientCommandTree("run", {}, "Run YDB stock workload") +{ + AddCommand(std::make_unique<TCommandStockRunInsertRandomOrder>()); + AddCommand(std::make_unique<TCommandStockRunSubmitRandomOrder>()); + AddCommand(std::make_unique<TCommandStockRunSubmitSameOrder>()); + AddCommand(std::make_unique<TCommandStockRunGetRandomCustomerHistory>()); + AddCommand(std::make_unique<TCommandStockRunGetCustomerHistory>()); +} + +TCommandStockRunInsertRandomOrder::TCommandStockRunInsertRandomOrder() + : TWorkloadCommand("insertRandomOrder", {}, "Inserts orders with random ID") + , ProductCount(0) +{} + +void TCommandStockRunInsertRandomOrder::Config(TConfig& config) { + TWorkloadCommand::Config(config); + + config.SetFreeArgsNum(0); + + config.Opts->AddLongOption('p', "products", "Products count to use in workload.") + .DefaultValue(100).StoreResult(&ProductCount); +} + +void TCommandStockRunInsertRandomOrder::Parse(TConfig& config) { + TClientCommand::Parse(config); +} + +int TCommandStockRunInsertRandomOrder::Run(TConfig& config) { + PrepareForRun(config); + + NYdbWorkload::TStockWorkloadParams params; + params.DbPath = config.Database; + params.ProductCount = ProductCount; + + NYdbWorkload::TWorkloadFactory factory; + auto workloadGen = factory.GetWorkloadQueryGenerator("stock", ¶ms); + if (workloadGen.get() == nullptr) { + throw TMissUseException() << "Invalid path to database." << Endl; + } + + return RunWorkload(workloadGen, static_cast<int>(NYdbWorkload::TStockWorkloadGenerator::EType::InsertRandomOrder)); +} + +TCommandStockRunSubmitRandomOrder::TCommandStockRunSubmitRandomOrder() + : TWorkloadCommand("submitRandomOrder", {}, "Submit random orders") + , ProductCount(0) +{} + +void TCommandStockRunSubmitRandomOrder::Config(TConfig& config) { + TWorkloadCommand::Config(config); + + config.SetFreeArgsNum(0); + + config.Opts->AddLongOption('p', "products", "Products count to use in workload.") + .DefaultValue(100).StoreResult(&ProductCount); +} + +void TCommandStockRunSubmitRandomOrder::Parse(TConfig& config) { + TClientCommand::Parse(config); +} + +int TCommandStockRunSubmitRandomOrder::Run(TConfig& config) { + PrepareForRun(config); + + NYdbWorkload::TStockWorkloadParams params; + params.DbPath = config.Database; + params.ProductCount = ProductCount; + + NYdbWorkload::TWorkloadFactory factory; + auto workloadGen = factory.GetWorkloadQueryGenerator("stock", ¶ms); + if (workloadGen.get() == nullptr) { + throw TMissUseException() << "Invalid path to database." << Endl; + } + + return RunWorkload(workloadGen, static_cast<int>(NYdbWorkload::TStockWorkloadGenerator::EType::SubmitRandomOrder)); +} + +TCommandStockRunSubmitSameOrder::TCommandStockRunSubmitSameOrder() + : TWorkloadCommand("submitSameOrder", {}, "Submit orders with same products") + , ProductCount(0) +{} + +void TCommandStockRunSubmitSameOrder::Config(TConfig& config) { + TWorkloadCommand::Config(config); + + config.SetFreeArgsNum(0); + + config.Opts->AddLongOption('p', "products", "Products count to use in workload.") + .DefaultValue(100).StoreResult(&ProductCount); +} + +void TCommandStockRunSubmitSameOrder::Parse(TConfig& config) { + TClientCommand::Parse(config); +} + +int TCommandStockRunSubmitSameOrder::Run(TConfig& config) { + PrepareForRun(config); + + NYdbWorkload::TStockWorkloadParams params; + params.DbPath = config.Database; + params.ProductCount = ProductCount; + + NYdbWorkload::TWorkloadFactory factory; + auto workloadGen = factory.GetWorkloadQueryGenerator("stock", ¶ms); + if (workloadGen.get() == nullptr) { + throw TMissUseException() << "Invalid path to database." << Endl; + } + + return RunWorkload(workloadGen, static_cast<int>(NYdbWorkload::TStockWorkloadGenerator::EType::SubmitSameOrder)); +} + +TCommandStockRunGetRandomCustomerHistory::TCommandStockRunGetRandomCustomerHistory() + : TWorkloadCommand("getRandomCustomerHistory", {}, "Selects orders of random customer") + , Limit(0) +{} + +void TCommandStockRunGetRandomCustomerHistory::Config(TConfig& config) { + TWorkloadCommand::Config(config); + + config.SetFreeArgsNum(0); + + config.Opts->AddLongOption('l', "limit", "Number of last orders to select.") + .DefaultValue(10).StoreResult(&Limit); +} + +void TCommandStockRunGetRandomCustomerHistory::Parse(TConfig& config) { + TClientCommand::Parse(config); +} + +int TCommandStockRunGetRandomCustomerHistory::Run(TConfig& config) { + PrepareForRun(config); + + NYdbWorkload::TStockWorkloadParams params; + params.DbPath = config.Database; + params.Limit = Limit; + + NYdbWorkload::TWorkloadFactory factory; + auto workloadGen = factory.GetWorkloadQueryGenerator("stock", ¶ms); + if (workloadGen.get() == nullptr) { + throw TMissUseException() << "Invalid path to database." << Endl; + } + + return RunWorkload(workloadGen, static_cast<int>(NYdbWorkload::TStockWorkloadGenerator::EType::GetRandomCustomerHistory)); +} + +TCommandStockRunGetCustomerHistory::TCommandStockRunGetCustomerHistory() + : TWorkloadCommand("getCustomerHistory", {}, "Selects orders of 10000th customer") + , Limit(0) +{} + +void TCommandStockRunGetCustomerHistory::Config(TConfig& config) { + TWorkloadCommand::Config(config); + + config.SetFreeArgsNum(0); + + config.Opts->AddLongOption('l', "limit", "Number of last orders to select.") + .DefaultValue(10).StoreResult(&Limit); +} + +void TCommandStockRunGetCustomerHistory::Parse(TConfig& config) { + TClientCommand::Parse(config); +} + +int TCommandStockRunGetCustomerHistory::Run(TConfig& config) { + PrepareForRun(config); + + NYdbWorkload::TStockWorkloadParams params; + params.DbPath = config.Database; + params.Limit = Limit; + + NYdbWorkload::TWorkloadFactory factory; + auto workloadGen = factory.GetWorkloadQueryGenerator("stock", ¶ms); + if (workloadGen.get() == nullptr) { + throw TMissUseException() << "Invalid path to database." << Endl; + } + + return RunWorkload(workloadGen, static_cast<int>(NYdbWorkload::TStockWorkloadGenerator::EType::GetCustomerHistory)); +} + +} // namespace NYdb::NConsoleClient { diff --git a/ydb/public/lib/ydb_cli/commands/stock_workload.h b/ydb/public/lib/ydb_cli/commands/stock_workload.h new file mode 100644 index 0000000000..f755d63226 --- /dev/null +++ b/ydb/public/lib/ydb_cli/commands/stock_workload.h @@ -0,0 +1,89 @@ +#pragma once + +#include "ydb/public/lib/ydb_cli/commands/ydb_workload.h" + +namespace NYdb { +namespace NConsoleClient { + +class TCommandStock : public TClientCommandTree { +public: + TCommandStock(); +}; + +class TCommandStockInit : public TWorkloadCommand { +public: + TCommandStockInit(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + size_t ProductCount; + size_t Quantity; + size_t OrderCount; + unsigned int MinPartitions; + bool PartitionsByLoad; +}; + +class TCommandStockRun : public TClientCommandTree { +public: + TCommandStockRun(); +}; + +class TCommandStockRunInsertRandomOrder : public TWorkloadCommand { +public: + TCommandStockRunInsertRandomOrder(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + size_t ProductCount; +}; + +class TCommandStockRunSubmitRandomOrder : public TWorkloadCommand { +public: + TCommandStockRunSubmitRandomOrder(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + size_t ProductCount; +}; + +class TCommandStockRunSubmitSameOrder : public TWorkloadCommand { +public: + TCommandStockRunSubmitSameOrder(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + size_t ProductCount; +}; + +class TCommandStockRunGetRandomCustomerHistory : public TWorkloadCommand { +public: + TCommandStockRunGetRandomCustomerHistory(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + unsigned int Limit; +}; + +class TCommandStockRunGetCustomerHistory : public TWorkloadCommand { +public: + TCommandStockRunGetCustomerHistory(); + virtual void Config(TConfig& config) override; + virtual void Parse(TConfig& config) override; + virtual int Run(TConfig& config) override; + +private: + unsigned int Limit; +}; + +} +}
\ No newline at end of file diff --git a/ydb/public/lib/ydb_cli/commands/ya.make b/ydb/public/lib/ydb_cli/commands/ya.make index 27c53fde2f..d453592849 100644 --- a/ydb/public/lib/ydb_cli/commands/ya.make +++ b/ydb/public/lib/ydb_cli/commands/ya.make @@ -3,6 +3,7 @@ LIBRARY(commands) OWNER(g:kikimr) SRCS( + stock_workload.cpp ydb_command.cpp ydb_profile.cpp ydb_root_common.cpp @@ -15,13 +16,17 @@ SRCS( ydb_service_stream.cpp ydb_service_table.cpp ydb_tools.cpp + ydb_workload.cpp ydb_yql.cpp ) PEERDIR( + library/cpp/histogram/hdr library/cpp/protobuf/json library/cpp/regex/pcre + library/cpp/threading/local_executor ydb/library/backup + ydb/library/workload ydb/public/lib/operation_id ydb/public/lib/ydb_cli/common ydb/public/lib/ydb_cli/dump diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp index ea817a3477..2cccf42a41 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp @@ -11,6 +11,8 @@ #include "ydb_tools.h" #include "ydb_yql.h" +#include "ydb_workload.h" + #include <util/folder/path.h> #include <util/folder/dirut.h> #include <util/string/strip.h> @@ -36,6 +38,7 @@ TClientCommandRootCommon::TClientCommandRootCommon(const TClientSettings& settin AddCommand(std::make_unique<TCommandInit>()); AddCommand(std::make_unique<TCommandYql>()); AddCommand(std::make_unique<TCommandStream>()); + AddCommand(std::make_unique<TCommandWorkload>()); } void TClientCommandRootCommon::ValidateSettings() { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp new file mode 100644 index 0000000000..ea57e47500 --- /dev/null +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.cpp @@ -0,0 +1,200 @@ +#include "ydb_workload.h" + +#include "stock_workload.h" + +#include <ydb/library/workload/workload_factory.h> +#include <ydb/public/lib/ydb_cli/commands/ydb_common.h> + +#include <library/cpp/threading/local_executor/local_executor.h> + +#include <atomic> +#include <iomanip> + +namespace NYdb::NConsoleClient { + +struct TWorkloadStats { + ui64 OpsCount; + ui64 Percentile50; + ui64 Percentile95; + ui64 Percentile99; + ui64 Percentile100; +}; + +TWorkloadStats GetWorkloadStats(const NHdr::THistogram& hdr) { + TWorkloadStats stats; + stats.OpsCount = hdr.GetTotalCount(); + stats.Percentile50 = hdr.GetValueAtPercentile(50.0); + stats.Percentile95 = hdr.GetValueAtPercentile(95.0); + stats.Percentile99 = hdr.GetValueAtPercentile(99.0); + stats.Percentile100 = hdr.GetMax(); + return stats; +} + +TCommandWorkload::TCommandWorkload() + : TClientCommandTree("workload", {}, "YDB workload service") +{ + AddCommand(std::make_unique<TCommandStock>()); +} + +TWorkloadCommand::TWorkloadCommand(const TString& name, const std::initializer_list<TString>& aliases, const TString& description) + : TYdbCommand(name, aliases, description) + , Seconds(0) + , Threads(0) + , Quiet(false) + , PrintTimestamp(false) + , WindowHist(1000, 2) // highestTrackableValue 1000ms = 1s, precision 2 + , TotalHist(1000, 2) + , TotalRetries(0) + , WindowRetryCount(0) + , TotalErrors(0) + , WindowErrors(0) +{} + +void TWorkloadCommand::Config(TConfig& config) { + TYdbCommand::Config(config); + + config.Opts->AddLongOption('s', "seconds", "Seconds to run workload.") + .DefaultValue(10).StoreResult(&Seconds); + config.Opts->AddLongOption('t', "threads", "Number of parallel threads in workload.") + .DefaultValue(10).StoreResult(&Threads); + config.Opts->AddLongOption("quiet", "Quiet mode. Doesn't print statistics each second.") + .StoreTrue(&Quiet); + config.Opts->AddLongOption("print-timestamp", "Print timestamp each second with statistics.") + .StoreTrue(&PrintTimestamp); +} + +void TWorkloadCommand::PrepareForRun(TConfig& config) { + auto driverConfig = TDriverConfig() + .SetEndpoint(config.Address) + .SetDatabase(config.Database) + .SetBalancingPolicy(EBalancingPolicy::UseAllNodes) + .SetCredentialsProviderFactory(config.CredentialsGetter(config)); + + if (config.EnableSsl) { + driverConfig.UseSecureConnection(config.CaCerts); + } + Driver = std::make_unique<NYdb::TDriver>(NYdb::TDriver(driverConfig)); + auto tableClientSettings = NTable::TClientSettings() + .SessionPoolSettings( + NTable::TSessionPoolSettings() + .MaxActiveSessions(10+Threads)); + TableClient = std::make_unique<NTable::TTableClient>(*Driver, tableClientSettings); +} + +void TWorkloadCommand::WorkerFn(int taskId, TWorkloadQueryGenPtr workloadGen, const int type) { + auto querySettings = NYdb::NTable::TExecDataQuerySettings() + .KeepInQueryCache(true) + .ClientTimeout(TDuration::Seconds(2)); + int retryCount = -1; + + NYdbWorkload::TQueryInfo queryInfo; + auto runQuery = [&queryInfo, &querySettings, &retryCount] (NYdb::NTable::TSession session) -> NYdb::TStatus { + ++retryCount; + TStatus result(EStatus::SUCCESS, NYql::TIssues()); + result = session.ExecuteDataQuery(queryInfo.Query.c_str(), + NYdb::NTable::TTxControl::BeginTx(NYdb::NTable::TTxSettings::SerializableRW()).CommitTx(), + queryInfo.Params, querySettings + ).GetValueSync(); + return result; + }; + + while (Now() < StopTime) { + auto queryInfoList = workloadGen->GetWorkload(type); + if (queryInfoList.empty()) { + Cerr << "Task ID: " << taskId << ". No queries to run." << Endl; + return; + } + + auto opStartTime = Now(); + NYdbWorkload::TQueryInfoList::iterator it; + for (it = queryInfoList.begin(); it != queryInfoList.end(); ++it) { + queryInfo = *it; + auto status = TableClient->RetryOperationSync(runQuery); + if (!status.IsSuccess()) { + TotalErrors++; + WindowErrors++; + // if (status.GetStatus() != EStatus::ABORTED) { + // Cerr << "Task ID: " << taskId << " Status: " << status.GetStatus() << " " << status.GetIssues().ToString() << Endl; + // } + break; + } + } + if (retryCount > 0) { + TotalRetries += retryCount; + WindowRetryCount += retryCount; + } + retryCount = -1; + if (it != queryInfoList.end()) { + continue; + } + + ui64 latency = (Now() - opStartTime).MilliSeconds(); + with_lock(HdrLock) { + WindowHist.RecordValue(latency); + TotalHist.RecordValue(latency); + } + } + TotalRetries += std::max(retryCount, 0); + WindowRetryCount += std::max(retryCount, 0); +} + +int TWorkloadCommand::RunWorkload(TWorkloadQueryGenPtr workloadGen, const int type) { + if (!Quiet) { + std::cout << "Elapsed\tTxs/Sec\tRetries\tErrors\tp50(ms)\tp95(ms)\tp99(ms)\tpMax(ms)"; + if (PrintTimestamp) { + std::cout << "\tTimestamp"; + } + std::cout << std::endl; + } + + StartTime = Now(); + StopTime = StartTime + TDuration::Seconds(Seconds); + + NPar::LocalExecutor().RunAdditionalThreads(Threads); + auto futures = NPar::LocalExecutor().ExecRangeWithFutures([this, &workloadGen, type](int id) { + WorkerFn(id, workloadGen, type); + }, 0, Threads, NPar::TLocalExecutor::MED_PRIORITY); + + int windowIt = 1; + while (Now() < StopTime) { + if (StartTime + windowIt * WINDOW_DURATION < Now()) { + PrintWindowStats(windowIt++); + } + Sleep(std::max(TDuration::Zero(), Now() - StartTime - windowIt * WINDOW_DURATION)); + } + + for (auto f : futures) { + f.Wait(); + } + + PrintWindowStats(windowIt++); + + auto stats = GetWorkloadStats(TotalHist); + std::cout << std::endl << "Txs\tTxs/Sec\tRetries\tErrors\tp50(ms)\tp95(ms)\tp99(ms)\tpMax(ms)" << std::endl + << stats.OpsCount << "\t" << std::setw(7) << stats.OpsCount / (Seconds * 1.0) << "\t" << TotalRetries.load() << "\t" + << TotalErrors.load() << "\t" << stats.Percentile50 << "\t" << stats.Percentile95 << "\t" + << stats.Percentile99 << "\t" << stats.Percentile100 << std::endl; + + return EXIT_SUCCESS; +} + +void TWorkloadCommand::PrintWindowStats(int windowIt) { + TWorkloadStats stats; + auto retries = WindowRetryCount.exchange(0); + auto errors = WindowErrors.exchange(0); + with_lock(HdrLock) { + stats = GetWorkloadStats(WindowHist); + WindowHist.Reset(); + } + if (!Quiet) { + std::cout << windowIt << "\t" << std::setw(7) << stats.OpsCount / WINDOW_DURATION.Seconds() << "\t" << retries << "\t" + << errors << "\t" << stats.Percentile50 << "\t" << stats.Percentile95 << "\t" + << stats.Percentile99 << "\t" << stats.Percentile100; + if (PrintTimestamp) { + std::cout << "\t" << Now().ToStringUpToSeconds(); + } + std::cout << std::endl; + } +} + +} // namespace NYdb::NConsoleClient diff --git a/ydb/public/lib/ydb_cli/commands/ydb_workload.h b/ydb/public/lib/ydb_cli/commands/ydb_workload.h new file mode 100644 index 0000000000..ddcb27a35c --- /dev/null +++ b/ydb/public/lib/ydb_cli/commands/ydb_workload.h @@ -0,0 +1,70 @@ +#pragma once + +#include <ydb/public/lib/ydb_cli/commands/ydb_command.h> + +#include <library/cpp/histogram/hdr/histogram.h> +#include <util/datetime/base.h> +#include <util/system/spinlock.h> + +#include <memory> +#include <string> + +namespace NYdbWorkload { + class IWorkloadQueryGenerator; +} + +namespace NYdb { +namespace NConsoleClient { + +class TCommandWorkload : public TClientCommandTree { +public: + TCommandWorkload(); +}; + +class TWorkloadCommand : public TYdbCommand { +public: + TWorkloadCommand( + const TString& name, + const std::initializer_list<TString>& aliases = std::initializer_list<TString>(), + const TString& description = TString() + ); + + virtual void Config(TConfig& config) override; + NTable::TSession GetSession(); + +protected: + using TWorkloadQueryGenPtr = std::shared_ptr<NYdbWorkload::IWorkloadQueryGenerator>; + + void PrepareForRun(TConfig& config); + + int RunWorkload(TWorkloadQueryGenPtr workloadGen, const int type); + void WorkerFn(int taskId, TWorkloadQueryGenPtr workloadGen, const int type); + void PrintWindowStats(int windowIt); + + static constexpr TDuration WINDOW_DURATION = TDuration::Seconds(1); + + std::unique_ptr<NYdb::TDriver> Driver; + std::unique_ptr<NTable::TTableClient> TableClient; + + size_t Seconds; + size_t Threads; + bool Quiet; + bool PrintTimestamp; + + TInstant StartTime; + TInstant StopTime; + + // Think about moving histograms to workload library. + // Histograms will also be useful in actor system workload. + TSpinLock HdrLock; + NHdr::THistogram WindowHist; + NHdr::THistogram TotalHist; + + std::atomic_uint64_t TotalRetries; + std::atomic_uint64_t WindowRetryCount; + std::atomic_uint64_t TotalErrors; + std::atomic_uint64_t WindowErrors; +}; + +} +} |