diff options
213 files changed, 34883 insertions, 4509 deletions
diff --git a/build/conf/license.conf b/build/conf/license.conf index ffb01e93f91..819f64216e0 100644 --- a/build/conf/license.conf +++ b/build/conf/license.conf @@ -253,6 +253,7 @@ LICENSES_REQUIRE_CITATION= \ BSD-3-Clause \ BSD-3-Clause WITH Google-Patent-License-Webm \ BSD-3-Clause WITH Google-Patent-License-Webrtc \ + BSD-3-Clause-LBNL \ BSD-4-Clause \ BSD-4-Clause-UC \ BSD-derived \ diff --git a/build/conf/opensource_export.conf b/build/conf/opensource_export.conf new file mode 100644 index 00000000000..3437b7efd8b --- /dev/null +++ b/build/conf/opensource_export.conf @@ -0,0 +1,20 @@ +when ($OPENSOURCE_PROJECT) { + select ($OPENSOURCE_PROJECT) { + "catboost" ? { + OPENSOURCE_REPLACE_OPENSSL="1.1.1t" + } + "yt" ? { + OPENSOURCE_REPLACE_LINUX_HEADERS="6.5.9" + } + "yt-cpp-sdk" ? { + USE_STL_SYSTEM=yes + USE_INTERNAL_STL=no + OS_SDK=ubuntu-20 + TSTRING_IS_STD_STRING=yes + _SEM_EXTRA_CXX_FLAGS=-DNO_CUSTOM_CHAR_PTR_STD_COMPARATOR + OPENSOURCE_REPLACE_PROTOBUF="3.19.1" + OPENSOURCE_REPLACE_LINUX_HEADERS="6.5.9" + # OPENSOURCE_REPLACE_OPENSSL="[>=3.2.0]" + } + } +} diff --git a/build/conf/proto.conf b/build/conf/proto.conf index 486b32f1063..1028a8c79ae 100644 --- a/build/conf/proto.conf +++ b/build/conf/proto.conf @@ -5,9 +5,18 @@ NEW_JAVA_PROTOC=yes when ($NEW_JAVA_PROTOC == "yes") { JAVA_PROTOC=${tool:"build/platform/java/protoc"} } -PROTOC_STYLEGUIDE_OUT=--cpp_styleguide_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE -PROTOC_PLUGIN_STYLEGUIDE=--plugin=protoc-gen-cpp_styleguide=${tool:"contrib/tools/protoc/plugins/cpp_styleguide"} -PROTOBUF_PATH=${ARCADIA_ROOT}/contrib/libs/protobuf/src +when ($OPENSOURCE_REPLACE_PROTOBUF) { + PROTOC_STYLEGUIDE_OUT= + PROTOC_PLUGIN_STYLEGUIDE= + PROTOBUF_PATH="${protobuf_INCLUDE_DIRS}" + DEFAULT_PROTOC_TOOLS= +} +otherwise { + PROTOC_STYLEGUIDE_OUT=--cpp_styleguide_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE + PROTOC_PLUGIN_STYLEGUIDE=--plugin=protoc-gen-cpp_styleguide=${tool:"contrib/tools/protoc/plugins/cpp_styleguide"} + PROTOBUF_PATH=${ARCADIA_ROOT}/contrib/libs/protobuf/src + DEFAULT_PROTOC_TOOLS=${hide;tool:"contrib/tools/protoc/bin"} ${hide;tool:"contrib/tools/protoc/plugins/cpp_styleguide"} +} USE_VANILLA_PROTOC=no PROTOC_TRANSITIVE_HEADERS=yes _PROTOC_FLAGS= @@ -352,7 +361,7 @@ GO_PROTO_CMDLINE=${cwd;rootdir;input:File} $YMAKE_PYTHON3 ${input:"build/scripts macro _GO_PROTO_CMD_IMPL(File, OPTS...) { .CMD=$GO_PROTO_CMDLINE $OPTS ${kv;hide:"p PB"} ${kv;hide:"pc yellow"} .PEERDIR=${GO_PROTOBUF_IMPORTS} ${GO_PROTOBUF_WELLKNOWN_TYPES} - .ADDINCL=FOR proto contrib/libs/protobuf/src + .ADDINCL=FOR proto ${PROTOBUF_PATH} } # tag:go-specific tag:proto @@ -366,12 +375,12 @@ macro _GO_PROTO_CMD(File) { ### Generate .yson.go from .proto using yt/yt/orm/go/codegen/yson/internal/proto-yson-gen/cmd/proto-yson-gen macro YT_ORM_PROTO_YSON(OUT_OPTS[], Files...) { .CMD=${cwd:BINDIR} $PROTOC --plugin=protoc-gen-custom=${tool:"yt/yt/orm/go/codegen/yson/internal/proto-yson-gen/cmd/proto-yson-gen"} -I=${ARCADIA_ROOT}/${PROTO_NAMESPACE} ${pre=-I=:_PROTO__INCLUDE} -I=${ARCADIA_ROOT} --custom_out="$OUT_OPTS paths=base_name:." --custom_opt="goroot=${GO_TOOLS_ROOT}" $_PROTOC_FLAGS ${input:Files} ${output;hide;noauto;nopath;noext;suf=.yson.go:Files} ${hide:PROTO_FAKEID} - .ADDINCL=FOR proto ${ARCADIA_ROOT}/${MODDIR} FOR proto ${ARCADIA_ROOT}/${GO_TEST_IMPORT_PATH} FOR proto yt ${ARCADIA_BUILD_ROOT}/yt FOR proto contrib/libs/protobuf/src + .ADDINCL=FOR proto ${ARCADIA_ROOT}/${MODDIR} FOR proto ${ARCADIA_ROOT}/${GO_TEST_IMPORT_PATH} FOR proto yt ${ARCADIA_BUILD_ROOT}/yt FOR proto ${PROTOBUF_PATH} .PEERDIR=$GOSTD/strings $GOSTD/fmt $GOSTD/errors $GOSTD/encoding/json library/go/core/xerrors yt/go/yson yt/go/yterrors yt/yt/orm/go/codegen/yson/ytypes contrib/libs/protobuf } -_SEM_CPP_PROTO_CMD=target_proto_messages PRIVATE ${input:File} $CPP_PROTO_OUTS_SEM ${output;hide;suf=${OBJ_SUF}.pb.o:File} ${hide;tool:"contrib/tools/protoc/bin"} ${hide;tool:"contrib/tools/protoc/plugins/cpp_styleguide"} \ +_SEM_CPP_PROTO_CMD=target_proto_messages PRIVATE ${input:File} $CPP_PROTO_OUTS_SEM ${output;hide;suf=${OBJ_SUF}.pb.o:File} $DEFAULT_PROTOC_TOOLS \ && set_global_flags COMMON_PROTOC_FLAGS \ && target_proto_outs --cpp_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE $PROTOC_STYLEGUIDE_OUT \ && target_proto_addincls ./$PROTO_NAMESPACE $ARCADIA_ROOT/$PROTO_NAMESPACE ${_PROTO__INCLUDE} $ARCADIA_BUILD_ROOT $PROTOBUF_PATH @@ -389,7 +398,7 @@ macro _CPP_VANILLA_PROTO_CMD(File) { .PEERDIR=contrib/libs/protobuf_std } -_SEM_CPP_EV_CMD=target_ev_messages PRIVATE ${input:File} $CPP_EV_OUTS_SEM ${output;hide;suf=${OBJ_SUF}.pb.o:File} ${hide;tool:"tools/event2cpp/bin"} ${hide;tool:"contrib/tools/protoc/bin"} ${hide;tool:"contrib/tools/protoc/plugins/cpp_styleguide"} \ +_SEM_CPP_EV_CMD=target_ev_messages PRIVATE ${input:File} $CPP_EV_OUTS_SEM ${output;hide;suf=${OBJ_SUF}.pb.o:File} ${hide;tool:"tools/event2cpp/bin"} $DEFAULT_PROTOC_TOOLS \ && set_global_flags COMMON_PROTOC_FLAGS \ && target_proto_outs --cpp_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE $PROTOC_STYLEGUIDE_OUT --event2cpp_out=$ARCADIA_BUILD_ROOT/$PROTO_NAMESPACE \ && target_proto_addincls ./$PROTO_NAMESPACE $ARCADIA_ROOT/$PROTO_NAMESPACE ${_PROTO__INCLUDE} $ARCADIA_BUILD_ROOT $PROTOBUF_PATH $ARCADIA_ROOT/library/cpp/eventlog diff --git a/build/conf/python.conf b/build/conf/python.conf index 72bcec2db7c..2df0e3e47e3 100644 --- a/build/conf/python.conf +++ b/build/conf/python.conf @@ -551,7 +551,12 @@ module PY2_LIBRARY: _LIBRARY { PY_PROTO_MYPY_PLUGIN_INTERNAL=$PY_PROTO_MYPY_PLUGIN_BASE ${output;hide;noauto;norel;nopath;noext;suf=__intpy2___pb2.pyi:File} ${hide;kv:"ext_out_name_for_${nopath;noext;suf=__intpy2___pb2.pyi:File} ${nopath;noext;suf=_pb2.pyi:File}"}) } SET(MODULE_LANG PY2) + ADD_CLANG_TIDY() + when ($TIDY_ENABLED == "yes") { + _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_project_map.json + _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_default_map.json + } } @@ -592,7 +597,12 @@ module PY3_LIBRARY: _LIBRARY { PY_PROTO_MYPY_PLUGIN_INTERNAL=$PY_PROTO_MYPY_PLUGIN_BASE ${output;hide;noauto;norel;nopath;noext;suf=__intpy3___pb2.pyi:File} ${hide;kv:"ext_out_name_for_${nopath;noext;suf=__intpy3___pb2.pyi:File} ${nopath;noext;suf=_pb2.pyi:File}"}) } SET(MODULE_LANG PY3) + ADD_CLANG_TIDY() + when ($TIDY_ENABLED == "yes") { + _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_project_map.json + _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_default_map.json + } } diff --git a/build/config/tests/cpp_style/config.clang-format b/build/config/tests/cpp_style/config.clang-format index c4bedb3f4e8..3a354b63b4a 100644 --- a/build/config/tests/cpp_style/config.clang-format +++ b/build/config/tests/cpp_style/config.clang-format @@ -3,7 +3,9 @@ AccessModifierOffset: -4 ConstructorInitializerIndentWidth: 4 AlignEscapedNewlinesLeft: false AlignEscapedNewlines: Left -AlignTrailingComments: true +AlignTrailingComments: + Kind: Always + OverEmptyLines: 0 AllowAllParametersOfDeclarationOnNextLine: true AllowShortBlocksOnASingleLine: false AllowShortIfStatementsOnASingleLine: false diff --git a/build/config/tests/cpp_style/config.clang-format-16 b/build/config/tests/cpp_style/config.clang-format-16 deleted file mode 100644 index 3a354b63b4a..00000000000 --- a/build/config/tests/cpp_style/config.clang-format-16 +++ /dev/null @@ -1,90 +0,0 @@ -Language: Cpp -AccessModifierOffset: -4 -ConstructorInitializerIndentWidth: 4 -AlignEscapedNewlinesLeft: false -AlignEscapedNewlines: Left -AlignTrailingComments: - Kind: Always - OverEmptyLines: 0 -AllowAllParametersOfDeclarationOnNextLine: true -AllowShortBlocksOnASingleLine: false -AllowShortIfStatementsOnASingleLine: false -AllowShortLoopsOnASingleLine: false -AllowShortFunctionsOnASingleLine: None -AlwaysBreakTemplateDeclarations: true -AlwaysBreakBeforeMultilineStrings: false -BreakBeforeBinaryOperators: false -BreakBeforeTernaryOperators: true -BreakConstructorInitializersBeforeComma: true -BinPackParameters: true -ColumnLimit: 0 -ConstructorInitializerAllOnOneLineOrOnePerLine: false -DerivePointerAlignment: false -ExperimentalAutoDetectBinPacking: false -IndentCaseLabels: true -IndentWrappedFunctionNames: false -IndentFunctionDeclarationAfterType: false -MaxEmptyLinesToKeep: 1 -KeepEmptyLinesAtTheStartOfBlocks: false -NamespaceIndentation: All -ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: true -PenaltyBreakBeforeFirstCallParameter: 19 -PenaltyBreakComment: 300 -PenaltyBreakString: 1000 -PenaltyBreakFirstLessLess: 120 -PenaltyExcessCharacter: 1000000 -PenaltyReturnTypeOnItsOwnLine: 60 -PointerAlignment: Left -SpacesBeforeTrailingComments: 1 -Cpp11BracedListStyle: true -Standard: c++20 -IndentWidth: 4 -TabWidth: 4 -UseTab: Never -BreakBeforeBraces: Attach - -BraceWrapping: - AfterClass: false - AfterControlStatement: false - AfterEnum: false - AfterFunction: false - AfterNamespace: false - AfterObjCDeclaration: false - AfterStruct: false - AfterUnion: false - BeforeCatch: false - BeforeElse: false - IndentBraces: true - -SpacesInParentheses: false -SpacesInAngles: false -SpaceInEmptyParentheses: false -SpacesInCStyleCastParentheses: false -SpacesInContainerLiterals: true -SpaceBeforeAssignmentOperators: true -ContinuationIndentWidth: 4 -CommentPragmas: '^ IWYU pragma:' -ForEachMacros: [ - foreach, - Q_FOREACH, - BOOST_FOREACH, - # Is not a foreach, but correct formatting is achieved - Y_DEFER -] -IfMacros: [ - with_lock # Is not an if, but correct formatting is achieved -] -SpaceBeforeParens: ControlStatements -DisableFormat: false -SortIncludes: false -IndentPPDirectives: BeforeHash - -NamespaceMacros: [Y_UNIT_TEST_SUITE, Y_UNIT_TEST] -SpaceBeforeInheritanceColon: false -AttributeMacros: [Y_PRINTF_FORMAT, Y_NO_SANITIZE, Y_FORCE_INLINE, Y_NO_INLINE, Y_WARN_UNUSED_RESULT, Y_HIDDEN, Y_PUBLIC, Y_PURE_FUNCTION] -IndentExternBlock: Indent -TypenameMacros: [Y_THREAD, Y_STATIC_THREAD, Y_POD_THREAD, Y_POD_STATIC_THREAD] -SpacesInLineCommentPrefix: - Minimum: 0 - Maximum: -1 diff --git a/build/config/tests/cpp_style/ya.make b/build/config/tests/cpp_style/ya.make index 6d2fab5a83a..bb5fccd5ea4 100644 --- a/build/config/tests/cpp_style/ya.make +++ b/build/config/tests/cpp_style/ya.make @@ -2,7 +2,6 @@ LIBRARY() RESOURCE_FILES( config.clang-format - config.clang-format-16 ) END() diff --git a/build/export_generators/hardcoded-cmake/disclaimer.jinja b/build/export_generators/hardcoded-cmake/disclaimer.jinja index d48958a576c..f7c762d5e53 100644 --- a/build/export_generators/hardcoded-cmake/disclaimer.jinja +++ b/build/export_generators/hardcoded-cmake/disclaimer.jinja @@ -1,5 +1,13 @@ -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. +# This file was generated by the build system used internally in the Yandex and called "ya" +# (https://github.com/yandex/yatool). +# +# Configuration files of ya build system usually named as ya.make. If ya.make file is presented +# at the root of the repository, then this repository supports ya build. +# +# If the repository supports both CMake and ya build configuration, please modify both of them. +# +# If only CMake build configuration is supported then modify only CMake files and note that only +# simple modifications are allowed like adding source-files to targets or adding simple properties +# like target_include_directories. These modifications will be ported to original ya.make files +# by maintainers. Any complex modifications which can't be easily ported back to the ya build +# system may be rejected. diff --git a/build/external_resources/gdb/a.yaml b/build/external_resources/gdb/a.yaml index d30bcedeb3a..cb106731b74 100644 --- a/build/external_resources/gdb/a.yaml +++ b/build/external_resources/gdb/a.yaml @@ -39,6 +39,7 @@ ci: platforms: title: Build only for particular platforms (comma separated) type: string + default: "" filters: - discovery: dir abs-paths: diff --git a/build/mapping.conf.json b/build/mapping.conf.json index d88e3876cc4..ab40627f4de 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -99,8 +99,13 @@ "4111727026": "https://devtools-registry.s3.yandex.net/4111727026", "4111727778": "https://devtools-registry.s3.yandex.net/4111727778", "5950103181": "https://devtools-registry.s3.yandex.net/5950103181", + "6113423937": "https://devtools-registry.s3.yandex.net/6113423937", "5950102802": "https://devtools-registry.s3.yandex.net/5950102802", + "6113422956": "https://devtools-registry.s3.yandex.net/6113422956", "5950104171": "https://devtools-registry.s3.yandex.net/5950104171", + "6113425489": "https://devtools-registry.s3.yandex.net/6113425489", + "6113421330": "https://devtools-registry.s3.yandex.net/6113421330", + "6113424629": "https://devtools-registry.s3.yandex.net/6113424629", "5599878022": "https://devtools-registry.s3.yandex.net/5599878022", "5599877541": "https://devtools-registry.s3.yandex.net/5599877541", "5599878769": "https://devtools-registry.s3.yandex.net/5599878769", @@ -111,6 +116,7 @@ "5786828167": "https://devtools-registry.s3.yandex.net/5786828167", "5786827891": "https://devtools-registry.s3.yandex.net/5786827891", "5786826302": "https://devtools-registry.s3.yandex.net/5786826302", + "6115438948": "https://devtools-registry.s3.yandex.net/6115438948", "2842390994": "https://devtools-registry.s3.yandex.net/2842390994", "5310288728": "https://devtools-registry.s3.yandex.net/5310288728", "5620327787": "https://devtools-registry.s3.yandex.net/5620327787", @@ -515,8 +521,13 @@ "4111727026": "contrib/libs/clang14/tools/clang-format/clang-format for darwin", "4111727778": "contrib/libs/clang14/tools/clang-format/clang-format for linux", "5950103181": "contrib/libs/clang16/tools/clang-format/clang-format for darwin", + "6113423937": "contrib/libs/clang16/tools/clang-format/clang-format for darwin", "5950102802": "contrib/libs/clang16/tools/clang-format/clang-format for darwin-arm64", + "6113422956": "contrib/libs/clang16/tools/clang-format/clang-format for darwin-arm64", "5950104171": "contrib/libs/clang16/tools/clang-format/clang-format for linux", + "6113425489": "contrib/libs/clang16/tools/clang-format/clang-format for linux", + "6113421330": "contrib/libs/clang16/tools/clang-format/clang-format for linux-aarch64", + "6113424629": "contrib/libs/clang16/tools/clang-format/clang-format for win32", "5599878022": "contrib/python/black/bin/black for darwin", "5599877541": "contrib/python/black/bin/black for darwin-arm64", "5599878769": "contrib/python/black/bin/black for linux", @@ -527,6 +538,7 @@ "5786828167": "contrib/tools/python3/python3 for linux", "5786827891": "contrib/tools/python3/python3 for linux-aarch64", "5786826302": "contrib/tools/python3/python3 for win32", + "6115438948": "devtools jacoco agent 0.8.7", "2842390994": "devtools jacoco agent 0.8.7 with shaded asm", "5310288728": "devtools jstyle runner 10.12.4", "5620327787": "devtools jstyle runner 10.12.4", diff --git a/build/sysincl/intrinsic.yml b/build/sysincl/intrinsic.yml index 622c5003f17..160e09a48b2 100644 --- a/build/sysincl/intrinsic.yml +++ b/build/sysincl/intrinsic.yml @@ -5,6 +5,7 @@ - arm_acle.h - arm_fp16.h - arm_neon.h + - arm_sve.h - arm64_neon.h - armintr.h - avx2intrin.h diff --git a/build/ymake.core.conf b/build/ymake.core.conf index fd1bb1e8c43..18b7851a4fc 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -71,6 +71,7 @@ when ($LOCAL && $XCODE) { @import "${CONF_ROOT}/conf/settings.conf" @import "${CONF_ROOT}/conf/opensource.conf" +@import "${CONF_ROOT}/conf/opensource_export.conf" @import "${CONF_ROOT}/conf/sysincl.conf" @import "${CONF_ROOT}/conf/license.conf" @import "${CONF_ROOT}/conf/docs.conf" @@ -1117,10 +1118,6 @@ MODULE_TYPE=UNKNOWN macro ADD_CLANG_TIDY() { ADD_YTEST($MODULE_PREFIX$REALPRJNAME clang_tidy) - when ($TIDY_ENABLED == "yes") { - _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_project_map.json - _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_default_map.json - } } # tag:internal @@ -1229,7 +1226,14 @@ module PROGRAM: _BASE_PROGRAM { .ALIASES=EXTRALIBS=PY_EXTRALIBS ADD_YTEST($MODULE_PREFIX$REALPRJNAME coverage.extractor) + ADD_CLANG_TIDY() + + when ($TIDY_ENABLED == "yes") { + _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_project_map.json + _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_default_map.json + } + _DONT_REQUIRE_LICENSE() } @@ -1890,6 +1894,10 @@ module LIBRARY: _LIBRARY { SET(MODULE_LANG CPP) ADD_CLANG_TIDY() + when ($TIDY_ENABLED == "yes") { + _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_project_map.json + _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_default_map.json + } } # tag:internal @@ -2081,7 +2089,12 @@ module DLL_UNIT: _LINK_UNIT { SET(MODULE_TYPE DLL) SET(MODULE_TAG DLL) SET(MODULE_LANG CPP) + ADD_CLANG_TIDY() + when ($TIDY_ENABLED == "yes") { + _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_project_map.json + _MAKEFILE_INCLUDE_LIKE_DEPS+=${ARCADIA_ROOT}/build/yandex_specific/config/clang_tidy/tidy_default_map.json + } when ($EXPORTS_FILE) { _EXPORT_SCRIPT_SEM=&& use_export_script ${input:EXPORTS_FILE} diff --git a/contrib/libs/apache/arrow/cpp/src/arrow/python/numpy_internal.h b/contrib/libs/apache/arrow/cpp/src/arrow/python/numpy_internal.h index 973f577cb13..50d1a0fcb75 100644 --- a/contrib/libs/apache/arrow/cpp/src/arrow/python/numpy_internal.h +++ b/contrib/libs/apache/arrow/cpp/src/arrow/python/numpy_internal.h @@ -43,12 +43,11 @@ class Ndarray1DIndexer { explicit Ndarray1DIndexer(PyArrayObject* arr) : Ndarray1DIndexer() { arr_ = arr; DCHECK_EQ(1, PyArray_NDIM(arr)) << "Only works with 1-dimensional arrays"; - Py_INCREF(arr); data_ = reinterpret_cast<uint8_t*>(PyArray_DATA(arr)); stride_ = PyArray_STRIDES(arr)[0]; } - ~Ndarray1DIndexer() { Py_XDECREF(arr_); } + ~Ndarray1DIndexer() = default; int64_t size() const { return PyArray_SIZE(arr_); } diff --git a/contrib/libs/croaring/AUTHORS b/contrib/libs/croaring/AUTHORS new file mode 100644 index 00000000000..a53c36d5942 --- /dev/null +++ b/contrib/libs/croaring/AUTHORS @@ -0,0 +1,15 @@ +# List of authors for copyright purposes + +Tom Cornebize +Luca Deri +Jacob Evans +Owen Kaser +Nathan Kurz +Daniel Lemire +Wojciech Muła +Chris O'Hara +François Saint-Jacques +Gregory Ssi-Yan-Kai +Andrei Gudkov +Guillaume Holley +Brian Esch diff --git a/contrib/libs/croaring/LICENSE b/contrib/libs/croaring/LICENSE new file mode 100644 index 00000000000..8b0ad80d743 --- /dev/null +++ b/contrib/libs/croaring/LICENSE @@ -0,0 +1,236 @@ +The CRoaring project is under a dual license (Apache/MIT). +Users of the library may choose one or the other license. + +------------------ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2022 The CRoaring authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +----------------------------------- + +MIT License + +Copyright 2016-2022 The CRoaring authors + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/contrib/libs/croaring/README.md b/contrib/libs/croaring/README.md new file mode 100644 index 00000000000..7f620eb6ca2 --- /dev/null +++ b/contrib/libs/croaring/README.md @@ -0,0 +1,896 @@ +# CRoaring + +[](https://github.com/RoaringBitmap/CRoaring/actions/workflows/ubuntu-noexcept-ci.yml) [](https://github.com/RoaringBitmap/CRoaring/actions/workflows/vs17-ci.yml) +[](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:croaring) + +[](http://roaringbitmap.github.io/CRoaring/) + + + +Portable Roaring bitmaps in C (and C++) with full support for your favorite compiler (GNU GCC, LLVM's clang, Visual Studio, Apple Xcode, Intel oneAPI). Included in the [Awesome C](https://github.com/kozross/awesome-c) list of open source C software. + +# Introduction + +Bitsets, also called bitmaps, are commonly used as fast data structures. Unfortunately, they can use too much memory. + To compensate, we often use compressed bitmaps. + +Roaring bitmaps are compressed bitmaps which tend to outperform conventional compressed bitmaps such as WAH, EWAH or Concise. +They are used by several major systems such as [Apache Lucene][lucene] and derivative systems such as [Solr][solr] and +[Elasticsearch][elasticsearch], [Metamarkets' Druid][druid], [LinkedIn Pinot][pinot], [Netflix Atlas][atlas], [Apache Spark][spark], [OpenSearchServer][opensearchserver], [Cloud Torrent][cloudtorrent], [Whoosh][whoosh], [InfluxDB](https://www.influxdata.com), [Pilosa][pilosa], [Bleve](http://www.blevesearch.com), [Microsoft Visual Studio Team Services (VSTS)][vsts], and eBay's [Apache Kylin][kylin]. The CRoaring library is used in several systems such as [Apache Doris](http://doris.incubator.apache.org), [ClickHouse](https://github.com/ClickHouse/ClickHouse), and [StarRocks](https://github.com/StarRocks/starrocks). The YouTube SQL Engine, [Google Procella](https://research.google/pubs/pub48388/), uses Roaring bitmaps for indexing. + +We published a peer-reviewed article on the design and evaluation of this library: + +- Roaring Bitmaps: Implementation of an Optimized Software Library, Software: Practice and Experience 48 (4), 2018 [arXiv:1709.07821](https://arxiv.org/abs/1709.07821) + +[lucene]: https://lucene.apache.org/ +[solr]: https://lucene.apache.org/solr/ +[elasticsearch]: https://www.elastic.co/products/elasticsearch +[druid]: http://druid.io/ +[spark]: https://spark.apache.org/ +[opensearchserver]: http://www.opensearchserver.com +[cloudtorrent]: https://github.com/jpillora/cloud-torrent +[whoosh]: https://bitbucket.org/mchaput/whoosh/wiki/Home +[pilosa]: https://www.pilosa.com/ +[kylin]: http://kylin.apache.org/ +[pinot]: http://github.com/linkedin/pinot/wiki +[vsts]: https://www.visualstudio.com/team-services/ +[atlas]: https://github.com/Netflix/atlas + +Roaring bitmaps are found to work well in many important applications: + +> Use Roaring for bitmap compression whenever possible. Do not use other bitmap compression methods ([Wang et al., SIGMOD 2017](http://db.ucsd.edu/wp-content/uploads/2017/03/sidm338-wangA.pdf)) + + +[There is a serialized format specification for interoperability between implementations](https://github.com/RoaringBitmap/RoaringFormatSpec/). Hence, it is possible to serialize a Roaring Bitmap from C++, read it in Java, modify it, serialize it back and read it in Go and Python. + +# Objective + +The primary goal of the CRoaring is to provide a high performance low-level implementation that fully take advantage +of the latest hardware. Roaring bitmaps are already available on a variety of platform through Java, Go, Rust... implementations. CRoaring is a library that seeks to achieve superior performance by staying close to the latest hardware. + + +(c) 2016-... The CRoaring authors. + + + +# Requirements + +- Linux, macOS, FreeBSD, Windows (MSYS2 and Microsoft Visual studio). +- We test the library with ARM, x64/x86 and POWER processors. We only support little endian systems (big endian systems are vanishingly rare). +- Recent C compiler supporting the C11 standard (GCC 7 or better, LLVM 7.0 or better, Xcode 11 or better, Microsoft Visual Studio 2022 or better, Intel oneAPI Compiler 2023.2 or better), there is also an optional C++ class that requires a C++ compiler supporting the C++11 standard. +- CMake (to contribute to the project, users can rely on amalgamation/unity builds if they do not wish to use CMake). +- The CMake system assumes that git is available. +- Under x64 systems, the library provides runtime dispatch so that optimized functions are called based on the detected CPU features. It works with GCC, clang (version 9 and up) and Visual Studio (2017 and up). Other systems (e.g., ARM) do not need runtime dispatch. + +Hardly anyone has access to an actual big-endian system. Nevertheless, +We support big-endian systems such as IBM s390x through emulators---except for +IO serialization which is only supported on little-endian systems (see [issue 423](https://github.com/RoaringBitmap/CRoaring/issues/423)). + + +# Quick Start + +The CRoaring library can be amalgamated into a single source file that makes it easier +for integration into other projects. Moreover, by making it possible to compile +all the critical code into one compilation unit, it can improve the performance. For +the rationale, please see the [SQLite documentation](https://www.sqlite.org/amalgamation.html), +or the corresponding [Wikipedia entry](https://en.wikipedia.org/wiki/Single_Compilation_Unit). +Users who choose this route, do not need to rely on CRoaring's build system (based on CMake). + +We offer amalgamated files as part of each release. + +Linux or macOS users might follow the following instructions if they have a recent C or C++ compiler installed and a standard utility (`wget`). + + + 1. Pull the library in a directory + ``` + wget https://github.com/RoaringBitmap/CRoaring/releases/download/v2.1.0/roaring.c + wget https://github.com/RoaringBitmap/CRoaring/releases/download/v2.1.0/roaring.h + wget https://github.com/RoaringBitmap/CRoaring/releases/download/v2.1.0/roaring.hh + ``` + 2. Create a new file named `demo.c` with this content: + ```C + #include <stdio.h> + #include <stdlib.h> + #include "roaring.c" + int main() { + roaring_bitmap_t *r1 = roaring_bitmap_create(); + for (uint32_t i = 100; i < 1000; i++) roaring_bitmap_add(r1, i); + printf("cardinality = %d\n", (int) roaring_bitmap_get_cardinality(r1)); + roaring_bitmap_free(r1); + + bitset_t *b = bitset_create(); + for (int k = 0; k < 1000; ++k) { + bitset_set(b, 3 * k); + } + printf("%zu \n", bitset_count(b)); + bitset_free(b); + return EXIT_SUCCESS; + } + ``` + 2. Create a new file named `demo.cpp` with this content: + ```C++ + #include <iostream> + #include "roaring.hh" // the amalgamated roaring.hh includes roaring64map.hh + #include "roaring.c" + int main() { + roaring::Roaring r1; + for (uint32_t i = 100; i < 1000; i++) { + r1.add(i); + } + std::cout << "cardinality = " << r1.cardinality() << std::endl; + + roaring::Roaring64Map r2; + for (uint64_t i = 18000000000000000100ull; i < 18000000000000001000ull; i++) { + r2.add(i); + } + std::cout << "cardinality = " << r2.cardinality() << std::endl; + return 0; + } + ``` + 2. Compile + ``` + cc -o demo demo.c + c++ -std=c++11 -o demopp demo.cpp + ``` + 3. `./demo` + ``` + cardinality = 900 + 1000 + ``` + 4. `./demopp` + ``` + cardinality = 900 + cardinality = 900 + ``` + + +# Using Roaring as a CPM dependency + + +If you like CMake and CPM, you can just a few lines in you `CMakeLists.txt` file to grab a `CRoaring` release. [See our CPM demonstration for further details](https://github.com/RoaringBitmap/CPMdemo). + + + +```CMake +cmake_minimum_required(VERSION 3.10) +project(roaring_demo + LANGUAGES CXX C +) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_C_STANDARD 11) + +add_executable(hello hello.cpp) +# You can add CPM.cmake like so: +# mkdir -p cmake +# wget -O cmake/CPM.cmake https://github.com/cpm-cmake/CPM.cmake/releases/latest/download/get_cpm.cmake +include(cmake/CPM.cmake) +CPMAddPackage( + NAME roaring + GITHUB_REPOSITORY "RoaringBitmap/CRoaring" + GIT_TAG v2.0.4 + OPTIONS "BUILD_TESTING OFF" +) + +target_link_libraries(hello roaring::roaring) +``` + + +# Using as a CMake dependency with FetchContent + +If you like CMake, you can just a few lines in you `CMakeLists.txt` file to grab a `CRoaring` release. [See our demonstration for further details](https://github.com/RoaringBitmap/croaring_cmake_demo_single_file). + +If you installed the CRoaring library locally, you may use it with CMake's `find_package` function as in this example: + +```CMake +cmake_minimum_required(VERSION 3.15) + +project(test_roaring_install VERSION 0.1.0 LANGUAGES CXX C) + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + + +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) + +find_package(roaring REQUIRED) + +file(WRITE main.cpp " +#include <iostream> +#include \"roaring/roaring.hh\" +int main() { + roaring::Roaring r1; + for (uint32_t i = 100; i < 1000; i++) { + r1.add(i); + } + std::cout << \"cardinality = \" << r1.cardinality() << std::endl; + return 0; +}") + +add_executable(repro main.cpp) +target_link_libraries(repro PUBLIC roaring::roaring) +``` + + +# Amalgamating + +To generate the amalgamated files yourself, you can invoke a bash script... + +```bash +./amalgamation.sh +``` + +If you prefer a silent output, you can use the following command to redirect ``stdout`` : + +```bash +./amalgamation.sh > /dev/null +``` + + +(Bash shells are standard under Linux and macOS. Bash shells are available under Windows as part of the [GitHub Desktop](https://desktop.github.com/) under the name ``Git Shell``. So if you have cloned the ``CRoaring`` GitHub repository from within the GitHub Desktop, you can right-click on ``CRoaring``, select ``Git Shell`` and then enter the above commands.) + +It is not necessary to invoke the script in the CRoaring directory. You can invoke +it from any directory where you want the amalgamation files to be written. + +It will generate three files for C users: ``roaring.h``, ``roaring.c`` and ``amalgamation_demo.c``... as well as some brief instructions. The ``amalgamation_demo.c`` file is a short example, whereas ``roaring.h`` and ``roaring.c`` are "amalgamated" files (including all source and header files for the project). This means that you can simply copy the files ``roaring.h`` and ``roaring.c`` into your project and be ready to go! No need to produce a library! See the ``amalgamation_demo.c`` file. + +# API + +The C interface is found in the file ``include/roaring/roaring.h``. We have C++ interface at `cpp/roaring.hh`. + +# Dealing with large volumes + +Some users have to deal with large volumes of data. It may be important for these users to be aware of the `addMany` (C++) `roaring_bitmap_or_many` (C) functions as it is much faster and economical to add values in batches when possible. Furthermore, calling periodically the `runOptimize` (C++) or `roaring_bitmap_run_optimize` (C) functions may help. + + +# Running microbenchmarks + +We have microbenchmarks constructed with the Google Benchmarks. +Under Linux or macOS, you may run them as follows: + +``` +cmake -B build +cmake --build build +./build/microbenchmarks/bench +``` + +By default, the benchmark tools picks one data set (e.g., `CRoaring/benchmarks/realdata/census1881`). +We have several data sets and you may pick others: + +``` +./build/microbenchmarks/bench benchmarks/realdata/wikileaks-noquotes +``` + +You may disable some functionality for the purpose of benchmarking. For example, assuming you +have an x64 processor, you could benchmark the code without AVX-512 even if both your processor +and compiler supports it: + +``` +cmake -B buildnoavx512 -D ROARING_DISABLE_AVX512=ON +cmake --build buildnoavx512 +./buildnoavx512/microbenchmarks/bench +``` + +You can benchmark without AVX or AVX-512 as well: + +``` +cmake -B buildnoavx -D ROARING_DISABLE_AVX=ON +cmake --build buildnoavx +./buildnoavx/microbenchmarks/bench +``` + +# Custom memory allocators +For general users, CRoaring would apply default allocator without extra codes. But global memory hook is also provided for those who want a custom memory allocator. Here is an example: +```C +#include <roaring.h> + +int main(){ + // define with your own memory hook + roaring_memory_t my_hook{my_malloc, my_free ...}; + // initialize global memory hook + roaring_init_memory_hook(my_hook); + // write you code here + ... +} +``` + + +# Example (C) + + +This example assumes that CRoaring has been build and that you are linking against the corresponding library. By default, CRoaring will install its header files in a `roaring` directory. If you are working from the amalgamation script, you may add the line `#include "roaring.c"` if you are not linking against a prebuilt CRoaring library and replace `#include <roaring/roaring.h>` by `#include "roaring.h"`. + +```c +#include <roaring/roaring.h> +#include <stdio.h> +#include <stdlib.h> +#include <assert.h> + +bool roaring_iterator_sumall(uint32_t value, void *param) { + *(uint32_t *)param += value; + return true; // iterate till the end +} + +int main() { + // create a new empty bitmap + roaring_bitmap_t *r1 = roaring_bitmap_create(); + // then we can add values + for (uint32_t i = 100; i < 1000; i++) roaring_bitmap_add(r1, i); + // check whether a value is contained + assert(roaring_bitmap_contains(r1, 500)); + // compute how many bits there are: + uint32_t cardinality = roaring_bitmap_get_cardinality(r1); + printf("Cardinality = %d \n", cardinality); + + // if your bitmaps have long runs, you can compress them by calling + // run_optimize + uint32_t expectedsizebasic = roaring_bitmap_portable_size_in_bytes(r1); + roaring_bitmap_run_optimize(r1); + uint32_t expectedsizerun = roaring_bitmap_portable_size_in_bytes(r1); + printf("size before run optimize %d bytes, and after %d bytes\n", + expectedsizebasic, expectedsizerun); + + // create a new bitmap containing the values {1,2,3,5,6} + roaring_bitmap_t *r2 = roaring_bitmap_from(1, 2, 3, 5, 6); + roaring_bitmap_printf(r2); // print it + + // we can also create a bitmap from a pointer to 32-bit integers + uint32_t somevalues[] = {2, 3, 4}; + roaring_bitmap_t *r3 = roaring_bitmap_of_ptr(3, somevalues); + + // we can also go in reverse and go from arrays to bitmaps + uint64_t card1 = roaring_bitmap_get_cardinality(r1); + uint32_t *arr1 = (uint32_t *)malloc(card1 * sizeof(uint32_t)); + assert(arr1 != NULL); + roaring_bitmap_to_uint32_array(r1, arr1); + roaring_bitmap_t *r1f = roaring_bitmap_of_ptr(card1, arr1); + free(arr1); + assert(roaring_bitmap_equals(r1, r1f)); // what we recover is equal + roaring_bitmap_free(r1f); + + // we can go from arrays to bitmaps from "offset" by "limit" + size_t offset = 100; + size_t limit = 1000; + uint32_t *arr3 = (uint32_t *)malloc(limit * sizeof(uint32_t)); + assert(arr3 != NULL); + roaring_bitmap_range_uint32_array(r1, offset, limit, arr3); + free(arr3); + + // we can copy and compare bitmaps + roaring_bitmap_t *z = roaring_bitmap_copy(r3); + assert(roaring_bitmap_equals(r3, z)); // what we recover is equal + roaring_bitmap_free(z); + + // we can compute union two-by-two + roaring_bitmap_t *r1_2_3 = roaring_bitmap_or(r1, r2); + roaring_bitmap_or_inplace(r1_2_3, r3); + + // we can compute a big union + const roaring_bitmap_t *allmybitmaps[] = {r1, r2, r3}; + roaring_bitmap_t *bigunion = roaring_bitmap_or_many(3, allmybitmaps); + assert( + roaring_bitmap_equals(r1_2_3, bigunion)); // what we recover is equal + // can also do the big union with a heap + roaring_bitmap_t *bigunionheap = + roaring_bitmap_or_many_heap(3, allmybitmaps); + assert(roaring_bitmap_equals(r1_2_3, bigunionheap)); + + roaring_bitmap_free(r1_2_3); + roaring_bitmap_free(bigunion); + roaring_bitmap_free(bigunionheap); + + // we can compute intersection two-by-two + roaring_bitmap_t *i1_2 = roaring_bitmap_and(r1, r2); + roaring_bitmap_free(i1_2); + + // we can write a bitmap to a pointer and recover it later + uint32_t expectedsize = roaring_bitmap_portable_size_in_bytes(r1); + char *serializedbytes = malloc(expectedsize); + roaring_bitmap_portable_serialize(r1, serializedbytes); + // Note: it is expected that the input follows the specification + // https://github.com/RoaringBitmap/RoaringFormatSpec + // otherwise the result may be unusable. + roaring_bitmap_t *t = roaring_bitmap_portable_deserialize_safe(serializedbytes, expectedsize); + if(t == NULL) { return EXIT_FAILURE; } + const char *reason = NULL; + if (!roaring_bitmap_internal_validate(t, &reason)) { + return EXIT_FAILURE; + } + assert(roaring_bitmap_equals(r1, t)); // what we recover is equal + roaring_bitmap_free(t); + // we can also check whether there is a bitmap at a memory location without + // reading it + size_t sizeofbitmap = + roaring_bitmap_portable_deserialize_size(serializedbytes, expectedsize); + assert(sizeofbitmap == + expectedsize); // sizeofbitmap would be zero if no bitmap were found + // we can also read the bitmap "safely" by specifying a byte size limit: + t = roaring_bitmap_portable_deserialize_safe(serializedbytes, expectedsize); + if(t == NULL) { + printf("Problem during deserialization.\n"); + // We could clear any memory and close any file here. + return EXIT_FAILURE; + } + // We can validate the bitmap we recovered to make sure it is proper. + const char *reason_failure = NULL; + if (!roaring_bitmap_internal_validate(t, &reason_failure)) { + printf("safely deserialized invalid bitmap: %s\n", reason_failure); + // We could clear any memory and close any file here. + return EXIT_FAILURE; + } + // It is still necessary for the content of seriallizedbytes to follow + // the standard: https://github.com/RoaringBitmap/RoaringFormatSpec + // This is guaranted when calling 'roaring_bitmap_portable_deserialize'. + assert(roaring_bitmap_equals(r1, t)); // what we recover is equal + roaring_bitmap_free(t); + + free(serializedbytes); + + // we can iterate over all values using custom functions + uint32_t counter = 0; + roaring_iterate(r1, roaring_iterator_sumall, &counter); + + // we can also create iterator structs + counter = 0; + roaring_uint32_iterator_t *i = roaring_iterator_create(r1); + while (i->has_value) { + counter++; // could use i->current_value + roaring_uint32_iterator_advance(i); + } + // you can skip over values and move the iterator with + // roaring_uint32_iterator_move_equalorlarger(i,someintvalue) + + roaring_uint32_iterator_free(i); + // roaring_bitmap_get_cardinality(r1) == counter + + // for greater speed, you can iterate over the data in bulk + i = roaring_iterator_create(r1); + uint32_t buffer[256]; + while (1) { + uint32_t ret = roaring_uint32_iterator_read(i, buffer, 256); + for (uint32_t j = 0; j < ret; j++) { + counter += buffer[j]; + } + if (ret < 256) { + break; + } + } + roaring_uint32_iterator_free(i); + + roaring_bitmap_free(r1); + roaring_bitmap_free(r2); + roaring_bitmap_free(r3); + return EXIT_SUCCESS; +} +``` + +# Compressed 64-bit Roaring bitmaps (C) + + +We also support efficient 64-bit compressed bitmaps in C: + +```c++ + roaring64_bitmap_t *r2 = roaring64_bitmap_create(); + for (uint64_t i = 100; i < 1000; i++) roaring64_bitmap_add(r2, i); + printf("cardinality (64-bit) = %d\n", (int) roaring64_bitmap_get_cardinality(r2)); + roaring64_bitmap_free(r2); +``` + + +# Conventional bitsets (C) + +We support convention bitsets (uncompressed) as part of the library. + +Simple example: + +```C +bitset_t * b = bitset_create(); +bitset_set(b,10); +bitset_get(b,10);// returns true +bitset_free(b); // frees memory +``` + +More advanced example: + +```C + bitset_t *b = bitset_create(); + for (int k = 0; k < 1000; ++k) { + bitset_set(b, 3 * k); + } + // We have bitset_count(b) == 1000. + // We have bitset_get(b, 3) is true + // You can iterate through the values: + size_t k = 0; + for (size_t i = 0; bitset_next_set_bit(b, &i); i++) { + // You will have i == k + k += 3; + } + // We support a wide range of operations on two bitsets such as + // bitset_inplace_symmetric_difference(b1,b2); + // bitset_inplace_symmetric_difference(b1,b2); + // bitset_inplace_difference(b1,b2);// should make no difference + // bitset_inplace_union(b1,b2); + // bitset_inplace_intersection(b1,b2); + // bitsets_disjoint + // bitsets_intersect +``` + +In some instances, you may want to convert a Roaring bitmap into a conventional (uncompressed) bitset. +Indeed, bitsets have advantages such as higher query performances in some cases. The following code +illustrates how you may do so: + +```C + roaring_bitmap_t *r1 = roaring_bitmap_create(); + for (uint32_t i = 100; i < 100000; i+= 1 + (i%5)) { + roaring_bitmap_add(r1, i); + } + for (uint32_t i = 100000; i < 500000; i+= 100) { + roaring_bitmap_add(r1, i); + } + roaring_bitmap_add_range(r1, 500000, 600000); + bitset_t * bitset = bitset_create(); + bool success = roaring_bitmap_to_bitset(r1, bitset); + assert(success); // could fail due to memory allocation. + assert(bitset_count(bitset) == roaring_bitmap_get_cardinality(r1)); + // You can then query the bitset: + for (uint32_t i = 100; i < 100000; i+= 1 + (i%5)) { + assert(bitset_get(bitset,i)); + } + for (uint32_t i = 100000; i < 500000; i+= 100) { + assert(bitset_get(bitset,i)); + } + // you must free the memory: + bitset_free(bitset); + roaring_bitmap_free(r1); +``` + +You should be aware that a convention bitset (`bitset_t *`) may use much more +memory than a Roaring bitmap in some cases. You should run benchmarks to determine +whether the conversion to a bitset has performance benefits in your case. + +# Example (C++) + + +This example assumes that CRoaring has been build and that you are linking against the corresponding library. By default, CRoaring will install its header files in a `roaring` directory so you may need to replace `#include "roaring.hh"` by `#include <roaring/roaring.hh>`. If you are working from the amalgamation script, you may add the line `#include "roaring.c"` if you are not linking against a CRoaring prebuilt library. + +```c++ +#include <iostream> + +#include "roaring.hh" + +using namespace roaring; + +int main() { + Roaring r1; + for (uint32_t i = 100; i < 1000; i++) { + r1.add(i); + } + + // check whether a value is contained + assert(r1.contains(500)); + + // compute how many bits there are: + uint32_t cardinality = r1.cardinality(); + + // if your bitmaps have long runs, you can compress them by calling + // run_optimize + uint32_t size = r1.getSizeInBytes(); + r1.runOptimize(); + + // you can enable "copy-on-write" for fast and shallow copies + r1.setCopyOnWrite(true); + + uint32_t compact_size = r1.getSizeInBytes(); + std::cout << "size before run optimize " << size << " bytes, and after " + << compact_size << " bytes." << std::endl; + + // create a new bitmap with varargs + Roaring r2 = Roaring::bitmapOf(5, 1, 2, 3, 5, 6); + + r2.printf(); + printf("\n"); + + // create a new bitmap with initializer list + Roaring r2i = Roaring::bitmapOfList({1, 2, 3, 5, 6}); + + assert(r2i == r2); + + // we can also create a bitmap from a pointer to 32-bit integers + const uint32_t values[] = {2, 3, 4}; + Roaring r3(3, values); + + // we can also go in reverse and go from arrays to bitmaps + uint64_t card1 = r1.cardinality(); + uint32_t *arr1 = new uint32_t[card1]; + r1.toUint32Array(arr1); + Roaring r1f(card1, arr1); + delete[] arr1; + + // bitmaps shall be equal + assert(r1 == r1f); + + // we can copy and compare bitmaps + Roaring z(r3); + assert(r3 == z); + + // we can compute union two-by-two + Roaring r1_2_3 = r1 | r2; + r1_2_3 |= r3; + + // we can compute a big union + const Roaring *allmybitmaps[] = {&r1, &r2, &r3}; + Roaring bigunion = Roaring::fastunion(3, allmybitmaps); + assert(r1_2_3 == bigunion); + + // we can compute intersection two-by-two + Roaring i1_2 = r1 & r2; + + // we can write a bitmap to a pointer and recover it later + uint32_t expectedsize = r1.getSizeInBytes(); + char *serializedbytes = new char[expectedsize]; + r1.write(serializedbytes); + // readSafe will not overflow, but the resulting bitmap + // is only valid and usable if the input follows the + // Roaring specification: https://github.com/RoaringBitmap/RoaringFormatSpec/ + Roaring t = Roaring::readSafe(serializedbytes, expectedsize); + assert(r1 == t); + delete[] serializedbytes; + + // we can iterate over all values using custom functions + uint32_t counter = 0; + r1.iterate( + [](uint32_t value, void *param) { + *(uint32_t *)param += value; + return true; + }, + &counter); + + // we can also iterate the C++ way + counter = 0; + for (Roaring::const_iterator i = t.begin(); i != t.end(); i++) { + ++counter; + } + // counter == t.cardinality() + + // we can move iterators to skip values + const uint32_t manyvalues[] = {2, 3, 4, 7, 8}; + Roaring rogue(5, manyvalues); + Roaring::const_iterator j = rogue.begin(); + j.equalorlarger(4); // *j == 4 + return EXIT_SUCCESS; +} + +``` + + + +# Building with cmake (Linux and macOS, Visual Studio users should see below) + +CRoaring follows the standard cmake workflow. Starting from the root directory of +the project (CRoaring), you can do: + +``` +mkdir -p build +cd build +cmake .. +cmake --build . +# follow by 'ctest' if you want to test. +# you can also type 'make install' to install the library on your system +# C header files typically get installed to /usr/local/include/roaring +# whereas C++ header files get installed to /usr/local/include/roaring +``` +(You can replace the ``build`` directory with any other directory name.) +By default all tests are built on all platforms, to skip building and running tests add `` -DENABLE_ROARING_TESTS=OFF `` to the command line. + +As with all ``cmake`` projects, you can specify the compilers you wish to use by adding (for example) ``-DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++`` to the ``cmake`` command line. + +If you are using clang or gcc and you know your target architecture, you can set the architecture by specifying `-DROARING_ARCH=arch`. For example, if you have many server but the oldest server is running the Intel `haswell` architecture, you can specify -`DROARING_ARCH=haswell`. In such cases, the produced binary will be optimized for processors having the characteristics of a haswell process and may not run on older architectures. You can find out the list of valid architecture values by typing `man gcc`. + + ``` + mkdir -p build_haswell + cd build_haswell + cmake -DROARING_ARCH=haswell .. + cmake --build . + ``` + +For a debug release, starting from the root directory of the project (CRoaring), try + +``` +mkdir -p debug +cd debug +cmake -DCMAKE_BUILD_TYPE=Debug -DROARING_SANITIZE=ON .. +ctest +``` + + +To check that your code abides by the style convention (make sure that ``clang-format`` is installed): + +``` +./tools/clang-format-check.sh +``` + +To reformat your code according to the style convention (make sure that ``clang-format`` is installed): + +``` +./tools/clang-format.sh +``` + +# Building (Visual Studio under Windows) + +We are assuming that you have a common Windows PC with at least Visual Studio 2015, and an x64 processor. + +To build with at least Visual Studio 2015 from the command line: +- Grab the CRoaring code from GitHub, e.g., by cloning it using [GitHub Desktop](https://desktop.github.com/). +- Install [CMake](https://cmake.org/download/). When you install it, make sure to ask that ``cmake`` be made available from the command line. +- Create a subdirectory within CRoaring, such as ``VisualStudio``. +- Using a shell, go to this newly created directory. For example, within GitHub Desktop, you can right-click on ``CRoaring`` in your GitHub repository list, and select ``Open in Git Shell``, then type ``cd VisualStudio`` in the newly created shell. +- Type ``cmake -DCMAKE_GENERATOR_PLATFORM=x64 ..`` in the shell while in the ``VisualStudio`` repository. (Alternatively, if you want to build a static library, you may use the command line ``cmake -DCMAKE_GENERATOR_PLATFORM=x64 -DROARING_BUILD_STATIC=ON ..``.) +- This last command created a Visual Studio solution file in the newly created directory (e.g., ``RoaringBitmap.sln``). Open this file in Visual Studio. You should now be able to build the project and run the tests. For example, in the ``Solution Explorer`` window (available from the ``View`` menu), right-click ``ALL_BUILD`` and select ``Build``. To test the code, still in the ``Solution Explorer`` window, select ``RUN_TESTS`` and select ``Build``. + +To build with at least Visual Studio 2017 directly in the IDE: +- Grab the CRoaring code from GitHub, e.g., by cloning it using [GitHub Desktop](https://desktop.github.com/). +- Select the ``Visual C++ tools for CMake`` optional component when installing the C++ Development Workload within Visual Studio. +- Within Visual Studio use ``File > Open > Folder...`` to open the CRoaring folder. +- Right click on ``CMakeLists.txt`` in the parent directory within ``Solution Explorer`` and select ``Build`` to build the project. +- For testing, in the Standard toolbar, drop the ``Select Startup Item...`` menu and choose one of the tests. Run the test by pressing the button to the left of the dropdown. + + +We have optimizations specific to AVX2 and AVX-512 in the code, and they are turned dynamically based on the detected hardware at runtime. + + +## Usage (Using `conan`) + +You can install the library using the conan package manager: + +``` +$ echo -e "[requires]\nroaring/0.3.3" > conanfile.txt +$ conan install . +``` + + +## Usage (Using `vcpkg` on Windows, Linux and macOS) + +[vcpkg](https://github.com/Microsoft/vcpkg) users on Windows, Linux and macOS can download and install `roaring` with one single command from their favorite shell. + +On Linux and macOS: + +``` +$ ./vcpkg install roaring +``` + +will build and install `roaring` as a static library. + +On Windows (64-bit): + +``` +.\vcpkg.exe install roaring:x64-windows +``` + +will build and install `roaring` as a shared library. + +``` +.\vcpkg.exe install roaring:x64-windows-static +``` + +will build and install `roaring` as a static library. + +These commands will also print out instructions on how to use the library from MSBuild or CMake-based projects. + +If you find the version of `roaring` shipped with `vcpkg` is out-of-date, feel free to report it to `vcpkg` community either by submiting an issue or by creating a PR. + +# SIMD-related throttling + +Our AVX2 code does not use floating-point numbers or multiplications, so it is not subject to turbo frequency throttling on many-core Intel processors. + +Our AVX-512 code is only enabled on recent hardware (Intel Ice Lake or better and AMD Zen 4) where SIMD-specific frequency throttling is not observed. + +# Thread safety + +Like, for example, STL containers or Java's default data structures, the CRoaring library has no built-in thread support. Thus whenever you modify a bitmap in one thread, it is unsafe to query it in others. It is safe however to query bitmaps (without modifying them) from several distinct threads, as long as you do not use the copy-on-write attribute. For example, you can safely copy a bitmap and use both copies in concurrently. One should probably avoid the use of the copy-on-write attribute in a threaded environment. + +Some of our users rely on "copy-on-write" (default to disabled). A bitmap with the copy-on-write flag +set to true might generate shared containers. A shared container is just a reference to a single +container with reference counting (we keep track of the number of shallow copies). If you copy shared +containers over several threads, this might be unsafe due to the need to update the counter concurrently. +Thus for shared containers, we use reference counting with an atomic counter. If the library is compiled +as a C library (the default), we use C11 atomics. Unfortunately, Visual Studio does not support C11 +atomics at this times (though this is subject to change). To compensate, we +use Windows-specific code in such instances (`_InterlockedDecrement` `_InterlockedIncrement`). + + +# How to best aggregate bitmaps? + +Suppose you want to compute the union (OR) of many bitmaps. How do you proceed? There are many +different strategies. + +You can use `roaring_bitmap_or_many(bitmapcount, bitmaps)` or `roaring_bitmap_or_many_heap(bitmapcount, bitmaps)` or you may +even roll your own aggregation: + +```C +roaring_bitmap_t *answer = roaring_bitmap_copy(bitmaps[0]); +for (size_t i = 1; i < bitmapcount; i++) { + roaring_bitmap_or_inplace(answer, bitmaps[i]); +} +``` + +All of them will work but they have different performance characteristics. The `roaring_bitmap_or_many_heap` should +probably only be used if, after benchmarking, you find that it is faster by a good margin: it uses more memory. + +The `roaring_bitmap_or_many` is meant as a good default. It works by trying to delay work as much as possible. +However, because it delays computations, it also does not optimize the format as the computation runs. It might +thus fail to see some useful pattern in the data such as long consecutive values. + +The approach based on repeated calls to `roaring_bitmap_or_inplace` +is also fine, and might even be faster in some cases. You can expect it to be faster if, after +a few calls, you get long sequences of consecutive values in the answer. That is, if the +final answer is all integers in the range [0,1000000), and this is apparent quickly, then the +later `roaring_bitmap_or_inplace` will be very fast. + +You should benchmark these alternatives on your own data to decide what is best. + +# Wrappers + +## Python +Tom Cornebize wrote a Python wrapper available at https://github.com/Ezibenroc/PyRoaringBitMap +Installing it is as easy as typing... + +``` +pip install pyroaring +``` + +## JavaScript + +Salvatore Previti wrote a Node/JavaScript wrapper available at https://github.com/SalvatorePreviti/roaring-node +Installing it is as easy as typing... + +``` +npm install roaring +``` + +## Swift + +Jérémie Piotte wrote a [Swift wrapper](https://github.com/RoaringBitmap/SwiftRoaring). + + +## C# + +Brandon Smith wrote a C# wrapper available at https://github.com/RogueException/CRoaring.Net (works for Windows and Linux under x64 processors) + + +## Go (golang) + +There is a Go (golang) wrapper available at https://github.com/RoaringBitmap/gocroaring + +## Rust + +Saulius Grigaliunas wrote a Rust wrapper available at https://github.com/saulius/croaring-rs + +## D + +Yuce Tekol wrote a D wrapper available at https://github.com/yuce/droaring + +## Redis + +Antonio Guilherme Ferreira Viggiano wrote a Redis Module available at https://github.com/aviggiano/redis-roaring + +## Zig + +Justin Whear wrote a Zig wrapper available at https://github.com/jwhear/roaring-zig + + +# Mailing list/discussion group + +https://groups.google.com/forum/#!forum/roaring-bitmaps + +# Contributing + +When contributing a change to the project, please run `tools/clang-format.sh` after making any changes. A github action runs on all PRs to ensure formatting is consistent with this. + +# References about Roaring + +- Daniel Lemire, Owen Kaser, Nathan Kurz, Luca Deri, Chris O'Hara, François Saint-Jacques, Gregory Ssi-Yan-Kai, Roaring Bitmaps: Implementation of an Optimized Software Library, Software: Practice and Experience Volume 48, Issue 4 April 2018 Pages 867-895 [arXiv:1709.07821](https://arxiv.org/abs/1709.07821) +- Samy Chambi, Daniel Lemire, Owen Kaser, Robert Godin, +Better bitmap performance with Roaring bitmaps, +Software: Practice and Experience Volume 46, Issue 5, pages 709–719, May 2016 [arXiv:1402.6407](http://arxiv.org/abs/1402.6407) +- Daniel Lemire, Gregory Ssi-Yan-Kai, Owen Kaser, Consistently faster and smaller compressed bitmaps with Roaring, Software: Practice and Experience Volume 46, Issue 11, pages 1547-1569, November 2016 [arXiv:1603.06549](http://arxiv.org/abs/1603.06549) +- Samy Chambi, Daniel Lemire, Robert Godin, Kamel Boukhalfa, Charles Allen, Fangjin Yang, Optimizing Druid with Roaring bitmaps, IDEAS 2016, 2016. http://r-libre.teluq.ca/950/ diff --git a/contrib/libs/croaring/SECURITY.md b/contrib/libs/croaring/SECURITY.md new file mode 100644 index 00000000000..1d9c45c86e1 --- /dev/null +++ b/contrib/libs/croaring/SECURITY.md @@ -0,0 +1,9 @@ +# Security Policy + +## Reporting a Vulnerability + +Please use the following contact information for reporting a vulnerability: + +- [Daniel Lemire]( https://www.teluq.ca/siteweb/univ/en/dlemire.html) - [email protected] + + diff --git a/contrib/libs/croaring/include/roaring/array_util.h b/contrib/libs/croaring/include/roaring/array_util.h new file mode 100644 index 00000000000..f062791cb7b --- /dev/null +++ b/contrib/libs/croaring/include/roaring/array_util.h @@ -0,0 +1,274 @@ +#ifndef ARRAY_UTIL_H +#define ARRAY_UTIL_H + +#include <stddef.h> // for size_t +#include <stdint.h> + +#include <roaring/portability.h> + +#if CROARING_IS_X64 +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/* + * Good old binary search. + * Assumes that array is sorted, has logarithmic complexity. + * if the result is x, then: + * if ( x>0 ) you have array[x] = ikey + * if ( x<0 ) then inserting ikey at position -x-1 in array (insuring that + * array[-x-1]=ikey) keys the array sorted. + */ +inline int32_t binarySearch(const uint16_t *array, int32_t lenarray, + uint16_t ikey) { + int32_t low = 0; + int32_t high = lenarray - 1; + while (low <= high) { + int32_t middleIndex = (low + high) >> 1; + uint16_t middleValue = array[middleIndex]; + if (middleValue < ikey) { + low = middleIndex + 1; + } else if (middleValue > ikey) { + high = middleIndex - 1; + } else { + return middleIndex; + } + } + return -(low + 1); +} + +/** + * Galloping search + * Assumes that array is sorted, has logarithmic complexity. + * if the result is x, then if x = length, you have that all values in array + * between pos and length are smaller than min. otherwise returns the first + * index x such that array[x] >= min. + */ +static inline int32_t advanceUntil(const uint16_t *array, int32_t pos, + int32_t length, uint16_t min) { + int32_t lower = pos + 1; + + if ((lower >= length) || (array[lower] >= min)) { + return lower; + } + + int32_t spansize = 1; + + while ((lower + spansize < length) && (array[lower + spansize] < min)) { + spansize <<= 1; + } + int32_t upper = (lower + spansize < length) ? lower + spansize : length - 1; + + if (array[upper] == min) { + return upper; + } + if (array[upper] < min) { + // means + // array + // has no + // item + // >= min + // pos = array.length; + return length; + } + + // we know that the next-smallest span was too small + lower += (spansize >> 1); + + int32_t mid = 0; + while (lower + 1 != upper) { + mid = (lower + upper) >> 1; + if (array[mid] == min) { + return mid; + } else if (array[mid] < min) { + lower = mid; + } else { + upper = mid; + } + } + return upper; +} + +/** + * Returns number of elements which are less than ikey. + * Array elements must be unique and sorted. + */ +static inline int32_t count_less(const uint16_t *array, int32_t lenarray, + uint16_t ikey) { + if (lenarray == 0) return 0; + int32_t pos = binarySearch(array, lenarray, ikey); + return pos >= 0 ? pos : -(pos + 1); +} + +/** + * Returns number of elements which are greater than ikey. + * Array elements must be unique and sorted. + */ +static inline int32_t count_greater(const uint16_t *array, int32_t lenarray, + uint16_t ikey) { + if (lenarray == 0) return 0; + int32_t pos = binarySearch(array, lenarray, ikey); + if (pos >= 0) { + return lenarray - (pos + 1); + } else { + return lenarray - (-pos - 1); + } +} + +/** + * From Schlegel et al., Fast Sorted-Set Intersection using SIMD Instructions + * Optimized by D. Lemire on May 3rd 2013 + * + * C should have capacity greater than the minimum of s_1 and s_b + 8 + * where 8 is sizeof(__m128i)/sizeof(uint16_t). + */ +int32_t intersect_vector16(const uint16_t *__restrict__ A, size_t s_a, + const uint16_t *__restrict__ B, size_t s_b, + uint16_t *C); + +int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, + const uint16_t *__restrict__ B, size_t s_b); + +/** + * Take an array container and write it out to a 32-bit array, using base + * as the offset. + */ +int array_container_to_uint32_array_vector16(void *vout, const uint16_t *array, + size_t cardinality, uint32_t base); +#if CROARING_COMPILER_SUPPORTS_AVX512 +int avx512_array_container_to_uint32_array(void *vout, const uint16_t *array, + size_t cardinality, uint32_t base); +#endif +/** + * Compute the cardinality of the intersection using SSE4 instructions + */ +int32_t intersect_vector16_cardinality(const uint16_t *__restrict__ A, + size_t s_a, + const uint16_t *__restrict__ B, + size_t s_b); + +/* Computes the intersection between one small and one large set of uint16_t. + * Stores the result into buffer and return the number of elements. */ +int32_t intersect_skewed_uint16(const uint16_t *smallarray, size_t size_s, + const uint16_t *largearray, size_t size_l, + uint16_t *buffer); + +/* Computes the size of the intersection between one small and one large set of + * uint16_t. */ +int32_t intersect_skewed_uint16_cardinality(const uint16_t *smallarray, + size_t size_s, + const uint16_t *largearray, + size_t size_l); + +/* Check whether the size of the intersection between one small and one large + * set of uint16_t is non-zero. */ +bool intersect_skewed_uint16_nonempty(const uint16_t *smallarray, size_t size_s, + const uint16_t *largearray, + size_t size_l); +/** + * Generic intersection function. + */ +int32_t intersect_uint16(const uint16_t *A, const size_t lenA, + const uint16_t *B, const size_t lenB, uint16_t *out); +/** + * Compute the size of the intersection (generic). + */ +int32_t intersect_uint16_cardinality(const uint16_t *A, const size_t lenA, + const uint16_t *B, const size_t lenB); + +/** + * Checking whether the size of the intersection is non-zero. + */ +bool intersect_uint16_nonempty(const uint16_t *A, const size_t lenA, + const uint16_t *B, const size_t lenB); +/** + * Generic union function. + */ +size_t union_uint16(const uint16_t *set_1, size_t size_1, const uint16_t *set_2, + size_t size_2, uint16_t *buffer); + +/** + * Generic XOR function. + */ +int32_t xor_uint16(const uint16_t *array_1, int32_t card_1, + const uint16_t *array_2, int32_t card_2, uint16_t *out); + +/** + * Generic difference function (ANDNOT). + */ +int difference_uint16(const uint16_t *a1, int length1, const uint16_t *a2, + int length2, uint16_t *a_out); + +/** + * Generic intersection function. + */ +size_t intersection_uint32(const uint32_t *A, const size_t lenA, + const uint32_t *B, const size_t lenB, uint32_t *out); + +/** + * Generic intersection function, returns just the cardinality. + */ +size_t intersection_uint32_card(const uint32_t *A, const size_t lenA, + const uint32_t *B, const size_t lenB); + +/** + * Generic union function. + */ +size_t union_uint32(const uint32_t *set_1, size_t size_1, const uint32_t *set_2, + size_t size_2, uint32_t *buffer); + +/** + * A fast SSE-based union function. + */ +uint32_t union_vector16(const uint16_t *__restrict__ set_1, uint32_t size_1, + const uint16_t *__restrict__ set_2, uint32_t size_2, + uint16_t *__restrict__ buffer); +/** + * A fast SSE-based XOR function. + */ +uint32_t xor_vector16(const uint16_t *__restrict__ array1, uint32_t length1, + const uint16_t *__restrict__ array2, uint32_t length2, + uint16_t *__restrict__ output); + +/** + * A fast SSE-based difference function. + */ +int32_t difference_vector16(const uint16_t *__restrict__ A, size_t s_a, + const uint16_t *__restrict__ B, size_t s_b, + uint16_t *C); + +/** + * Generic union function, returns just the cardinality. + */ +size_t union_uint32_card(const uint32_t *set_1, size_t size_1, + const uint32_t *set_2, size_t size_2); + +/** + * combines union_uint16 and union_vector16 optimally + */ +size_t fast_union_uint16(const uint16_t *set_1, size_t size_1, + const uint16_t *set_2, size_t size_2, + uint16_t *buffer); + +bool memequals(const void *s1, const void *s2, size_t n); + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif +#endif diff --git a/contrib/libs/croaring/include/roaring/art/art.h b/contrib/libs/croaring/include/roaring/art/art.h new file mode 100644 index 00000000000..e191c1eedaa --- /dev/null +++ b/contrib/libs/croaring/include/roaring/art/art.h @@ -0,0 +1,192 @@ +#ifndef ART_ART_H +#define ART_ART_H + +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> + +/* + * This file contains an implementation of an Adaptive Radix Tree as described + * in https://db.in.tum.de/~leis/papers/ART.pdf. + * + * The ART contains the keys in _byte lexographical_ order. + * + * Other features: + * * Fixed 48 bit key length: all keys are assumed to be be 48 bits in size. + * This allows us to put the key and key prefixes directly in nodes, reducing + * indirection at no additional memory overhead. + * * Key compression: the only inner nodes created are at points where key + * chunks _differ_. This means that if there are two entries with different + * high 48 bits, then there is only one inner node containing the common key + * prefix, and two leaves. + * * Intrusive leaves: the leaf struct is included in user values. This removes + * a layer of indirection. + */ + +// Fixed length of keys in the ART. All keys are assumed to be of this length. +#define ART_KEY_BYTES 6 + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +typedef uint8_t art_key_chunk_t; +typedef struct art_node_s art_node_t; + +/** + * Wrapper to allow an empty tree. + */ +typedef struct art_s { + art_node_t *root; +} art_t; + +/** + * Values inserted into the tree have to be cast-able to art_val_t. This + * improves performance by reducing indirection. + * + * NOTE: Value pointers must be unique! This is because each value struct + * contains the key corresponding to the value. + */ +typedef struct art_val_s { + art_key_chunk_t key[ART_KEY_BYTES]; +} art_val_t; + +/** + * Compares two keys, returns their relative order: + * * Key 1 < key 2: returns a negative value + * * Key 1 == key 2: returns 0 + * * Key 1 > key 2: returns a positive value + */ +int art_compare_keys(const art_key_chunk_t key1[], + const art_key_chunk_t key2[]); + +/** + * Inserts the given key and value. + */ +void art_insert(art_t *art, const art_key_chunk_t *key, art_val_t *val); + +/** + * Returns the value erased, NULL if not found. + */ +art_val_t *art_erase(art_t *art, const art_key_chunk_t *key); + +/** + * Returns the value associated with the given key, NULL if not found. + */ +art_val_t *art_find(const art_t *art, const art_key_chunk_t *key); + +/** + * Returns true if the ART is empty. + */ +bool art_is_empty(const art_t *art); + +/** + * Frees the nodes of the ART except the values, which the user is expected to + * free. + */ +void art_free(art_t *art); + +/** + * Returns the size in bytes of the ART. Includes size of pointers to values, + * but not the values themselves. + */ +size_t art_size_in_bytes(const art_t *art); + +/** + * Prints the ART using printf, useful for debugging. + */ +void art_printf(const art_t *art); + +/** + * Callback for validating the value stored in a leaf. + * + * Should return true if the value is valid, false otherwise + * If false is returned, `*reason` should be set to a static string describing + * the reason for the failure. + */ +typedef bool (*art_validate_cb_t)(const art_val_t *val, const char **reason); + +/** + * Validate the ART tree, ensuring it is internally consistent. + */ +bool art_internal_validate(const art_t *art, const char **reason, + art_validate_cb_t validate_cb); + +/** + * ART-internal iterator bookkeeping. Users should treat this as an opaque type. + */ +typedef struct art_iterator_frame_s { + art_node_t *node; + uint8_t index_in_node; +} art_iterator_frame_t; + +/** + * Users should only access `key` and `value` in iterators. The iterator is + * valid when `value != NULL`. + */ +typedef struct art_iterator_s { + art_key_chunk_t key[ART_KEY_BYTES]; + art_val_t *value; + + uint8_t depth; // Key depth + uint8_t frame; // Node depth + + // State for each node in the ART the iterator has travelled from the root. + // This is `ART_KEY_BYTES + 1` because it includes state for the leaf too. + art_iterator_frame_t frames[ART_KEY_BYTES + 1]; +} art_iterator_t; + +/** + * Creates an iterator initialzed to the first or last entry in the ART, + * depending on `first`. The iterator is not valid if there are no entries in + * the ART. + */ +art_iterator_t art_init_iterator(const art_t *art, bool first); + +/** + * Returns an initialized iterator positioned at a key equal to or greater than + * the given key, if it exists. + */ +art_iterator_t art_lower_bound(const art_t *art, const art_key_chunk_t *key); + +/** + * Returns an initialized iterator positioned at a key greater than the given + * key, if it exists. + */ +art_iterator_t art_upper_bound(const art_t *art, const art_key_chunk_t *key); + +/** + * The following iterator movement functions return true if a new entry was + * encountered. + */ +bool art_iterator_move(art_iterator_t *iterator, bool forward); +bool art_iterator_next(art_iterator_t *iterator); +bool art_iterator_prev(art_iterator_t *iterator); + +/** + * Moves the iterator forward to a key equal to or greater than the given key. + */ +bool art_iterator_lower_bound(art_iterator_t *iterator, + const art_key_chunk_t *key); + +/** + * Insert the value and positions the iterator at the key. + */ +void art_iterator_insert(art_t *art, art_iterator_t *iterator, + const art_key_chunk_t *key, art_val_t *val); + +/** + * Erase the value pointed at by the iterator. Moves the iterator to the next + * leaf. Returns the value erased or NULL if nothing was erased. + */ +art_val_t *art_iterator_erase(art_t *art, art_iterator_t *iterator); + +#ifdef __cplusplus +} // extern "C" +} // namespace roaring +} // namespace internal +#endif + +#endif diff --git a/contrib/libs/croaring/include/roaring/bitset/bitset.h b/contrib/libs/croaring/include/roaring/bitset/bitset.h new file mode 100644 index 00000000000..e79415f2e4f --- /dev/null +++ b/contrib/libs/croaring/include/roaring/bitset/bitset.h @@ -0,0 +1,292 @@ +#ifndef CBITSET_BITSET_H +#define CBITSET_BITSET_H + +// For compatibility with MSVC with the use of `restrict` +#if (__STDC_VERSION__ >= 199901L) || \ + (defined(__GNUC__) && defined(__STDC_VERSION__)) +#define CBITSET_RESTRICT restrict +#else +#define CBITSET_RESTRICT +#endif // (__STDC_VERSION__ >= 199901L) || (defined(__GNUC__) && + // defined(__STDC_VERSION__ )) + +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <roaring/portability.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace api { +#endif + +struct bitset_s { + uint64_t *CBITSET_RESTRICT array; + /* For simplicity and performance, we prefer to have a size and a capacity + * that is a multiple of 64 bits. Thus we only track the size and the + * capacity in terms of 64-bit words allocated */ + size_t arraysize; + size_t capacity; +}; + +typedef struct bitset_s bitset_t; + +/* Create a new bitset. Return NULL in case of failure. */ +bitset_t *bitset_create(void); + +/* Create a new bitset able to contain size bits. Return NULL in case of + * failure. */ +bitset_t *bitset_create_with_capacity(size_t size); + +/* Free memory. */ +void bitset_free(bitset_t *bitset); + +/* Set all bits to zero. */ +void bitset_clear(bitset_t *bitset); + +/* Set all bits to one. */ +void bitset_fill(bitset_t *bitset); + +/* Create a copy */ +bitset_t *bitset_copy(const bitset_t *bitset); + +/* For advanced users: Resize the bitset so that it can support newarraysize * + * 64 bits. Return true in case of success, false for failure. Pad with zeroes + * new buffer areas if requested. */ +bool bitset_resize(bitset_t *bitset, size_t newarraysize, bool padwithzeroes); + +/* returns how many bytes of memory the backend buffer uses */ +inline size_t bitset_size_in_bytes(const bitset_t *bitset) { + return bitset->arraysize * sizeof(uint64_t); +} + +/* returns how many bits can be accessed */ +inline size_t bitset_size_in_bits(const bitset_t *bitset) { + return bitset->arraysize * 64; +} + +/* returns how many words (64-bit) of memory the backend buffer uses */ +inline size_t bitset_size_in_words(const bitset_t *bitset) { + return bitset->arraysize; +} + +/* For advanced users: Grow the bitset so that it can support newarraysize * 64 + * bits with padding. Return true in case of success, false for failure. */ +bool bitset_grow(bitset_t *bitset, size_t newarraysize); + +/* attempts to recover unused memory, return false in case of + * roaring_reallocation failure */ +bool bitset_trim(bitset_t *bitset); + +/* shifts all bits by 's' positions so that the bitset representing values + * 1,2,10 would represent values 1+s, 2+s, 10+s */ +void bitset_shift_left(bitset_t *bitset, size_t s); + +/* shifts all bits by 's' positions so that the bitset representing values + * 1,2,10 would represent values 1-s, 2-s, 10-s, negative values are deleted */ +void bitset_shift_right(bitset_t *bitset, size_t s); + +/* Set the ith bit. Attempts to resize the bitset if needed (may silently fail) + */ +inline void bitset_set(bitset_t *bitset, size_t i) { + size_t shiftedi = i / 64; + if (shiftedi >= bitset->arraysize) { + if (!bitset_grow(bitset, shiftedi + 1)) { + return; + } + } + bitset->array[shiftedi] |= ((uint64_t)1) << (i % 64); +} + +/* Set the ith bit to the specified value. Attempts to resize the bitset if + * needed (may silently fail) */ +inline void bitset_set_to_value(bitset_t *bitset, size_t i, bool flag) { + size_t shiftedi = i / 64; + uint64_t mask = ((uint64_t)1) << (i % 64); + uint64_t dynmask = ((uint64_t)flag) << (i % 64); + if (shiftedi >= bitset->arraysize) { + if (!bitset_grow(bitset, shiftedi + 1)) { + return; + } + } + uint64_t w = bitset->array[shiftedi]; + w &= ~mask; + w |= dynmask; + bitset->array[shiftedi] = w; +} + +/* Get the value of the ith bit. */ +inline bool bitset_get(const bitset_t *bitset, size_t i) { + size_t shiftedi = i / 64; + if (shiftedi >= bitset->arraysize) { + return false; + } + return (bitset->array[shiftedi] & (((uint64_t)1) << (i % 64))) != 0; +} + +/* Count number of bits set. */ +size_t bitset_count(const bitset_t *bitset); + +/* Find the index of the first bit set. Or zero if the bitset is empty. */ +size_t bitset_minimum(const bitset_t *bitset); + +/* Find the index of the last bit set. Or zero if the bitset is empty. */ +size_t bitset_maximum(const bitset_t *bitset); + +/* compute the union in-place (to b1), returns true if successful, to generate a + * new bitset first call bitset_copy */ +bool bitset_inplace_union(bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2); + +/* report the size of the union (without materializing it) */ +size_t bitset_union_count(const bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2); + +/* compute the intersection in-place (to b1), to generate a new bitset first + * call bitset_copy */ +void bitset_inplace_intersection(bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2); + +/* report the size of the intersection (without materializing it) */ +size_t bitset_intersection_count(const bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2); + +/* returns true if the bitsets contain no common elements */ +bool bitsets_disjoint(const bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2); + +/* returns true if the bitsets contain any common elements */ +bool bitsets_intersect(const bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2); + +/* returns true if b1 contains all of the set bits of b2 */ +bool bitset_contains_all(const bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2); + +/* compute the difference in-place (to b1), to generate a new bitset first call + * bitset_copy */ +void bitset_inplace_difference(bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2); + +/* compute the size of the difference */ +size_t bitset_difference_count(const bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2); + +/* compute the symmetric difference in-place (to b1), return true if successful, + * to generate a new bitset first call bitset_copy */ +bool bitset_inplace_symmetric_difference(bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2); + +/* compute the size of the symmetric difference */ +size_t bitset_symmetric_difference_count(const bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2); + +/* iterate over the set bits + like so : + for(size_t i = 0; bitset_next_set_bit(b,&i) ; i++) { + //..... + } + */ +inline bool bitset_next_set_bit(const bitset_t *bitset, size_t *i) { + size_t x = *i / 64; + if (x >= bitset->arraysize) { + return false; + } + uint64_t w = bitset->array[x]; + w >>= (*i & 63); + if (w != 0) { + *i += roaring_trailing_zeroes(w); + return true; + } + x++; + while (x < bitset->arraysize) { + w = bitset->array[x]; + if (w != 0) { + *i = x * 64 + roaring_trailing_zeroes(w); + return true; + } + x++; + } + return false; +} + +/* iterate over the set bits + like so : + size_t buffer[256]; + size_t howmany = 0; + for(size_t startfrom = 0; (howmany = bitset_next_set_bits(b,buffer,256, + &startfrom)) > 0 ; startfrom++) { + //..... + } + */ +inline size_t bitset_next_set_bits(const bitset_t *bitset, size_t *buffer, + size_t capacity, size_t *startfrom) { + if (capacity == 0) return 0; // sanity check + size_t x = *startfrom / 64; + if (x >= bitset->arraysize) { + return 0; // nothing more to iterate over + } + uint64_t w = bitset->array[x]; + w >>= (*startfrom & 63); + size_t howmany = 0; + size_t base = x << 6; + while (howmany < capacity) { + while (w != 0) { + uint64_t t = w & (~w + 1); + int r = roaring_trailing_zeroes(w); + buffer[howmany++] = r + base; + if (howmany == capacity) goto end; + w ^= t; + } + x += 1; + if (x == bitset->arraysize) { + break; + } + base += 64; + w = bitset->array[x]; + } +end: + if (howmany > 0) { + *startfrom = buffer[howmany - 1]; + } + return howmany; +} + +typedef bool (*bitset_iterator)(size_t value, void *param); + +// return true if uninterrupted +inline bool bitset_for_each(const bitset_t *b, bitset_iterator iterator, + void *ptr) { + size_t base = 0; + for (size_t i = 0; i < b->arraysize; ++i) { + uint64_t w = b->array[i]; + while (w != 0) { + uint64_t t = w & (~w + 1); + int r = roaring_trailing_zeroes(w); + if (!iterator(r + base, ptr)) return false; + w ^= t; + } + base += 64; + } + return true; +} + +inline void bitset_print(const bitset_t *b) { + printf("{"); + for (size_t i = 0; bitset_next_set_bit(b, &i); i++) { + printf("%zu, ", i); + } + printf("}"); +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace api { +#endif + +#endif diff --git a/contrib/libs/croaring/include/roaring/bitset_util.h b/contrib/libs/croaring/include/roaring/bitset_util.h new file mode 100644 index 00000000000..7a7da00b7d8 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/bitset_util.h @@ -0,0 +1,718 @@ +#ifndef BITSET_UTIL_H +#define BITSET_UTIL_H + +#include <stdint.h> + +#include <roaring/portability.h> +#include <roaring/utilasm.h> + +#if CROARING_IS_X64 +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/* + * Set all bits in indexes [begin,end) to true. + */ +static inline void bitset_set_range(uint64_t *words, uint32_t start, + uint32_t end) { + if (start == end) return; + uint32_t firstword = start / 64; + uint32_t endword = (end - 1) / 64; + if (firstword == endword) { + words[firstword] |= ((~UINT64_C(0)) << (start % 64)) & + ((~UINT64_C(0)) >> ((~end + 1) % 64)); + return; + } + words[firstword] |= (~UINT64_C(0)) << (start % 64); + for (uint32_t i = firstword + 1; i < endword; i++) { + words[i] = ~UINT64_C(0); + } + words[endword] |= (~UINT64_C(0)) >> ((~end + 1) % 64); +} + +/* + * Find the cardinality of the bitset in [begin,begin+lenminusone] + */ +static inline int bitset_lenrange_cardinality(const uint64_t *words, + uint32_t start, + uint32_t lenminusone) { + uint32_t firstword = start / 64; + uint32_t endword = (start + lenminusone) / 64; + if (firstword == endword) { + return roaring_hamming(words[firstword] & + ((~UINT64_C(0)) >> ((63 - lenminusone) % 64)) + << (start % 64)); + } + int answer = + roaring_hamming(words[firstword] & ((~UINT64_C(0)) << (start % 64))); + for (uint32_t i = firstword + 1; i < endword; i++) { + answer += roaring_hamming(words[i]); + } + answer += roaring_hamming(words[endword] & + (~UINT64_C(0)) >> + (((~start + 1) - lenminusone - 1) % 64)); + return answer; +} + +/* + * Check whether the cardinality of the bitset in [begin,begin+lenminusone] is 0 + */ +static inline bool bitset_lenrange_empty(const uint64_t *words, uint32_t start, + uint32_t lenminusone) { + uint32_t firstword = start / 64; + uint32_t endword = (start + lenminusone) / 64; + if (firstword == endword) { + return (words[firstword] & ((~UINT64_C(0)) >> ((63 - lenminusone) % 64)) + << (start % 64)) == 0; + } + if (((words[firstword] & ((~UINT64_C(0)) << (start % 64)))) != 0) { + return false; + } + for (uint32_t i = firstword + 1; i < endword; i++) { + if (words[i] != 0) { + return false; + } + } + if ((words[endword] & + (~UINT64_C(0)) >> (((~start + 1) - lenminusone - 1) % 64)) != 0) { + return false; + } + return true; +} + +/* + * Set all bits in indexes [begin,begin+lenminusone] to true. + */ +static inline void bitset_set_lenrange(uint64_t *words, uint32_t start, + uint32_t lenminusone) { + uint32_t firstword = start / 64; + uint32_t endword = (start + lenminusone) / 64; + if (firstword == endword) { + words[firstword] |= ((~UINT64_C(0)) >> ((63 - lenminusone) % 64)) + << (start % 64); + return; + } + uint64_t temp = words[endword]; + words[firstword] |= (~UINT64_C(0)) << (start % 64); + for (uint32_t i = firstword + 1; i < endword; i += 2) + words[i] = words[i + 1] = ~UINT64_C(0); + words[endword] = + temp | (~UINT64_C(0)) >> (((~start + 1) - lenminusone - 1) % 64); +} + +/* + * Flip all the bits in indexes [begin,end). + */ +static inline void bitset_flip_range(uint64_t *words, uint32_t start, + uint32_t end) { + if (start == end) return; + uint32_t firstword = start / 64; + uint32_t endword = (end - 1) / 64; + words[firstword] ^= ~((~UINT64_C(0)) << (start % 64)); + for (uint32_t i = firstword; i < endword; i++) { + words[i] = ~words[i]; + } + words[endword] ^= ((~UINT64_C(0)) >> ((~end + 1) % 64)); +} + +/* + * Set all bits in indexes [begin,end) to false. + */ +static inline void bitset_reset_range(uint64_t *words, uint32_t start, + uint32_t end) { + if (start == end) return; + uint32_t firstword = start / 64; + uint32_t endword = (end - 1) / 64; + if (firstword == endword) { + words[firstword] &= ~(((~UINT64_C(0)) << (start % 64)) & + ((~UINT64_C(0)) >> ((~end + 1) % 64))); + return; + } + words[firstword] &= ~((~UINT64_C(0)) << (start % 64)); + for (uint32_t i = firstword + 1; i < endword; i++) { + words[i] = UINT64_C(0); + } + words[endword] &= ~((~UINT64_C(0)) >> ((~end + 1) % 64)); +} + +/* + * Given a bitset containing "length" 64-bit words, write out the position + * of all the set bits to "out", values start at "base". + * + * The "out" pointer should be sufficient to store the actual number of bits + * set. + * + * Returns how many values were actually decoded. + * + * This function should only be expected to be faster than + * bitset_extract_setbits + * when the density of the bitset is high. + * + * This function uses AVX2 decoding. + */ +size_t bitset_extract_setbits_avx2(const uint64_t *words, size_t length, + uint32_t *out, size_t outcapacity, + uint32_t base); + +size_t bitset_extract_setbits_avx512(const uint64_t *words, size_t length, + uint32_t *out, size_t outcapacity, + uint32_t base); +/* + * Given a bitset containing "length" 64-bit words, write out the position + * of all the set bits to "out", values start at "base". + * + * The "out" pointer should be sufficient to store the actual number of bits + *set. + * + * Returns how many values were actually decoded. + */ +size_t bitset_extract_setbits(const uint64_t *words, size_t length, + uint32_t *out, uint32_t base); + +/* + * Given a bitset containing "length" 64-bit words, write out the position + * of all the set bits to "out" as 16-bit integers, values start at "base" (can + *be set to zero) + * + * The "out" pointer should be sufficient to store the actual number of bits + *set. + * + * Returns how many values were actually decoded. + * + * This function should only be expected to be faster than + *bitset_extract_setbits_uint16 + * when the density of the bitset is high. + * + * This function uses SSE decoding. + */ +size_t bitset_extract_setbits_sse_uint16(const uint64_t *words, size_t length, + uint16_t *out, size_t outcapacity, + uint16_t base); + +size_t bitset_extract_setbits_avx512_uint16(const uint64_t *words, + size_t length, uint16_t *out, + size_t outcapacity, uint16_t base); + +/* + * Given a bitset containing "length" 64-bit words, write out the position + * of all the set bits to "out", values start at "base" + * (can be set to zero) + * + * The "out" pointer should be sufficient to store the actual number of bits + *set. + * + * Returns how many values were actually decoded. + */ +size_t bitset_extract_setbits_uint16(const uint64_t *words, size_t length, + uint16_t *out, uint16_t base); + +/* + * Given two bitsets containing "length" 64-bit words, write out the position + * of all the common set bits to "out", values start at "base" + * (can be set to zero) + * + * The "out" pointer should be sufficient to store the actual number of bits + * set. + * + * Returns how many values were actually decoded. + */ +size_t bitset_extract_intersection_setbits_uint16( + const uint64_t *__restrict__ words1, const uint64_t *__restrict__ words2, + size_t length, uint16_t *out, uint16_t base); + +/* + * Given a bitset having cardinality card, set all bit values in the list (there + * are length of them) + * and return the updated cardinality. This evidently assumes that the bitset + * already contained data. + */ +uint64_t bitset_set_list_withcard(uint64_t *words, uint64_t card, + const uint16_t *list, uint64_t length); +/* + * Given a bitset, set all bit values in the list (there + * are length of them). + */ +void bitset_set_list(uint64_t *words, const uint16_t *list, uint64_t length); + +/* + * Given a bitset having cardinality card, unset all bit values in the list + * (there are length of them) + * and return the updated cardinality. This evidently assumes that the bitset + * already contained data. + */ +uint64_t bitset_clear_list(uint64_t *words, uint64_t card, const uint16_t *list, + uint64_t length); + +/* + * Given a bitset having cardinality card, toggle all bit values in the list + * (there are length of them) + * and return the updated cardinality. This evidently assumes that the bitset + * already contained data. + */ + +uint64_t bitset_flip_list_withcard(uint64_t *words, uint64_t card, + const uint16_t *list, uint64_t length); + +void bitset_flip_list(uint64_t *words, const uint16_t *list, uint64_t length); + +#if CROARING_IS_X64 +/*** + * BEGIN Harley-Seal popcount functions. + */ +CROARING_TARGET_AVX2 +/** + * Compute the population count of a 256-bit word + * This is not especially fast, but it is convenient as part of other functions. + */ +static inline __m256i popcount256(__m256i v) { + const __m256i lookuppos = _mm256_setr_epi8( + /* 0 */ 4 + 0, /* 1 */ 4 + 1, /* 2 */ 4 + 1, /* 3 */ 4 + 2, + /* 4 */ 4 + 1, /* 5 */ 4 + 2, /* 6 */ 4 + 2, /* 7 */ 4 + 3, + /* 8 */ 4 + 1, /* 9 */ 4 + 2, /* a */ 4 + 2, /* b */ 4 + 3, + /* c */ 4 + 2, /* d */ 4 + 3, /* e */ 4 + 3, /* f */ 4 + 4, + + /* 0 */ 4 + 0, /* 1 */ 4 + 1, /* 2 */ 4 + 1, /* 3 */ 4 + 2, + /* 4 */ 4 + 1, /* 5 */ 4 + 2, /* 6 */ 4 + 2, /* 7 */ 4 + 3, + /* 8 */ 4 + 1, /* 9 */ 4 + 2, /* a */ 4 + 2, /* b */ 4 + 3, + /* c */ 4 + 2, /* d */ 4 + 3, /* e */ 4 + 3, /* f */ 4 + 4); + const __m256i lookupneg = _mm256_setr_epi8( + /* 0 */ 4 - 0, /* 1 */ 4 - 1, /* 2 */ 4 - 1, /* 3 */ 4 - 2, + /* 4 */ 4 - 1, /* 5 */ 4 - 2, /* 6 */ 4 - 2, /* 7 */ 4 - 3, + /* 8 */ 4 - 1, /* 9 */ 4 - 2, /* a */ 4 - 2, /* b */ 4 - 3, + /* c */ 4 - 2, /* d */ 4 - 3, /* e */ 4 - 3, /* f */ 4 - 4, + + /* 0 */ 4 - 0, /* 1 */ 4 - 1, /* 2 */ 4 - 1, /* 3 */ 4 - 2, + /* 4 */ 4 - 1, /* 5 */ 4 - 2, /* 6 */ 4 - 2, /* 7 */ 4 - 3, + /* 8 */ 4 - 1, /* 9 */ 4 - 2, /* a */ 4 - 2, /* b */ 4 - 3, + /* c */ 4 - 2, /* d */ 4 - 3, /* e */ 4 - 3, /* f */ 4 - 4); + const __m256i low_mask = _mm256_set1_epi8(0x0f); + + const __m256i lo = _mm256_and_si256(v, low_mask); + const __m256i hi = _mm256_and_si256(_mm256_srli_epi16(v, 4), low_mask); + const __m256i popcnt1 = _mm256_shuffle_epi8(lookuppos, lo); + const __m256i popcnt2 = _mm256_shuffle_epi8(lookupneg, hi); + return _mm256_sad_epu8(popcnt1, popcnt2); +} +CROARING_UNTARGET_AVX2 + +CROARING_TARGET_AVX2 +/** + * Simple CSA over 256 bits + */ +static inline void CSA(__m256i *h, __m256i *l, __m256i a, __m256i b, + __m256i c) { + const __m256i u = _mm256_xor_si256(a, b); + *h = _mm256_or_si256(_mm256_and_si256(a, b), _mm256_and_si256(u, c)); + *l = _mm256_xor_si256(u, c); +} +CROARING_UNTARGET_AVX2 + +CROARING_TARGET_AVX2 +/** + * Fast Harley-Seal AVX population count function + */ +inline static uint64_t avx2_harley_seal_popcount256(const __m256i *data, + const uint64_t size) { + __m256i total = _mm256_setzero_si256(); + __m256i ones = _mm256_setzero_si256(); + __m256i twos = _mm256_setzero_si256(); + __m256i fours = _mm256_setzero_si256(); + __m256i eights = _mm256_setzero_si256(); + __m256i sixteens = _mm256_setzero_si256(); + __m256i twosA, twosB, foursA, foursB, eightsA, eightsB; + + const uint64_t limit = size - size % 16; + uint64_t i = 0; + + for (; i < limit; i += 16) { + CSA(&twosA, &ones, ones, _mm256_lddqu_si256(data + i), + _mm256_lddqu_si256(data + i + 1)); + CSA(&twosB, &ones, ones, _mm256_lddqu_si256(data + i + 2), + _mm256_lddqu_si256(data + i + 3)); + CSA(&foursA, &twos, twos, twosA, twosB); + CSA(&twosA, &ones, ones, _mm256_lddqu_si256(data + i + 4), + _mm256_lddqu_si256(data + i + 5)); + CSA(&twosB, &ones, ones, _mm256_lddqu_si256(data + i + 6), + _mm256_lddqu_si256(data + i + 7)); + CSA(&foursB, &twos, twos, twosA, twosB); + CSA(&eightsA, &fours, fours, foursA, foursB); + CSA(&twosA, &ones, ones, _mm256_lddqu_si256(data + i + 8), + _mm256_lddqu_si256(data + i + 9)); + CSA(&twosB, &ones, ones, _mm256_lddqu_si256(data + i + 10), + _mm256_lddqu_si256(data + i + 11)); + CSA(&foursA, &twos, twos, twosA, twosB); + CSA(&twosA, &ones, ones, _mm256_lddqu_si256(data + i + 12), + _mm256_lddqu_si256(data + i + 13)); + CSA(&twosB, &ones, ones, _mm256_lddqu_si256(data + i + 14), + _mm256_lddqu_si256(data + i + 15)); + CSA(&foursB, &twos, twos, twosA, twosB); + CSA(&eightsB, &fours, fours, foursA, foursB); + CSA(&sixteens, &eights, eights, eightsA, eightsB); + + total = _mm256_add_epi64(total, popcount256(sixteens)); + } + + total = _mm256_slli_epi64(total, 4); // * 16 + total = _mm256_add_epi64( + total, _mm256_slli_epi64(popcount256(eights), 3)); // += 8 * ... + total = _mm256_add_epi64( + total, _mm256_slli_epi64(popcount256(fours), 2)); // += 4 * ... + total = _mm256_add_epi64( + total, _mm256_slli_epi64(popcount256(twos), 1)); // += 2 * ... + total = _mm256_add_epi64(total, popcount256(ones)); + for (; i < size; i++) + total = + _mm256_add_epi64(total, popcount256(_mm256_lddqu_si256(data + i))); + + return (uint64_t)(_mm256_extract_epi64(total, 0)) + + (uint64_t)(_mm256_extract_epi64(total, 1)) + + (uint64_t)(_mm256_extract_epi64(total, 2)) + + (uint64_t)(_mm256_extract_epi64(total, 3)); +} +CROARING_UNTARGET_AVX2 + +#define AVXPOPCNTFNC(opname, avx_intrinsic) \ + static inline uint64_t avx2_harley_seal_popcount256_##opname( \ + const __m256i *data1, const __m256i *data2, const uint64_t size) { \ + __m256i total = _mm256_setzero_si256(); \ + __m256i ones = _mm256_setzero_si256(); \ + __m256i twos = _mm256_setzero_si256(); \ + __m256i fours = _mm256_setzero_si256(); \ + __m256i eights = _mm256_setzero_si256(); \ + __m256i sixteens = _mm256_setzero_si256(); \ + __m256i twosA, twosB, foursA, foursB, eightsA, eightsB; \ + __m256i A1, A2; \ + const uint64_t limit = size - size % 16; \ + uint64_t i = 0; \ + for (; i < limit; i += 16) { \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i), \ + _mm256_lddqu_si256(data2 + i)); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 1), \ + _mm256_lddqu_si256(data2 + i + 1)); \ + CSA(&twosA, &ones, ones, A1, A2); \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 2), \ + _mm256_lddqu_si256(data2 + i + 2)); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 3), \ + _mm256_lddqu_si256(data2 + i + 3)); \ + CSA(&twosB, &ones, ones, A1, A2); \ + CSA(&foursA, &twos, twos, twosA, twosB); \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 4), \ + _mm256_lddqu_si256(data2 + i + 4)); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 5), \ + _mm256_lddqu_si256(data2 + i + 5)); \ + CSA(&twosA, &ones, ones, A1, A2); \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 6), \ + _mm256_lddqu_si256(data2 + i + 6)); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 7), \ + _mm256_lddqu_si256(data2 + i + 7)); \ + CSA(&twosB, &ones, ones, A1, A2); \ + CSA(&foursB, &twos, twos, twosA, twosB); \ + CSA(&eightsA, &fours, fours, foursA, foursB); \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 8), \ + _mm256_lddqu_si256(data2 + i + 8)); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 9), \ + _mm256_lddqu_si256(data2 + i + 9)); \ + CSA(&twosA, &ones, ones, A1, A2); \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 10), \ + _mm256_lddqu_si256(data2 + i + 10)); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 11), \ + _mm256_lddqu_si256(data2 + i + 11)); \ + CSA(&twosB, &ones, ones, A1, A2); \ + CSA(&foursA, &twos, twos, twosA, twosB); \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 12), \ + _mm256_lddqu_si256(data2 + i + 12)); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 13), \ + _mm256_lddqu_si256(data2 + i + 13)); \ + CSA(&twosA, &ones, ones, A1, A2); \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 14), \ + _mm256_lddqu_si256(data2 + i + 14)); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 15), \ + _mm256_lddqu_si256(data2 + i + 15)); \ + CSA(&twosB, &ones, ones, A1, A2); \ + CSA(&foursB, &twos, twos, twosA, twosB); \ + CSA(&eightsB, &fours, fours, foursA, foursB); \ + CSA(&sixteens, &eights, eights, eightsA, eightsB); \ + total = _mm256_add_epi64(total, popcount256(sixteens)); \ + } \ + total = _mm256_slli_epi64(total, 4); \ + total = _mm256_add_epi64(total, \ + _mm256_slli_epi64(popcount256(eights), 3)); \ + total = \ + _mm256_add_epi64(total, _mm256_slli_epi64(popcount256(fours), 2)); \ + total = \ + _mm256_add_epi64(total, _mm256_slli_epi64(popcount256(twos), 1)); \ + total = _mm256_add_epi64(total, popcount256(ones)); \ + for (; i < size; i++) { \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i), \ + _mm256_lddqu_si256(data2 + i)); \ + total = _mm256_add_epi64(total, popcount256(A1)); \ + } \ + return (uint64_t)(_mm256_extract_epi64(total, 0)) + \ + (uint64_t)(_mm256_extract_epi64(total, 1)) + \ + (uint64_t)(_mm256_extract_epi64(total, 2)) + \ + (uint64_t)(_mm256_extract_epi64(total, 3)); \ + } \ + static inline uint64_t avx2_harley_seal_popcount256andstore_##opname( \ + const __m256i *__restrict__ data1, const __m256i *__restrict__ data2, \ + __m256i *__restrict__ out, const uint64_t size) { \ + __m256i total = _mm256_setzero_si256(); \ + __m256i ones = _mm256_setzero_si256(); \ + __m256i twos = _mm256_setzero_si256(); \ + __m256i fours = _mm256_setzero_si256(); \ + __m256i eights = _mm256_setzero_si256(); \ + __m256i sixteens = _mm256_setzero_si256(); \ + __m256i twosA, twosB, foursA, foursB, eightsA, eightsB; \ + __m256i A1, A2; \ + const uint64_t limit = size - size % 16; \ + uint64_t i = 0; \ + for (; i < limit; i += 16) { \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i), \ + _mm256_lddqu_si256(data2 + i)); \ + _mm256_storeu_si256(out + i, A1); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 1), \ + _mm256_lddqu_si256(data2 + i + 1)); \ + _mm256_storeu_si256(out + i + 1, A2); \ + CSA(&twosA, &ones, ones, A1, A2); \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 2), \ + _mm256_lddqu_si256(data2 + i + 2)); \ + _mm256_storeu_si256(out + i + 2, A1); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 3), \ + _mm256_lddqu_si256(data2 + i + 3)); \ + _mm256_storeu_si256(out + i + 3, A2); \ + CSA(&twosB, &ones, ones, A1, A2); \ + CSA(&foursA, &twos, twos, twosA, twosB); \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 4), \ + _mm256_lddqu_si256(data2 + i + 4)); \ + _mm256_storeu_si256(out + i + 4, A1); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 5), \ + _mm256_lddqu_si256(data2 + i + 5)); \ + _mm256_storeu_si256(out + i + 5, A2); \ + CSA(&twosA, &ones, ones, A1, A2); \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 6), \ + _mm256_lddqu_si256(data2 + i + 6)); \ + _mm256_storeu_si256(out + i + 6, A1); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 7), \ + _mm256_lddqu_si256(data2 + i + 7)); \ + _mm256_storeu_si256(out + i + 7, A2); \ + CSA(&twosB, &ones, ones, A1, A2); \ + CSA(&foursB, &twos, twos, twosA, twosB); \ + CSA(&eightsA, &fours, fours, foursA, foursB); \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 8), \ + _mm256_lddqu_si256(data2 + i + 8)); \ + _mm256_storeu_si256(out + i + 8, A1); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 9), \ + _mm256_lddqu_si256(data2 + i + 9)); \ + _mm256_storeu_si256(out + i + 9, A2); \ + CSA(&twosA, &ones, ones, A1, A2); \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 10), \ + _mm256_lddqu_si256(data2 + i + 10)); \ + _mm256_storeu_si256(out + i + 10, A1); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 11), \ + _mm256_lddqu_si256(data2 + i + 11)); \ + _mm256_storeu_si256(out + i + 11, A2); \ + CSA(&twosB, &ones, ones, A1, A2); \ + CSA(&foursA, &twos, twos, twosA, twosB); \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 12), \ + _mm256_lddqu_si256(data2 + i + 12)); \ + _mm256_storeu_si256(out + i + 12, A1); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 13), \ + _mm256_lddqu_si256(data2 + i + 13)); \ + _mm256_storeu_si256(out + i + 13, A2); \ + CSA(&twosA, &ones, ones, A1, A2); \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 14), \ + _mm256_lddqu_si256(data2 + i + 14)); \ + _mm256_storeu_si256(out + i + 14, A1); \ + A2 = avx_intrinsic(_mm256_lddqu_si256(data1 + i + 15), \ + _mm256_lddqu_si256(data2 + i + 15)); \ + _mm256_storeu_si256(out + i + 15, A2); \ + CSA(&twosB, &ones, ones, A1, A2); \ + CSA(&foursB, &twos, twos, twosA, twosB); \ + CSA(&eightsB, &fours, fours, foursA, foursB); \ + CSA(&sixteens, &eights, eights, eightsA, eightsB); \ + total = _mm256_add_epi64(total, popcount256(sixteens)); \ + } \ + total = _mm256_slli_epi64(total, 4); \ + total = _mm256_add_epi64(total, \ + _mm256_slli_epi64(popcount256(eights), 3)); \ + total = \ + _mm256_add_epi64(total, _mm256_slli_epi64(popcount256(fours), 2)); \ + total = \ + _mm256_add_epi64(total, _mm256_slli_epi64(popcount256(twos), 1)); \ + total = _mm256_add_epi64(total, popcount256(ones)); \ + for (; i < size; i++) { \ + A1 = avx_intrinsic(_mm256_lddqu_si256(data1 + i), \ + _mm256_lddqu_si256(data2 + i)); \ + _mm256_storeu_si256(out + i, A1); \ + total = _mm256_add_epi64(total, popcount256(A1)); \ + } \ + return (uint64_t)(_mm256_extract_epi64(total, 0)) + \ + (uint64_t)(_mm256_extract_epi64(total, 1)) + \ + (uint64_t)(_mm256_extract_epi64(total, 2)) + \ + (uint64_t)(_mm256_extract_epi64(total, 3)); \ + } + +CROARING_TARGET_AVX2 +AVXPOPCNTFNC(or, _mm256_or_si256) +CROARING_UNTARGET_AVX2 + +CROARING_TARGET_AVX2 +AVXPOPCNTFNC(union, _mm256_or_si256) +CROARING_UNTARGET_AVX2 + +CROARING_TARGET_AVX2 +AVXPOPCNTFNC(and, _mm256_and_si256) +CROARING_UNTARGET_AVX2 + +CROARING_TARGET_AVX2 +AVXPOPCNTFNC(intersection, _mm256_and_si256) +CROARING_UNTARGET_AVX2 + +CROARING_TARGET_AVX2 +AVXPOPCNTFNC(xor, _mm256_xor_si256) +CROARING_UNTARGET_AVX2 + +CROARING_TARGET_AVX2 +AVXPOPCNTFNC(andnot, _mm256_andnot_si256) +CROARING_UNTARGET_AVX2 + +#define VPOPCNT_AND_ADD(ptr, i, accu) \ + const __m512i v##i = _mm512_loadu_si512((const __m512i *)ptr + i); \ + const __m512i p##i = _mm512_popcnt_epi64(v##i); \ + accu = _mm512_add_epi64(accu, p##i); + +#if CROARING_COMPILER_SUPPORTS_AVX512 +CROARING_TARGET_AVX512 +static inline uint64_t sum_epu64_256(const __m256i v) { + return (uint64_t)(_mm256_extract_epi64(v, 0)) + + (uint64_t)(_mm256_extract_epi64(v, 1)) + + (uint64_t)(_mm256_extract_epi64(v, 2)) + + (uint64_t)(_mm256_extract_epi64(v, 3)); +} + +static inline uint64_t simd_sum_epu64(const __m512i v) { + __m256i lo = _mm512_extracti64x4_epi64(v, 0); + __m256i hi = _mm512_extracti64x4_epi64(v, 1); + + return sum_epu64_256(lo) + sum_epu64_256(hi); +} + +static inline uint64_t avx512_vpopcount(const __m512i *data, + const uint64_t size) { + const uint64_t limit = size - size % 4; + __m512i total = _mm512_setzero_si512(); + uint64_t i = 0; + + for (; i < limit; i += 4) { + VPOPCNT_AND_ADD(data + i, 0, total); + VPOPCNT_AND_ADD(data + i, 1, total); + VPOPCNT_AND_ADD(data + i, 2, total); + VPOPCNT_AND_ADD(data + i, 3, total); + } + + for (; i < size; i++) { + total = _mm512_add_epi64( + total, _mm512_popcnt_epi64(_mm512_loadu_si512(data + i))); + } + + return simd_sum_epu64(total); +} +CROARING_UNTARGET_AVX512 +#endif + +#define AVXPOPCNTFNC512(opname, avx_intrinsic) \ + static inline uint64_t avx512_harley_seal_popcount512_##opname( \ + const __m512i *data1, const __m512i *data2, const uint64_t size) { \ + __m512i total = _mm512_setzero_si512(); \ + const uint64_t limit = size - size % 4; \ + uint64_t i = 0; \ + for (; i < limit; i += 4) { \ + __m512i a1 = avx_intrinsic(_mm512_loadu_si512(data1 + i), \ + _mm512_loadu_si512(data2 + i)); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a1)); \ + __m512i a2 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 1), \ + _mm512_loadu_si512(data2 + i + 1)); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a2)); \ + __m512i a3 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 2), \ + _mm512_loadu_si512(data2 + i + 2)); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a3)); \ + __m512i a4 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 3), \ + _mm512_loadu_si512(data2 + i + 3)); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a4)); \ + } \ + for (; i < size; i++) { \ + __m512i a = avx_intrinsic(_mm512_loadu_si512(data1 + i), \ + _mm512_loadu_si512(data2 + i)); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a)); \ + } \ + return simd_sum_epu64(total); \ + } \ + static inline uint64_t avx512_harley_seal_popcount512andstore_##opname( \ + const __m512i *__restrict__ data1, const __m512i *__restrict__ data2, \ + __m512i *__restrict__ out, const uint64_t size) { \ + __m512i total = _mm512_setzero_si512(); \ + const uint64_t limit = size - size % 4; \ + uint64_t i = 0; \ + for (; i < limit; i += 4) { \ + __m512i a1 = avx_intrinsic(_mm512_loadu_si512(data1 + i), \ + _mm512_loadu_si512(data2 + i)); \ + _mm512_storeu_si512(out + i, a1); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a1)); \ + __m512i a2 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 1), \ + _mm512_loadu_si512(data2 + i + 1)); \ + _mm512_storeu_si512(out + i + 1, a2); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a2)); \ + __m512i a3 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 2), \ + _mm512_loadu_si512(data2 + i + 2)); \ + _mm512_storeu_si512(out + i + 2, a3); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a3)); \ + __m512i a4 = avx_intrinsic(_mm512_loadu_si512(data1 + i + 3), \ + _mm512_loadu_si512(data2 + i + 3)); \ + _mm512_storeu_si512(out + i + 3, a4); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a4)); \ + } \ + for (; i < size; i++) { \ + __m512i a = avx_intrinsic(_mm512_loadu_si512(data1 + i), \ + _mm512_loadu_si512(data2 + i)); \ + _mm512_storeu_si512(out + i, a); \ + total = _mm512_add_epi64(total, _mm512_popcnt_epi64(a)); \ + } \ + return simd_sum_epu64(total); \ + } + +#if CROARING_COMPILER_SUPPORTS_AVX512 +CROARING_TARGET_AVX512 +AVXPOPCNTFNC512(or, _mm512_or_si512) +AVXPOPCNTFNC512(union, _mm512_or_si512) +AVXPOPCNTFNC512(and, _mm512_and_si512) +AVXPOPCNTFNC512(intersection, _mm512_and_si512) +AVXPOPCNTFNC512(xor, _mm512_xor_si512) +AVXPOPCNTFNC512(andnot, _mm512_andnot_si512) +CROARING_UNTARGET_AVX512 +#endif +/*** + * END Harley-Seal popcount functions. + */ + +#endif // CROARING_IS_X64 + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif +#endif diff --git a/contrib/libs/croaring/include/roaring/containers/array.h b/contrib/libs/croaring/include/roaring/containers/array.h new file mode 100644 index 00000000000..8d5c84a0cb5 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/containers/array.h @@ -0,0 +1,521 @@ +/* + * array.h + * + */ + +#ifndef INCLUDE_CONTAINERS_ARRAY_H_ +#define INCLUDE_CONTAINERS_ARRAY_H_ + +#include <string.h> + +#include <roaring/roaring_types.h> // roaring_iterator + +// Include other headers after roaring_types.h +#include <roaring/array_util.h> // binarySearch()/memequals() for inlining +#include <roaring/containers/container_defs.h> // container_t, perfparameters +#include <roaring/portability.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { + +// Note: in pure C++ code, you should avoid putting `using` in header files +using api::roaring_iterator; +using api::roaring_iterator64; + +namespace internal { +#endif + +/* Containers with DEFAULT_MAX_SIZE or less integers should be arrays */ +enum { DEFAULT_MAX_SIZE = 4096 }; + +/* struct array_container - sparse representation of a bitmap + * + * @cardinality: number of indices in `array` (and the bitmap) + * @capacity: allocated size of `array` + * @array: sorted list of integers + */ +STRUCT_CONTAINER(array_container_s) { + int32_t cardinality; + int32_t capacity; + uint16_t *array; +}; + +typedef struct array_container_s array_container_t; + +#define CAST_array(c) CAST(array_container_t *, c) // safer downcast +#define const_CAST_array(c) CAST(const array_container_t *, c) +#define movable_CAST_array(c) movable_CAST(array_container_t **, c) + +/* Create a new array with default. Return NULL in case of failure. See also + * array_container_create_given_capacity. */ +array_container_t *array_container_create(void); + +/* Create a new array with a specified capacity size. Return NULL in case of + * failure. */ +array_container_t *array_container_create_given_capacity(int32_t size); + +/* Create a new array containing all values in [min,max). */ +array_container_t *array_container_create_range(uint32_t min, uint32_t max); + +/* + * Shrink the capacity to the actual size, return the number of bytes saved. + */ +int array_container_shrink_to_fit(array_container_t *src); + +/* Free memory owned by `array'. */ +void array_container_free(array_container_t *array); + +/* Duplicate container */ +array_container_t *array_container_clone(const array_container_t *src); + +/* Get the cardinality of `array'. */ +ALLOW_UNALIGNED +static inline int array_container_cardinality(const array_container_t *array) { + return array->cardinality; +} + +static inline bool array_container_nonzero_cardinality( + const array_container_t *array) { + return array->cardinality > 0; +} + +/* Copy one container into another. We assume that they are distinct. */ +void array_container_copy(const array_container_t *src, array_container_t *dst); + +/* Add all the values in [min,max) (included) at a distance k*step from min. + The container must have a size less or equal to DEFAULT_MAX_SIZE after this + addition. */ +void array_container_add_from_range(array_container_t *arr, uint32_t min, + uint32_t max, uint16_t step); + +static inline bool array_container_empty(const array_container_t *array) { + return array->cardinality == 0; +} + +/* check whether the cardinality is equal to the capacity (this does not mean + * that it contains 1<<16 elements) */ +static inline bool array_container_full(const array_container_t *array) { + return array->cardinality == array->capacity; +} + +/* Compute the union of `src_1' and `src_2' and write the result to `dst' + * It is assumed that `dst' is distinct from both `src_1' and `src_2'. */ +void array_container_union(const array_container_t *src_1, + const array_container_t *src_2, + array_container_t *dst); + +/* symmetric difference, see array_container_union */ +void array_container_xor(const array_container_t *array_1, + const array_container_t *array_2, + array_container_t *out); + +/* Computes the intersection of src_1 and src_2 and write the result to + * dst. It is assumed that dst is distinct from both src_1 and src_2. */ +void array_container_intersection(const array_container_t *src_1, + const array_container_t *src_2, + array_container_t *dst); + +/* Check whether src_1 and src_2 intersect. */ +bool array_container_intersect(const array_container_t *src_1, + const array_container_t *src_2); + +/* computers the size of the intersection between two arrays. + */ +int array_container_intersection_cardinality(const array_container_t *src_1, + const array_container_t *src_2); + +/* computes the intersection of array1 and array2 and write the result to + * array1. + * */ +void array_container_intersection_inplace(array_container_t *src_1, + const array_container_t *src_2); + +/* + * Write out the 16-bit integers contained in this container as a list of 32-bit + * integers using base + * as the starting value (it might be expected that base has zeros in its 16 + * least significant bits). + * The function returns the number of values written. + * The caller is responsible for allocating enough memory in out. + */ +int array_container_to_uint32_array(void *vout, const array_container_t *cont, + uint32_t base); + +/* Compute the number of runs */ +int32_t array_container_number_of_runs(const array_container_t *ac); + +/* + * Print this container using printf (useful for debugging). + */ +void array_container_printf(const array_container_t *v); + +/* + * Print this container using printf as a comma-separated list of 32-bit + * integers starting at base. + */ +void array_container_printf_as_uint32_array(const array_container_t *v, + uint32_t base); + +bool array_container_validate(const array_container_t *v, const char **reason); + +/** + * Return the serialized size in bytes of a container having cardinality "card". + */ +static inline int32_t array_container_serialized_size_in_bytes(int32_t card) { + return card * 2 + 2; +} + +/** + * Increase capacity to at least min. + * Whether the existing data needs to be copied over depends on the "preserve" + * parameter. If preserve is false, then the new content will be uninitialized, + * otherwise the old content is copied. + */ +void array_container_grow(array_container_t *container, int32_t min, + bool preserve); + +bool array_container_iterate(const array_container_t *cont, uint32_t base, + roaring_iterator iterator, void *ptr); +bool array_container_iterate64(const array_container_t *cont, uint32_t base, + roaring_iterator64 iterator, uint64_t high_bits, + void *ptr); + +/** + * Writes the underlying array to buf, outputs how many bytes were written. + * This is meant to be byte-by-byte compatible with the Java and Go versions of + * Roaring. + * The number of bytes written should be + * array_container_size_in_bytes(container). + * + */ +int32_t array_container_write(const array_container_t *container, char *buf); +/** + * Reads the instance from buf, outputs how many bytes were read. + * This is meant to be byte-by-byte compatible with the Java and Go versions of + * Roaring. + * The number of bytes read should be array_container_size_in_bytes(container). + * You need to provide the (known) cardinality. + */ +int32_t array_container_read(int32_t cardinality, array_container_t *container, + const char *buf); + +/** + * Return the serialized size in bytes of a container (see + * bitset_container_write) + * This is meant to be compatible with the Java and Go versions of Roaring and + * assumes + * that the cardinality of the container is already known. + * + */ +ALLOW_UNALIGNED +static inline int32_t array_container_size_in_bytes( + const array_container_t *container) { + return container->cardinality * sizeof(uint16_t); +} + +/** + * Return true if the two arrays have the same content. + */ +ALLOW_UNALIGNED +static inline bool array_container_equals(const array_container_t *container1, + const array_container_t *container2) { + if (container1->cardinality != container2->cardinality) { + return false; + } + return memequals(container1->array, container2->array, + container1->cardinality * 2); +} + +/** + * Return true if container1 is a subset of container2. + */ +bool array_container_is_subset(const array_container_t *container1, + const array_container_t *container2); + +/** + * If the element of given rank is in this container, supposing that the first + * element has rank start_rank, then the function returns true and sets element + * accordingly. + * Otherwise, it returns false and update start_rank. + */ +static inline bool array_container_select(const array_container_t *container, + uint32_t *start_rank, uint32_t rank, + uint32_t *element) { + int card = array_container_cardinality(container); + if (*start_rank + card <= rank) { + *start_rank += card; + return false; + } else { + *element = container->array[rank - *start_rank]; + return true; + } +} + +/* Computes the difference of array1 and array2 and write the result + * to array out. + * Array out does not need to be distinct from array_1 + */ +void array_container_andnot(const array_container_t *array_1, + const array_container_t *array_2, + array_container_t *out); + +/* Append x to the set. Assumes that the value is larger than any preceding + * values. */ +static inline void array_container_append(array_container_t *arr, + uint16_t pos) { + const int32_t capacity = arr->capacity; + + if (array_container_full(arr)) { + array_container_grow(arr, capacity + 1, true); + } + + arr->array[arr->cardinality++] = pos; +} + +/** + * Add value to the set if final cardinality doesn't exceed max_cardinality. + * Return code: + * 1 -- value was added + * 0 -- value was already present + * -1 -- value was not added because cardinality would exceed max_cardinality + */ +static inline int array_container_try_add(array_container_t *arr, + uint16_t value, + int32_t max_cardinality) { + const int32_t cardinality = arr->cardinality; + + // best case, we can append. + if ((array_container_empty(arr) || arr->array[cardinality - 1] < value) && + cardinality < max_cardinality) { + array_container_append(arr, value); + return 1; + } + + const int32_t loc = binarySearch(arr->array, cardinality, value); + + if (loc >= 0) { + return 0; + } else if (cardinality < max_cardinality) { + if (array_container_full(arr)) { + array_container_grow(arr, arr->capacity + 1, true); + } + const int32_t insert_idx = -loc - 1; + memmove(arr->array + insert_idx + 1, arr->array + insert_idx, + (cardinality - insert_idx) * sizeof(uint16_t)); + arr->array[insert_idx] = value; + arr->cardinality++; + return 1; + } else { + return -1; + } +} + +/* Add value to the set. Returns true if x was not already present. */ +static inline bool array_container_add(array_container_t *arr, uint16_t value) { + return array_container_try_add(arr, value, INT32_MAX) == 1; +} + +/* Remove x from the set. Returns true if x was present. */ +static inline bool array_container_remove(array_container_t *arr, + uint16_t pos) { + const int32_t idx = binarySearch(arr->array, arr->cardinality, pos); + const bool is_present = idx >= 0; + if (is_present) { + memmove(arr->array + idx, arr->array + idx + 1, + (arr->cardinality - idx - 1) * sizeof(uint16_t)); + arr->cardinality--; + } + + return is_present; +} + +/* Check whether x is present. */ +inline bool array_container_contains(const array_container_t *arr, + uint16_t pos) { + // return binarySearch(arr->array, arr->cardinality, pos) >= 0; + // binary search with fallback to linear search for short ranges + int32_t low = 0; + const uint16_t *carr = (const uint16_t *)arr->array; + int32_t high = arr->cardinality - 1; + // while (high - low >= 0) { + while (high >= low + 16) { + int32_t middleIndex = (low + high) >> 1; + uint16_t middleValue = carr[middleIndex]; + if (middleValue < pos) { + low = middleIndex + 1; + } else if (middleValue > pos) { + high = middleIndex - 1; + } else { + return true; + } + } + + for (int i = low; i <= high; i++) { + uint16_t v = carr[i]; + if (v == pos) { + return true; + } + if (v > pos) return false; + } + return false; +} + +void array_container_offset(const array_container_t *c, container_t **loc, + container_t **hic, uint16_t offset); + +//* Check whether a range of values from range_start (included) to range_end +//(excluded) is present. */ +static inline bool array_container_contains_range(const array_container_t *arr, + uint32_t range_start, + uint32_t range_end) { + const int32_t range_count = range_end - range_start; + const uint16_t rs_included = (uint16_t)range_start; + const uint16_t re_included = (uint16_t)(range_end - 1); + + // Empty range is always included + if (range_count <= 0) { + return true; + } + if (range_count > arr->cardinality) { + return false; + } + + const int32_t start = + binarySearch(arr->array, arr->cardinality, rs_included); + // If this sorted array contains all items in the range: + // * the start item must be found + // * the last item in range range_count must exist, and be the expected end + // value + return (start >= 0) && (arr->cardinality >= start + range_count) && + (arr->array[start + range_count - 1] == re_included); +} + +/* Returns the smallest value (assumes not empty) */ +inline uint16_t array_container_minimum(const array_container_t *arr) { + if (arr->cardinality == 0) return 0; + return arr->array[0]; +} + +/* Returns the largest value (assumes not empty) */ +inline uint16_t array_container_maximum(const array_container_t *arr) { + if (arr->cardinality == 0) return 0; + return arr->array[arr->cardinality - 1]; +} + +/* Returns the number of values equal or smaller than x */ +inline int array_container_rank(const array_container_t *arr, uint16_t x) { + const int32_t idx = binarySearch(arr->array, arr->cardinality, x); + const bool is_present = idx >= 0; + if (is_present) { + return idx + 1; + } else { + return -idx - 1; + } +} + +/* bulk version of array_container_rank(); return number of consumed elements + */ +inline uint32_t array_container_rank_many(const array_container_t *arr, + uint64_t start_rank, + const uint32_t *begin, + const uint32_t *end, uint64_t *ans) { + const uint16_t high = (uint16_t)((*begin) >> 16); + uint32_t pos = 0; + const uint32_t *iter = begin; + for (; iter != end; iter++) { + uint32_t x = *iter; + uint16_t xhigh = (uint16_t)(x >> 16); + if (xhigh != high) return iter - begin; // stop at next container + + const int32_t idx = + binarySearch(arr->array + pos, arr->cardinality - pos, (uint16_t)x); + const bool is_present = idx >= 0; + if (is_present) { + *(ans++) = start_rank + pos + (idx + 1); + pos = idx + 1; + } else { + *(ans++) = start_rank + pos + (-idx - 1); + } + } + return iter - begin; +} + +/* Returns the index of x , if not exsist return -1 */ +inline int array_container_get_index(const array_container_t *arr, uint16_t x) { + const int32_t idx = binarySearch(arr->array, arr->cardinality, x); + const bool is_present = idx >= 0; + if (is_present) { + return idx; + } else { + return -1; + } +} + +/* Returns the index of the first value equal or larger than x, or -1 */ +inline int array_container_index_equalorlarger(const array_container_t *arr, + uint16_t x) { + const int32_t idx = binarySearch(arr->array, arr->cardinality, x); + const bool is_present = idx >= 0; + if (is_present) { + return idx; + } else { + int32_t candidate = -idx - 1; + if (candidate < arr->cardinality) return candidate; + return -1; + } +} + +/* + * Adds all values in range [min,max] using hint: + * nvals_less is the number of array values less than $min + * nvals_greater is the number of array values greater than $max + */ +static inline void array_container_add_range_nvals(array_container_t *array, + uint32_t min, uint32_t max, + int32_t nvals_less, + int32_t nvals_greater) { + int32_t union_cardinality = nvals_less + (max - min + 1) + nvals_greater; + if (union_cardinality > array->capacity) { + array_container_grow(array, union_cardinality, true); + } + memmove(&(array->array[union_cardinality - nvals_greater]), + &(array->array[array->cardinality - nvals_greater]), + nvals_greater * sizeof(uint16_t)); + for (uint32_t i = 0; i <= max - min; i++) { + array->array[nvals_less + i] = (uint16_t)(min + i); + } + array->cardinality = union_cardinality; +} + +/** + * Adds all values in range [min,max]. This function is currently unused + * and left as a documentation. + */ +/*static inline void array_container_add_range(array_container_t *array, + uint32_t min, uint32_t max) { + int32_t nvals_greater = count_greater(array->array, array->cardinality, +max); int32_t nvals_less = count_less(array->array, array->cardinality - +nvals_greater, min); array_container_add_range_nvals(array, min, max, +nvals_less, nvals_greater); +}*/ + +/* + * Removes all elements array[pos] .. array[pos+count-1] + */ +static inline void array_container_remove_range(array_container_t *array, + uint32_t pos, uint32_t count) { + if (count != 0) { + memmove(&(array->array[pos]), &(array->array[pos + count]), + (array->cardinality - pos - count) * sizeof(uint16_t)); + array->cardinality -= count; + } +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#endif /* INCLUDE_CONTAINERS_ARRAY_H_ */ diff --git a/contrib/libs/croaring/include/roaring/containers/bitset.h b/contrib/libs/croaring/include/roaring/containers/bitset.h new file mode 100644 index 00000000000..a738a1ad63a --- /dev/null +++ b/contrib/libs/croaring/include/roaring/containers/bitset.h @@ -0,0 +1,514 @@ +/* + * bitset.h + * + */ + +#ifndef INCLUDE_CONTAINERS_BITSET_H_ +#define INCLUDE_CONTAINERS_BITSET_H_ + +#include <stdbool.h> +#include <stdint.h> + +#include <roaring/roaring_types.h> // roaring_iterator + +// Include other headers after roaring_types.h +#include <roaring/containers/container_defs.h> // container_t, perfparameters +#include <roaring/portability.h> +#include <roaring/roaring_types.h> // roaring_iterator +#include <roaring/utilasm.h> // ASM_XXX macros + +#ifdef __cplusplus +extern "C" { +namespace roaring { + +// Note: in pure C++ code, you should avoid putting `using` in header files +using api::roaring_iterator; +using api::roaring_iterator64; + +namespace internal { +#endif + +enum { + BITSET_CONTAINER_SIZE_IN_WORDS = (1 << 16) / 64, + BITSET_UNKNOWN_CARDINALITY = -1 +}; + +STRUCT_CONTAINER(bitset_container_s) { + int32_t cardinality; + uint64_t *words; +}; + +typedef struct bitset_container_s bitset_container_t; + +#define CAST_bitset(c) CAST(bitset_container_t *, c) // safer downcast +#define const_CAST_bitset(c) CAST(const bitset_container_t *, c) +#define movable_CAST_bitset(c) movable_CAST(bitset_container_t **, c) + +/* Create a new bitset. Return NULL in case of failure. */ +bitset_container_t *bitset_container_create(void); + +/* Free memory. */ +void bitset_container_free(bitset_container_t *bitset); + +/* Clear bitset (sets bits to 0). */ +void bitset_container_clear(bitset_container_t *bitset); + +/* Set all bits to 1. */ +void bitset_container_set_all(bitset_container_t *bitset); + +/* Duplicate bitset */ +bitset_container_t *bitset_container_clone(const bitset_container_t *src); + +/* Set the bit in [begin,end). WARNING: as of April 2016, this method is slow + * and + * should not be used in performance-sensitive code. Ever. */ +void bitset_container_set_range(bitset_container_t *bitset, uint32_t begin, + uint32_t end); + +#if defined(CROARING_ASMBITMANIPOPTIMIZATION) && defined(__AVX2__) +/* Set the ith bit. */ +static inline void bitset_container_set(bitset_container_t *bitset, + uint16_t pos) { + uint64_t shift = 6; + uint64_t offset; + uint64_t p = pos; + ASM_SHIFT_RIGHT(p, shift, offset); + uint64_t load = bitset->words[offset]; + ASM_SET_BIT_INC_WAS_CLEAR(load, p, bitset->cardinality); + bitset->words[offset] = load; +} + +/* Unset the ith bit. Currently unused. Could be used for optimization. */ +/*static inline void bitset_container_unset(bitset_container_t *bitset, + uint16_t pos) { + uint64_t shift = 6; + uint64_t offset; + uint64_t p = pos; + ASM_SHIFT_RIGHT(p, shift, offset); + uint64_t load = bitset->words[offset]; + ASM_CLEAR_BIT_DEC_WAS_SET(load, p, bitset->cardinality); + bitset->words[offset] = load; +}*/ + +/* Add `pos' to `bitset'. Returns true if `pos' was not present. Might be slower + * than bitset_container_set. */ +static inline bool bitset_container_add(bitset_container_t *bitset, + uint16_t pos) { + uint64_t shift = 6; + uint64_t offset; + uint64_t p = pos; + ASM_SHIFT_RIGHT(p, shift, offset); + uint64_t load = bitset->words[offset]; + // could be possibly slightly further optimized + const int32_t oldcard = bitset->cardinality; + ASM_SET_BIT_INC_WAS_CLEAR(load, p, bitset->cardinality); + bitset->words[offset] = load; + return bitset->cardinality - oldcard; +} + +/* Remove `pos' from `bitset'. Returns true if `pos' was present. Might be + * slower than bitset_container_unset. */ +static inline bool bitset_container_remove(bitset_container_t *bitset, + uint16_t pos) { + uint64_t shift = 6; + uint64_t offset; + uint64_t p = pos; + ASM_SHIFT_RIGHT(p, shift, offset); + uint64_t load = bitset->words[offset]; + // could be possibly slightly further optimized + const int32_t oldcard = bitset->cardinality; + ASM_CLEAR_BIT_DEC_WAS_SET(load, p, bitset->cardinality); + bitset->words[offset] = load; + return oldcard - bitset->cardinality; +} + +/* Get the value of the ith bit. */ +inline bool bitset_container_get(const bitset_container_t *bitset, + uint16_t pos) { + uint64_t word = bitset->words[pos >> 6]; + const uint64_t p = pos; + ASM_INPLACESHIFT_RIGHT(word, p); + return word & 1; +} + +#else + +/* Set the ith bit. */ +static inline void bitset_container_set(bitset_container_t *bitset, + uint16_t pos) { + const uint64_t old_word = bitset->words[pos >> 6]; + const int index = pos & 63; + const uint64_t new_word = old_word | (UINT64_C(1) << index); + bitset->cardinality += (uint32_t)((old_word ^ new_word) >> index); + bitset->words[pos >> 6] = new_word; +} + +/* Unset the ith bit. Currently unused. */ +/*static inline void bitset_container_unset(bitset_container_t *bitset, + uint16_t pos) { + const uint64_t old_word = bitset->words[pos >> 6]; + const int index = pos & 63; + const uint64_t new_word = old_word & (~(UINT64_C(1) << index)); + bitset->cardinality -= (uint32_t)((old_word ^ new_word) >> index); + bitset->words[pos >> 6] = new_word; +}*/ + +/* Add `pos' to `bitset'. Returns true if `pos' was not present. Might be slower + * than bitset_container_set. */ +static inline bool bitset_container_add(bitset_container_t *bitset, + uint16_t pos) { + const uint64_t old_word = bitset->words[pos >> 6]; + const int index = pos & 63; + const uint64_t new_word = old_word | (UINT64_C(1) << index); + const uint64_t increment = (old_word ^ new_word) >> index; + bitset->cardinality += (uint32_t)increment; + bitset->words[pos >> 6] = new_word; + return increment > 0; +} + +/* Remove `pos' from `bitset'. Returns true if `pos' was present. Might be + * slower than bitset_container_unset. */ +static inline bool bitset_container_remove(bitset_container_t *bitset, + uint16_t pos) { + const uint64_t old_word = bitset->words[pos >> 6]; + const int index = pos & 63; + const uint64_t new_word = old_word & (~(UINT64_C(1) << index)); + const uint64_t increment = (old_word ^ new_word) >> index; + bitset->cardinality -= (uint32_t)increment; + bitset->words[pos >> 6] = new_word; + return increment > 0; +} + +/* Get the value of the ith bit. */ +inline bool bitset_container_get(const bitset_container_t *bitset, + uint16_t pos) { + const uint64_t word = bitset->words[pos >> 6]; + return (word >> (pos & 63)) & 1; +} + +#endif + +/* + * Check if all bits are set in a range of positions from pos_start (included) + * to pos_end (excluded). + */ +static inline bool bitset_container_get_range(const bitset_container_t *bitset, + uint32_t pos_start, + uint32_t pos_end) { + const uint32_t start = pos_start >> 6; + const uint32_t end = pos_end >> 6; + + const uint64_t first = ~((1ULL << (pos_start & 0x3F)) - 1); + const uint64_t last = (1ULL << (pos_end & 0x3F)) - 1; + + if (start == end) + return ((bitset->words[end] & first & last) == (first & last)); + if ((bitset->words[start] & first) != first) return false; + + if ((end < BITSET_CONTAINER_SIZE_IN_WORDS) && + ((bitset->words[end] & last) != last)) { + return false; + } + + for (uint32_t i = start + 1; + (i < BITSET_CONTAINER_SIZE_IN_WORDS) && (i < end); ++i) { + if (bitset->words[i] != UINT64_C(0xFFFFFFFFFFFFFFFF)) return false; + } + + return true; +} + +/* Check whether `bitset' is present in `array'. Calls bitset_container_get. */ +inline bool bitset_container_contains(const bitset_container_t *bitset, + uint16_t pos) { + return bitset_container_get(bitset, pos); +} + +/* + * Check whether a range of bits from position `pos_start' (included) to + * `pos_end' (excluded) is present in `bitset'. Calls bitset_container_get_all. + */ +static inline bool bitset_container_contains_range( + const bitset_container_t *bitset, uint32_t pos_start, uint32_t pos_end) { + return bitset_container_get_range(bitset, pos_start, pos_end); +} + +/* Get the number of bits set */ +ALLOW_UNALIGNED +static inline int bitset_container_cardinality( + const bitset_container_t *bitset) { + return bitset->cardinality; +} + +/* Copy one container into another. We assume that they are distinct. */ +void bitset_container_copy(const bitset_container_t *source, + bitset_container_t *dest); + +/* Add all the values [min,max) at a distance k*step from min: min, + * min+step,.... */ +void bitset_container_add_from_range(bitset_container_t *bitset, uint32_t min, + uint32_t max, uint16_t step); + +/* Get the number of bits set (force computation). This does not modify bitset. + * To update the cardinality, you should do + * bitset->cardinality = bitset_container_compute_cardinality(bitset).*/ +int bitset_container_compute_cardinality(const bitset_container_t *bitset); + +/* Check whether this bitset is empty, + * it never modifies the bitset struct. */ +static inline bool bitset_container_empty(const bitset_container_t *bitset) { + if (bitset->cardinality == BITSET_UNKNOWN_CARDINALITY) { + for (int i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i++) { + if ((bitset->words[i]) != 0) return false; + } + return true; + } + return bitset->cardinality == 0; +} + +/* Get whether there is at least one bit set (see bitset_container_empty for + the reverse), the bitset is never modified */ +static inline bool bitset_container_const_nonzero_cardinality( + const bitset_container_t *bitset) { + return !bitset_container_empty(bitset); +} + +/* + * Check whether the two bitsets intersect + */ +bool bitset_container_intersect(const bitset_container_t *src_1, + const bitset_container_t *src_2); + +/* Computes the union of bitsets `src_1' and `src_2' into `dst' and return the + * cardinality. */ +int bitset_container_or(const bitset_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* Computes the union of bitsets `src_1' and `src_2' and return the cardinality. + */ +int bitset_container_or_justcard(const bitset_container_t *src_1, + const bitset_container_t *src_2); + +/* Computes the union of bitsets `src_1' and `src_2' into `dst' and return the + * cardinality. Same as bitset_container_or. */ +int bitset_container_union(const bitset_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* Computes the union of bitsets `src_1' and `src_2' and return the + * cardinality. Same as bitset_container_or_justcard. */ +int bitset_container_union_justcard(const bitset_container_t *src_1, + const bitset_container_t *src_2); + +/* Computes the union of bitsets `src_1' and `src_2' into `dst', but does + * not update the cardinality. Provided to optimize chained operations. */ +int bitset_container_union_nocard(const bitset_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* Computes the union of bitsets `src_1' and `src_2' into `dst', but does not + * update the cardinality. Provided to optimize chained operations. */ +int bitset_container_or_nocard(const bitset_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* Computes the intersection of bitsets `src_1' and `src_2' into `dst' and + * return the cardinality. */ +int bitset_container_and(const bitset_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* Computes the intersection of bitsets `src_1' and `src_2' and return the + * cardinality. */ +int bitset_container_and_justcard(const bitset_container_t *src_1, + const bitset_container_t *src_2); + +/* Computes the intersection of bitsets `src_1' and `src_2' into `dst' and + * return the cardinality. Same as bitset_container_and. */ +int bitset_container_intersection(const bitset_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* Computes the intersection of bitsets `src_1' and `src_2' and return the + * cardinality. Same as bitset_container_and_justcard. */ +int bitset_container_intersection_justcard(const bitset_container_t *src_1, + const bitset_container_t *src_2); + +/* Computes the intersection of bitsets `src_1' and `src_2' into `dst', but does + * not update the cardinality. Provided to optimize chained operations. */ +int bitset_container_intersection_nocard(const bitset_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* Computes the intersection of bitsets `src_1' and `src_2' into `dst', but does + * not update the cardinality. Provided to optimize chained operations. */ +int bitset_container_and_nocard(const bitset_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* Computes the exclusive or of bitsets `src_1' and `src_2' into `dst' and + * return the cardinality. */ +int bitset_container_xor(const bitset_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* Computes the exclusive or of bitsets `src_1' and `src_2' and return the + * cardinality. */ +int bitset_container_xor_justcard(const bitset_container_t *src_1, + const bitset_container_t *src_2); + +/* Computes the exclusive or of bitsets `src_1' and `src_2' into `dst', but does + * not update the cardinality. Provided to optimize chained operations. */ +int bitset_container_xor_nocard(const bitset_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* Computes the and not of bitsets `src_1' and `src_2' into `dst' and return the + * cardinality. */ +int bitset_container_andnot(const bitset_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* Computes the and not of bitsets `src_1' and `src_2' and return the + * cardinality. */ +int bitset_container_andnot_justcard(const bitset_container_t *src_1, + const bitset_container_t *src_2); + +/* Computes the and not or of bitsets `src_1' and `src_2' into `dst', but does + * not update the cardinality. Provided to optimize chained operations. */ +int bitset_container_andnot_nocard(const bitset_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +void bitset_container_offset(const bitset_container_t *c, container_t **loc, + container_t **hic, uint16_t offset); +/* + * Write out the 16-bit integers contained in this container as a list of 32-bit + * integers using base + * as the starting value (it might be expected that base has zeros in its 16 + * least significant bits). + * The function returns the number of values written. + * The caller is responsible for allocating enough memory in out. + * The out pointer should point to enough memory (the cardinality times 32 + * bits). + */ +int bitset_container_to_uint32_array(uint32_t *out, + const bitset_container_t *bc, + uint32_t base); + +/* + * Print this container using printf (useful for debugging). + */ +void bitset_container_printf(const bitset_container_t *v); + +/* + * Print this container using printf as a comma-separated list of 32-bit + * integers starting at base. + */ +void bitset_container_printf_as_uint32_array(const bitset_container_t *v, + uint32_t base); + +bool bitset_container_validate(const bitset_container_t *v, + const char **reason); + +/** + * Return the serialized size in bytes of a container. + */ +static inline int32_t bitset_container_serialized_size_in_bytes(void) { + return BITSET_CONTAINER_SIZE_IN_WORDS * 8; +} + +/** + * Return the the number of runs. + */ +int bitset_container_number_of_runs(bitset_container_t *bc); + +bool bitset_container_iterate(const bitset_container_t *cont, uint32_t base, + roaring_iterator iterator, void *ptr); +bool bitset_container_iterate64(const bitset_container_t *cont, uint32_t base, + roaring_iterator64 iterator, uint64_t high_bits, + void *ptr); + +/** + * Writes the underlying array to buf, outputs how many bytes were written. + * This is meant to be byte-by-byte compatible with the Java and Go versions of + * Roaring. + * The number of bytes written should be + * bitset_container_size_in_bytes(container). + */ +int32_t bitset_container_write(const bitset_container_t *container, char *buf); + +/** + * Reads the instance from buf, outputs how many bytes were read. + * This is meant to be byte-by-byte compatible with the Java and Go versions of + * Roaring. + * The number of bytes read should be bitset_container_size_in_bytes(container). + * You need to provide the (known) cardinality. + */ +int32_t bitset_container_read(int32_t cardinality, + bitset_container_t *container, const char *buf); +/** + * Return the serialized size in bytes of a container (see + * bitset_container_write). + * This is meant to be compatible with the Java and Go versions of Roaring and + * assumes + * that the cardinality of the container is already known or can be computed. + */ +static inline int32_t bitset_container_size_in_bytes( + const bitset_container_t *container) { + (void)container; + return BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); +} + +/** + * Return true if the two containers have the same content. + */ +bool bitset_container_equals(const bitset_container_t *container1, + const bitset_container_t *container2); + +/** + * Return true if container1 is a subset of container2. + */ +bool bitset_container_is_subset(const bitset_container_t *container1, + const bitset_container_t *container2); + +/** + * If the element of given rank is in this container, supposing that the first + * element has rank start_rank, then the function returns true and sets element + * accordingly. + * Otherwise, it returns false and update start_rank. + */ +bool bitset_container_select(const bitset_container_t *container, + uint32_t *start_rank, uint32_t rank, + uint32_t *element); + +/* Returns the smallest value (assumes not empty) */ +uint16_t bitset_container_minimum(const bitset_container_t *container); + +/* Returns the largest value (assumes not empty) */ +uint16_t bitset_container_maximum(const bitset_container_t *container); + +/* Returns the number of values equal or smaller than x */ +int bitset_container_rank(const bitset_container_t *container, uint16_t x); + +/* bulk version of bitset_container_rank(); return number of consumed elements + */ +uint32_t bitset_container_rank_many(const bitset_container_t *container, + uint64_t start_rank, const uint32_t *begin, + const uint32_t *end, uint64_t *ans); + +/* Returns the index of x , if not exsist return -1 */ +int bitset_container_get_index(const bitset_container_t *container, uint16_t x); + +/* Returns the index of the first value equal or larger than x, or -1 */ +int bitset_container_index_equalorlarger(const bitset_container_t *container, + uint16_t x); + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#endif /* INCLUDE_CONTAINERS_BITSET_H_ */ diff --git a/contrib/libs/croaring/include/roaring/containers/container_defs.h b/contrib/libs/croaring/include/roaring/containers/container_defs.h new file mode 100644 index 00000000000..402bad787d2 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/containers/container_defs.h @@ -0,0 +1,100 @@ +/* + * container_defs.h + * + * Unlike containers.h (which is a file aggregating all the container includes, + * like array.h, bitset.h, and run.h) this is a file included BY those headers + * to do things like define the container base class `container_t`. + */ + +#ifndef INCLUDE_CONTAINERS_CONTAINER_DEFS_H_ +#define INCLUDE_CONTAINERS_CONTAINER_DEFS_H_ + +#ifdef __cplusplus +#include <type_traits> // used by casting helper for compile-time check +#endif + +// The preferences are a separate file to separate out tweakable parameters +#include <roaring/containers/perfparameters.h> + +#ifdef __cplusplus +namespace roaring { +namespace internal { // No extern "C" (contains template) +#endif + +/* + * Since roaring_array_t's definition is not opaque, the container type is + * part of the API. If it's not going to be `void*` then it needs a name, and + * expectations are to prefix C library-exported names with `roaring_` etc. + * + * Rather than force the whole codebase to use the name `roaring_container_t`, + * the few API appearances use the macro ROARING_CONTAINER_T. Those includes + * are prior to containers.h, so make a short private alias of `container_t`. + * Then undefine the awkward macro so it's not used any more than it has to be. + */ +typedef ROARING_CONTAINER_T container_t; +#undef ROARING_CONTAINER_T + +/* + * See ROARING_CONTAINER_T for notes on using container_t as a base class. + * This macro helps make the following pattern look nicer: + * + * #ifdef __cplusplus + * struct roaring_array_s : public container_t { + * #else + * struct roaring_array_s { + * #endif + * int32_t cardinality; + * int32_t capacity; + * uint16_t *array; + * } + */ +#if defined(__cplusplus) +#define STRUCT_CONTAINER(name) struct name : public container_t /* { ... } */ +#else +#define STRUCT_CONTAINER(name) struct name /* { ... } */ +#endif + +/** + * Since container_t* is not void* in C++, "dangerous" casts are not needed to + * downcast; only a static_cast<> is needed. Define a macro for static casting + * which helps make casts more visible, and catches problems at compile-time + * when building the C sources in C++ mode: + * + * void some_func(container_t **c, ...) { // double pointer, not single + * array_container_t *ac1 = (array_container_t *)(c); // uncaught!! + * + * array_container_t *ac2 = CAST(array_container_t *, c) // C++ errors + * array_container_t *ac3 = CAST_array(c); // shorthand for #2, errors + * } + * + * Trickier to do is a cast from `container**` to `array_container_t**`. This + * needs a reinterpret_cast<>, which sacrifices safety...so a template is used + * leveraging <type_traits> to make sure it's legal in the C++ build. + */ +#ifdef __cplusplus +#define CAST(type, value) static_cast<type>(value) +#define movable_CAST(type, value) movable_CAST_HELPER<type>(value) + +template <typename PPDerived, typename Base> +PPDerived movable_CAST_HELPER(Base **ptr_to_ptr) { + typedef typename std::remove_pointer<PPDerived>::type PDerived; + typedef typename std::remove_pointer<PDerived>::type Derived; + static_assert(std::is_base_of<Base, Derived>::value, + "use movable_CAST() for container_t** => xxx_container_t**"); + return reinterpret_cast<Derived **>(ptr_to_ptr); +} +#else +#define CAST(type, value) ((type)value) +#define movable_CAST(type, value) ((type)value) +#endif + +// Use for converting e.g. an `array_container_t**` to a `container_t**` +// +#define movable_CAST_base(c) movable_CAST(container_t **, c) + +#ifdef __cplusplus +} +} // namespace roaring { namespace internal { +#endif + +#endif /* INCLUDE_CONTAINERS_CONTAINER_DEFS_H_ */ diff --git a/contrib/libs/croaring/include/roaring/containers/containers.h b/contrib/libs/croaring/include/roaring/containers/containers.h new file mode 100644 index 00000000000..30b0cde1aae --- /dev/null +++ b/contrib/libs/croaring/include/roaring/containers/containers.h @@ -0,0 +1,2486 @@ +#ifndef CONTAINERS_CONTAINERS_H +#define CONTAINERS_CONTAINERS_H + +#include <assert.h> +#include <stdbool.h> +#include <stdio.h> + +#include <roaring/bitset_util.h> +#include <roaring/containers/array.h> +#include <roaring/containers/bitset.h> +#include <roaring/containers/convert.h> +#include <roaring/containers/mixed_andnot.h> +#include <roaring/containers/mixed_equal.h> +#include <roaring/containers/mixed_intersection.h> +#include <roaring/containers/mixed_negation.h> +#include <roaring/containers/mixed_subset.h> +#include <roaring/containers/mixed_union.h> +#include <roaring/containers/mixed_xor.h> +#include <roaring/containers/run.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +// would enum be possible or better? + +/** + * The switch case statements follow + * BITSET_CONTAINER_TYPE -- ARRAY_CONTAINER_TYPE -- RUN_CONTAINER_TYPE + * so it makes more sense to number them 1, 2, 3 (in the vague hope that the + * compiler might exploit this ordering). + */ + +#define BITSET_CONTAINER_TYPE 1 +#define ARRAY_CONTAINER_TYPE 2 +#define RUN_CONTAINER_TYPE 3 +#define SHARED_CONTAINER_TYPE 4 + +/** + * Macros for pairing container type codes, suitable for switch statements. + * Use PAIR_CONTAINER_TYPES() for the switch, CONTAINER_PAIR() for the cases: + * + * switch (PAIR_CONTAINER_TYPES(type1, type2)) { + * case CONTAINER_PAIR(BITSET,ARRAY): + * ... + * } + */ +#define PAIR_CONTAINER_TYPES(type1, type2) (4 * (type1) + (type2)) + +#define CONTAINER_PAIR(name1, name2) \ + (4 * (name1##_CONTAINER_TYPE) + (name2##_CONTAINER_TYPE)) + +/** + * A shared container is a wrapper around a container + * with reference counting. + */ +STRUCT_CONTAINER(shared_container_s) { + container_t *container; + uint8_t typecode; + croaring_refcount_t counter; // to be managed atomically +}; + +typedef struct shared_container_s shared_container_t; + +#define CAST_shared(c) CAST(shared_container_t *, c) // safer downcast +#define const_CAST_shared(c) CAST(const shared_container_t *, c) +#define movable_CAST_shared(c) movable_CAST(shared_container_t **, c) + +/* + * With copy_on_write = true + * Create a new shared container if the typecode is not SHARED_CONTAINER_TYPE, + * otherwise, increase the count + * If copy_on_write = false, then clone. + * Return NULL in case of failure. + **/ +container_t *get_copy_of_container(container_t *container, uint8_t *typecode, + bool copy_on_write); + +/* Frees a shared container (actually decrement its counter and only frees when + * the counter falls to zero). */ +void shared_container_free(shared_container_t *container); + +/* extract a copy from the shared container, freeing the shared container if +there is just one instance left, +clone instances when the counter is higher than one +*/ +container_t *shared_container_extract_copy(shared_container_t *container, + uint8_t *typecode); + +/* access to container underneath */ +static inline const container_t *container_unwrap_shared( + const container_t *candidate_shared_container, uint8_t *type) { + if (*type == SHARED_CONTAINER_TYPE) { + *type = const_CAST_shared(candidate_shared_container)->typecode; + assert(*type != SHARED_CONTAINER_TYPE); + return const_CAST_shared(candidate_shared_container)->container; + } else { + return candidate_shared_container; + } +} + +/* access to container underneath */ +static inline container_t *container_mutable_unwrap_shared(container_t *c, + uint8_t *type) { + if (*type == SHARED_CONTAINER_TYPE) { // the passed in container is shared + *type = CAST_shared(c)->typecode; + assert(*type != SHARED_CONTAINER_TYPE); + return CAST_shared(c)->container; // return the enclosed container + } else { + return c; // wasn't shared, so return as-is + } +} + +/* access to container underneath and queries its type */ +static inline uint8_t get_container_type(const container_t *c, uint8_t type) { + if (type == SHARED_CONTAINER_TYPE) { + return const_CAST_shared(c)->typecode; + } else { + return type; + } +} + +/** + * Copies a container, requires a typecode. This allocates new memory, caller + * is responsible for deallocation. If the container is not shared, then it is + * physically cloned. Sharable containers are not cloneable. + */ +container_t *container_clone(const container_t *container, uint8_t typecode); + +/* access to container underneath, cloning it if needed */ +static inline container_t *get_writable_copy_if_shared(container_t *c, + uint8_t *type) { + if (*type == SHARED_CONTAINER_TYPE) { // shared, return enclosed container + return shared_container_extract_copy(CAST_shared(c), type); + } else { + return c; // not shared, so return as-is + } +} + +/** + * End of shared container code + */ + +static const char *container_names[] = {"bitset", "array", "run", "shared"}; +static const char *shared_container_names[] = { + "bitset (shared)", "array (shared)", "run (shared)"}; + +// no matter what the initial container was, convert it to a bitset +// if a new container is produced, caller responsible for freeing the previous +// one +// container should not be a shared container +static inline bitset_container_t *container_to_bitset(container_t *c, + uint8_t typecode) { + bitset_container_t *result = NULL; + switch (typecode) { + case BITSET_CONTAINER_TYPE: + return CAST_bitset(c); // nothing to do + case ARRAY_CONTAINER_TYPE: + result = bitset_container_from_array(CAST_array(c)); + return result; + case RUN_CONTAINER_TYPE: + result = bitset_container_from_run(CAST_run(c)); + return result; + case SHARED_CONTAINER_TYPE: + assert(false); + roaring_unreachable; + } + assert(false); + roaring_unreachable; + return 0; // unreached +} + +/** + * Get the container name from the typecode + * (unused at time of writing) + */ +/*static inline const char *get_container_name(uint8_t typecode) { + switch (typecode) { + case BITSET_CONTAINER_TYPE: + return container_names[0]; + case ARRAY_CONTAINER_TYPE: + return container_names[1]; + case RUN_CONTAINER_TYPE: + return container_names[2]; + case SHARED_CONTAINER_TYPE: + return container_names[3]; + default: + assert(false); + roaring_unreachable; + return "unknown"; + } +}*/ + +static inline const char *get_full_container_name(const container_t *c, + uint8_t typecode) { + switch (typecode) { + case BITSET_CONTAINER_TYPE: + return container_names[0]; + case ARRAY_CONTAINER_TYPE: + return container_names[1]; + case RUN_CONTAINER_TYPE: + return container_names[2]; + case SHARED_CONTAINER_TYPE: + switch (const_CAST_shared(c)->typecode) { + case BITSET_CONTAINER_TYPE: + return shared_container_names[0]; + case ARRAY_CONTAINER_TYPE: + return shared_container_names[1]; + case RUN_CONTAINER_TYPE: + return shared_container_names[2]; + default: + assert(false); + roaring_unreachable; + return "unknown"; + } + break; + default: + assert(false); + roaring_unreachable; + return "unknown"; + } + roaring_unreachable; + return NULL; +} + +/** + * Get the container cardinality (number of elements), requires a typecode + */ +static inline int container_get_cardinality(const container_t *c, + uint8_t typecode) { + c = container_unwrap_shared(c, &typecode); + switch (typecode) { + case BITSET_CONTAINER_TYPE: + return bitset_container_cardinality(const_CAST_bitset(c)); + case ARRAY_CONTAINER_TYPE: + return array_container_cardinality(const_CAST_array(c)); + case RUN_CONTAINER_TYPE: + return run_container_cardinality(const_CAST_run(c)); + } + assert(false); + roaring_unreachable; + return 0; // unreached +} + +// returns true if a container is known to be full. Note that a lazy bitset +// container +// might be full without us knowing +static inline bool container_is_full(const container_t *c, uint8_t typecode) { + c = container_unwrap_shared(c, &typecode); + switch (typecode) { + case BITSET_CONTAINER_TYPE: + return bitset_container_cardinality(const_CAST_bitset(c)) == + (1 << 16); + case ARRAY_CONTAINER_TYPE: + return array_container_cardinality(const_CAST_array(c)) == + (1 << 16); + case RUN_CONTAINER_TYPE: + return run_container_is_full(const_CAST_run(c)); + } + assert(false); + roaring_unreachable; + return 0; // unreached +} + +static inline int container_shrink_to_fit(container_t *c, uint8_t type) { + c = container_mutable_unwrap_shared(c, &type); + switch (type) { + case BITSET_CONTAINER_TYPE: + return 0; // no shrinking possible + case ARRAY_CONTAINER_TYPE: + return array_container_shrink_to_fit(CAST_array(c)); + case RUN_CONTAINER_TYPE: + return run_container_shrink_to_fit(CAST_run(c)); + } + assert(false); + roaring_unreachable; + return 0; // unreached +} + +/** + * make a container with a run of ones + */ +/* initially always use a run container, even if an array might be + * marginally + * smaller */ +static inline container_t *container_range_of_ones(uint32_t range_start, + uint32_t range_end, + uint8_t *result_type) { + assert(range_end >= range_start); + uint64_t cardinality = range_end - range_start + 1; + if (cardinality <= 2) { + *result_type = ARRAY_CONTAINER_TYPE; + return array_container_create_range(range_start, range_end); + } else { + *result_type = RUN_CONTAINER_TYPE; + return run_container_create_range(range_start, range_end); + } +} + +/* Create a container with all the values between in [min,max) at a + distance k*step from min. */ +static inline container_t *container_from_range(uint8_t *type, uint32_t min, + uint32_t max, uint16_t step) { + if (step == 0) return NULL; // being paranoid + if (step == 1) { + return container_range_of_ones(min, max, type); + // Note: the result is not always a run (need to check the cardinality) + //*type = RUN_CONTAINER_TYPE; + // return run_container_create_range(min, max); + } + int size = (max - min + step - 1) / step; + if (size <= DEFAULT_MAX_SIZE) { // array container + *type = ARRAY_CONTAINER_TYPE; + array_container_t *array = array_container_create_given_capacity(size); + array_container_add_from_range(array, min, max, step); + assert(array->cardinality == size); + return array; + } else { // bitset container + *type = BITSET_CONTAINER_TYPE; + bitset_container_t *bitset = bitset_container_create(); + bitset_container_add_from_range(bitset, min, max, step); + assert(bitset->cardinality == size); + return bitset; + } +} + +/** + * "repair" the container after lazy operations. + */ +static inline container_t *container_repair_after_lazy(container_t *c, + uint8_t *type) { + c = get_writable_copy_if_shared(c, type); // !!! unnecessary cloning + container_t *result = NULL; + switch (*type) { + case BITSET_CONTAINER_TYPE: { + bitset_container_t *bc = CAST_bitset(c); + bc->cardinality = bitset_container_compute_cardinality(bc); + if (bc->cardinality <= DEFAULT_MAX_SIZE) { + result = array_container_from_bitset(bc); + bitset_container_free(bc); + *type = ARRAY_CONTAINER_TYPE; + return result; + } + return c; + } + case ARRAY_CONTAINER_TYPE: + return c; // nothing to do + case RUN_CONTAINER_TYPE: + return convert_run_to_efficient_container_and_free(CAST_run(c), + type); + case SHARED_CONTAINER_TYPE: + assert(false); + } + assert(false); + roaring_unreachable; + return 0; // unreached +} + +/** + * Writes the underlying array to buf, outputs how many bytes were written. + * This is meant to be byte-by-byte compatible with the Java and Go versions of + * Roaring. + * The number of bytes written should be + * container_write(container, buf). + * + */ +static inline int32_t container_write(const container_t *c, uint8_t typecode, + char *buf) { + c = container_unwrap_shared(c, &typecode); + switch (typecode) { + case BITSET_CONTAINER_TYPE: + return bitset_container_write(const_CAST_bitset(c), buf); + case ARRAY_CONTAINER_TYPE: + return array_container_write(const_CAST_array(c), buf); + case RUN_CONTAINER_TYPE: + return run_container_write(const_CAST_run(c), buf); + } + assert(false); + roaring_unreachable; + return 0; // unreached +} + +/** + * Get the container size in bytes under portable serialization (see + * container_write), requires a + * typecode + */ +static inline int32_t container_size_in_bytes(const container_t *c, + uint8_t typecode) { + c = container_unwrap_shared(c, &typecode); + switch (typecode) { + case BITSET_CONTAINER_TYPE: + return bitset_container_size_in_bytes(const_CAST_bitset(c)); + case ARRAY_CONTAINER_TYPE: + return array_container_size_in_bytes(const_CAST_array(c)); + case RUN_CONTAINER_TYPE: + return run_container_size_in_bytes(const_CAST_run(c)); + } + assert(false); + roaring_unreachable; + return 0; // unreached +} + +/** + * print the container (useful for debugging), requires a typecode + */ +void container_printf(const container_t *container, uint8_t typecode); + +/** + * print the content of the container as a comma-separated list of 32-bit values + * starting at base, requires a typecode + */ +void container_printf_as_uint32_array(const container_t *container, + uint8_t typecode, uint32_t base); + +bool container_internal_validate(const container_t *container, uint8_t typecode, + const char **reason); + +/** + * Checks whether a container is not empty, requires a typecode + */ +static inline bool container_nonzero_cardinality(const container_t *c, + uint8_t typecode) { + c = container_unwrap_shared(c, &typecode); + switch (typecode) { + case BITSET_CONTAINER_TYPE: + return bitset_container_const_nonzero_cardinality( + const_CAST_bitset(c)); + case ARRAY_CONTAINER_TYPE: + return array_container_nonzero_cardinality(const_CAST_array(c)); + case RUN_CONTAINER_TYPE: + return run_container_nonzero_cardinality(const_CAST_run(c)); + } + assert(false); + roaring_unreachable; + return 0; // unreached +} + +/** + * Recover memory from a container, requires a typecode + */ +void container_free(container_t *container, uint8_t typecode); + +/** + * Convert a container to an array of values, requires a typecode as well as a + * "base" (most significant values) + * Returns number of ints added. + */ +static inline int container_to_uint32_array(uint32_t *output, + const container_t *c, + uint8_t typecode, uint32_t base) { + c = container_unwrap_shared(c, &typecode); + switch (typecode) { + case BITSET_CONTAINER_TYPE: + return bitset_container_to_uint32_array(output, + const_CAST_bitset(c), base); + case ARRAY_CONTAINER_TYPE: + return array_container_to_uint32_array(output, const_CAST_array(c), + base); + case RUN_CONTAINER_TYPE: + return run_container_to_uint32_array(output, const_CAST_run(c), + base); + } + assert(false); + roaring_unreachable; + return 0; // unreached +} + +/** + * Add a value to a container, requires a typecode, fills in new_typecode and + * return (possibly different) container. + * This function may allocate a new container, and caller is responsible for + * memory deallocation + */ +static inline container_t *container_add( + container_t *c, uint16_t val, + uint8_t typecode, // !!! should be second argument? + uint8_t *new_typecode) { + c = get_writable_copy_if_shared(c, &typecode); + switch (typecode) { + case BITSET_CONTAINER_TYPE: + bitset_container_set(CAST_bitset(c), val); + *new_typecode = BITSET_CONTAINER_TYPE; + return c; + case ARRAY_CONTAINER_TYPE: { + array_container_t *ac = CAST_array(c); + if (array_container_try_add(ac, val, DEFAULT_MAX_SIZE) != -1) { + *new_typecode = ARRAY_CONTAINER_TYPE; + return ac; + } else { + bitset_container_t *bitset = bitset_container_from_array(ac); + bitset_container_add(bitset, val); + *new_typecode = BITSET_CONTAINER_TYPE; + return bitset; + } + } break; + case RUN_CONTAINER_TYPE: + // per Java, no container type adjustments are done (revisit?) + run_container_add(CAST_run(c), val); + *new_typecode = RUN_CONTAINER_TYPE; + return c; + default: + assert(false); + roaring_unreachable; + return NULL; + } +} + +/** + * Remove a value from a container, requires a typecode, fills in new_typecode + * and + * return (possibly different) container. + * This function may allocate a new container, and caller is responsible for + * memory deallocation + */ +static inline container_t *container_remove( + container_t *c, uint16_t val, + uint8_t typecode, // !!! should be second argument? + uint8_t *new_typecode) { + c = get_writable_copy_if_shared(c, &typecode); + switch (typecode) { + case BITSET_CONTAINER_TYPE: + if (bitset_container_remove(CAST_bitset(c), val)) { + int card = bitset_container_cardinality(CAST_bitset(c)); + if (card <= DEFAULT_MAX_SIZE) { + *new_typecode = ARRAY_CONTAINER_TYPE; + return array_container_from_bitset(CAST_bitset(c)); + } + } + *new_typecode = typecode; + return c; + case ARRAY_CONTAINER_TYPE: + *new_typecode = typecode; + array_container_remove(CAST_array(c), val); + return c; + case RUN_CONTAINER_TYPE: + // per Java, no container type adjustments are done (revisit?) + run_container_remove(CAST_run(c), val); + *new_typecode = RUN_CONTAINER_TYPE; + return c; + default: + assert(false); + roaring_unreachable; + return NULL; + } +} + +/** + * Check whether a value is in a container, requires a typecode + */ +static inline bool container_contains( + const container_t *c, uint16_t val, + uint8_t typecode // !!! should be second argument? +) { + c = container_unwrap_shared(c, &typecode); + switch (typecode) { + case BITSET_CONTAINER_TYPE: + return bitset_container_get(const_CAST_bitset(c), val); + case ARRAY_CONTAINER_TYPE: + return array_container_contains(const_CAST_array(c), val); + case RUN_CONTAINER_TYPE: + return run_container_contains(const_CAST_run(c), val); + default: + assert(false); + roaring_unreachable; + return false; + } +} + +/** + * Check whether a range of values from range_start (included) to range_end + * (excluded) is in a container, requires a typecode + */ +static inline bool container_contains_range( + const container_t *c, uint32_t range_start, uint32_t range_end, + uint8_t typecode // !!! should be second argument? +) { + c = container_unwrap_shared(c, &typecode); + switch (typecode) { + case BITSET_CONTAINER_TYPE: + return bitset_container_get_range(const_CAST_bitset(c), range_start, + range_end); + case ARRAY_CONTAINER_TYPE: + return array_container_contains_range(const_CAST_array(c), + range_start, range_end); + case RUN_CONTAINER_TYPE: + return run_container_contains_range(const_CAST_run(c), range_start, + range_end); + default: + assert(false); + roaring_unreachable; + return false; + } +} + +/** + * Returns true if the two containers have the same content. Note that + * two containers having different types can be "equal" in this sense. + */ +static inline bool container_equals(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2) { + c1 = container_unwrap_shared(c1, &type1); + c2 = container_unwrap_shared(c2, &type2); + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + return bitset_container_equals(const_CAST_bitset(c1), + const_CAST_bitset(c2)); + + case CONTAINER_PAIR(BITSET, RUN): + return run_container_equals_bitset(const_CAST_run(c2), + const_CAST_bitset(c1)); + + case CONTAINER_PAIR(RUN, BITSET): + return run_container_equals_bitset(const_CAST_run(c1), + const_CAST_bitset(c2)); + + case CONTAINER_PAIR(BITSET, ARRAY): + // java would always return false? + return array_container_equal_bitset(const_CAST_array(c2), + const_CAST_bitset(c1)); + + case CONTAINER_PAIR(ARRAY, BITSET): + // java would always return false? + return array_container_equal_bitset(const_CAST_array(c1), + const_CAST_bitset(c2)); + + case CONTAINER_PAIR(ARRAY, RUN): + return run_container_equals_array(const_CAST_run(c2), + const_CAST_array(c1)); + + case CONTAINER_PAIR(RUN, ARRAY): + return run_container_equals_array(const_CAST_run(c1), + const_CAST_array(c2)); + + case CONTAINER_PAIR(ARRAY, ARRAY): + return array_container_equals(const_CAST_array(c1), + const_CAST_array(c2)); + + case CONTAINER_PAIR(RUN, RUN): + return run_container_equals(const_CAST_run(c1), const_CAST_run(c2)); + + default: + assert(false); + roaring_unreachable; + return false; + } +} + +/** + * Returns true if the container c1 is a subset of the container c2. Note that + * c1 can be a subset of c2 even if they have a different type. + */ +static inline bool container_is_subset(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2) { + c1 = container_unwrap_shared(c1, &type1); + c2 = container_unwrap_shared(c2, &type2); + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + return bitset_container_is_subset(const_CAST_bitset(c1), + const_CAST_bitset(c2)); + + case CONTAINER_PAIR(BITSET, RUN): + return bitset_container_is_subset_run(const_CAST_bitset(c1), + const_CAST_run(c2)); + + case CONTAINER_PAIR(RUN, BITSET): + return run_container_is_subset_bitset(const_CAST_run(c1), + const_CAST_bitset(c2)); + + case CONTAINER_PAIR(BITSET, ARRAY): + return false; // by construction, size(c1) > size(c2) + + case CONTAINER_PAIR(ARRAY, BITSET): + return array_container_is_subset_bitset(const_CAST_array(c1), + const_CAST_bitset(c2)); + + case CONTAINER_PAIR(ARRAY, RUN): + return array_container_is_subset_run(const_CAST_array(c1), + const_CAST_run(c2)); + + case CONTAINER_PAIR(RUN, ARRAY): + return run_container_is_subset_array(const_CAST_run(c1), + const_CAST_array(c2)); + + case CONTAINER_PAIR(ARRAY, ARRAY): + return array_container_is_subset(const_CAST_array(c1), + const_CAST_array(c2)); + + case CONTAINER_PAIR(RUN, RUN): + return run_container_is_subset(const_CAST_run(c1), + const_CAST_run(c2)); + + default: + assert(false); + roaring_unreachable; + return false; + } +} + +// macro-izations possibilities for generic non-inplace binary-op dispatch + +/** + * Compute intersection between two containers, generate a new container (having + * type result_type), requires a typecode. This allocates new memory, caller + * is responsible for deallocation. + */ +static inline container_t *container_and(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type) { + c1 = container_unwrap_shared(c1, &type1); + c2 = container_unwrap_shared(c2, &type2); + container_t *result = NULL; + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + *result_type = + bitset_bitset_container_intersection( + const_CAST_bitset(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, ARRAY): + result = array_container_create(); + array_container_intersection( + const_CAST_array(c1), const_CAST_array(c2), CAST_array(result)); + *result_type = ARRAY_CONTAINER_TYPE; // never bitset + return result; + + case CONTAINER_PAIR(RUN, RUN): + result = run_container_create(); + run_container_intersection(const_CAST_run(c1), const_CAST_run(c2), + CAST_run(result)); + return convert_run_to_efficient_container_and_free(CAST_run(result), + result_type); + + case CONTAINER_PAIR(BITSET, ARRAY): + result = array_container_create(); + array_bitset_container_intersection(const_CAST_array(c2), + const_CAST_bitset(c1), + CAST_array(result)); + *result_type = ARRAY_CONTAINER_TYPE; // never bitset + return result; + + case CONTAINER_PAIR(ARRAY, BITSET): + result = array_container_create(); + *result_type = ARRAY_CONTAINER_TYPE; // never bitset + array_bitset_container_intersection(const_CAST_array(c1), + const_CAST_bitset(c2), + CAST_array(result)); + return result; + + case CONTAINER_PAIR(BITSET, RUN): + *result_type = + run_bitset_container_intersection( + const_CAST_run(c2), const_CAST_bitset(c1), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(RUN, BITSET): + *result_type = + run_bitset_container_intersection( + const_CAST_run(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, RUN): + result = array_container_create(); + *result_type = ARRAY_CONTAINER_TYPE; // never bitset + array_run_container_intersection( + const_CAST_array(c1), const_CAST_run(c2), CAST_array(result)); + return result; + + case CONTAINER_PAIR(RUN, ARRAY): + result = array_container_create(); + *result_type = ARRAY_CONTAINER_TYPE; // never bitset + array_run_container_intersection( + const_CAST_array(c2), const_CAST_run(c1), CAST_array(result)); + return result; + + default: + assert(false); + roaring_unreachable; + return NULL; + } +} + +/** + * Compute the size of the intersection between two containers. + */ +static inline int container_and_cardinality(const container_t *c1, + uint8_t type1, + const container_t *c2, + uint8_t type2) { + c1 = container_unwrap_shared(c1, &type1); + c2 = container_unwrap_shared(c2, &type2); + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + return bitset_container_and_justcard(const_CAST_bitset(c1), + const_CAST_bitset(c2)); + + case CONTAINER_PAIR(ARRAY, ARRAY): + return array_container_intersection_cardinality( + const_CAST_array(c1), const_CAST_array(c2)); + + case CONTAINER_PAIR(RUN, RUN): + return run_container_intersection_cardinality(const_CAST_run(c1), + const_CAST_run(c2)); + + case CONTAINER_PAIR(BITSET, ARRAY): + return array_bitset_container_intersection_cardinality( + const_CAST_array(c2), const_CAST_bitset(c1)); + + case CONTAINER_PAIR(ARRAY, BITSET): + return array_bitset_container_intersection_cardinality( + const_CAST_array(c1), const_CAST_bitset(c2)); + + case CONTAINER_PAIR(BITSET, RUN): + return run_bitset_container_intersection_cardinality( + const_CAST_run(c2), const_CAST_bitset(c1)); + + case CONTAINER_PAIR(RUN, BITSET): + return run_bitset_container_intersection_cardinality( + const_CAST_run(c1), const_CAST_bitset(c2)); + + case CONTAINER_PAIR(ARRAY, RUN): + return array_run_container_intersection_cardinality( + const_CAST_array(c1), const_CAST_run(c2)); + + case CONTAINER_PAIR(RUN, ARRAY): + return array_run_container_intersection_cardinality( + const_CAST_array(c2), const_CAST_run(c1)); + + default: + assert(false); + roaring_unreachable; + return 0; + } +} + +/** + * Check whether two containers intersect. + */ +static inline bool container_intersect(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2) { + c1 = container_unwrap_shared(c1, &type1); + c2 = container_unwrap_shared(c2, &type2); + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + return bitset_container_intersect(const_CAST_bitset(c1), + const_CAST_bitset(c2)); + + case CONTAINER_PAIR(ARRAY, ARRAY): + return array_container_intersect(const_CAST_array(c1), + const_CAST_array(c2)); + + case CONTAINER_PAIR(RUN, RUN): + return run_container_intersect(const_CAST_run(c1), + const_CAST_run(c2)); + + case CONTAINER_PAIR(BITSET, ARRAY): + return array_bitset_container_intersect(const_CAST_array(c2), + const_CAST_bitset(c1)); + + case CONTAINER_PAIR(ARRAY, BITSET): + return array_bitset_container_intersect(const_CAST_array(c1), + const_CAST_bitset(c2)); + + case CONTAINER_PAIR(BITSET, RUN): + return run_bitset_container_intersect(const_CAST_run(c2), + const_CAST_bitset(c1)); + + case CONTAINER_PAIR(RUN, BITSET): + return run_bitset_container_intersect(const_CAST_run(c1), + const_CAST_bitset(c2)); + + case CONTAINER_PAIR(ARRAY, RUN): + return array_run_container_intersect(const_CAST_array(c1), + const_CAST_run(c2)); + + case CONTAINER_PAIR(RUN, ARRAY): + return array_run_container_intersect(const_CAST_array(c2), + const_CAST_run(c1)); + + default: + assert(false); + roaring_unreachable; + return 0; + } +} + +/** + * Compute intersection between two containers, with result in the first + container if possible. If the returned pointer is identical to c1, + then the container has been modified. If the returned pointer is different + from c1, then a new container has been created and the caller is responsible + for freeing it. + The type of the first container may change. Returns the modified + (and possibly new) container. +*/ +static inline container_t *container_iand(container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type) { + c1 = get_writable_copy_if_shared(c1, &type1); + c2 = container_unwrap_shared(c2, &type2); + container_t *result = NULL; + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + *result_type = bitset_bitset_container_intersection_inplace( + CAST_bitset(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, ARRAY): + array_container_intersection_inplace(CAST_array(c1), + const_CAST_array(c2)); + *result_type = ARRAY_CONTAINER_TYPE; + return c1; + + case CONTAINER_PAIR(RUN, RUN): + result = run_container_create(); + run_container_intersection(const_CAST_run(c1), const_CAST_run(c2), + CAST_run(result)); + // as of January 2016, Java code used non-in-place intersection for + // two runcontainers + return convert_run_to_efficient_container_and_free(CAST_run(result), + result_type); + + case CONTAINER_PAIR(BITSET, ARRAY): + // c1 is a bitmap so no inplace possible + result = array_container_create(); + array_bitset_container_intersection(const_CAST_array(c2), + const_CAST_bitset(c1), + CAST_array(result)); + *result_type = ARRAY_CONTAINER_TYPE; // never bitset + return result; + + case CONTAINER_PAIR(ARRAY, BITSET): + *result_type = ARRAY_CONTAINER_TYPE; // never bitset + array_bitset_container_intersection( + const_CAST_array(c1), const_CAST_bitset(c2), + CAST_array(c1)); // result is allowed to be same as c1 + return c1; + + case CONTAINER_PAIR(BITSET, RUN): + // will attempt in-place computation + *result_type = run_bitset_container_intersection( + const_CAST_run(c2), const_CAST_bitset(c1), &c1) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return c1; + + case CONTAINER_PAIR(RUN, BITSET): + *result_type = + run_bitset_container_intersection( + const_CAST_run(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, RUN): + result = array_container_create(); + *result_type = ARRAY_CONTAINER_TYPE; // never bitset + array_run_container_intersection( + const_CAST_array(c1), const_CAST_run(c2), CAST_array(result)); + return result; + + case CONTAINER_PAIR(RUN, ARRAY): + result = array_container_create(); + *result_type = ARRAY_CONTAINER_TYPE; // never bitset + array_run_container_intersection( + const_CAST_array(c2), const_CAST_run(c1), CAST_array(result)); + return result; + + default: + assert(false); + roaring_unreachable; + return NULL; + } +} + +/** + * Compute union between two containers, generate a new container (having type + * result_type), requires a typecode. This allocates new memory, caller + * is responsible for deallocation. + */ +static inline container_t *container_or(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type) { + c1 = container_unwrap_shared(c1, &type1); + c2 = container_unwrap_shared(c2, &type2); + container_t *result = NULL; + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + result = bitset_container_create(); + bitset_container_or(const_CAST_bitset(c1), const_CAST_bitset(c2), + CAST_bitset(result)); + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, ARRAY): + *result_type = + array_array_container_union(const_CAST_array(c1), + const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(RUN, RUN): + result = run_container_create(); + run_container_union(const_CAST_run(c1), const_CAST_run(c2), + CAST_run(result)); + *result_type = RUN_CONTAINER_TYPE; + // todo: could be optimized since will never convert to array + result = convert_run_to_efficient_container_and_free( + CAST_run(result), result_type); + return result; + + case CONTAINER_PAIR(BITSET, ARRAY): + result = bitset_container_create(); + array_bitset_container_union(const_CAST_array(c2), + const_CAST_bitset(c1), + CAST_bitset(result)); + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, BITSET): + result = bitset_container_create(); + array_bitset_container_union(const_CAST_array(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(BITSET, RUN): + if (run_container_is_full(const_CAST_run(c2))) { + result = run_container_create(); + *result_type = RUN_CONTAINER_TYPE; + run_container_copy(const_CAST_run(c2), CAST_run(result)); + return result; + } + result = bitset_container_create(); + run_bitset_container_union( + const_CAST_run(c2), const_CAST_bitset(c1), CAST_bitset(result)); + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(RUN, BITSET): + if (run_container_is_full(const_CAST_run(c1))) { + result = run_container_create(); + *result_type = RUN_CONTAINER_TYPE; + run_container_copy(const_CAST_run(c1), CAST_run(result)); + return result; + } + result = bitset_container_create(); + run_bitset_container_union( + const_CAST_run(c1), const_CAST_bitset(c2), CAST_bitset(result)); + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, RUN): + result = run_container_create(); + array_run_container_union(const_CAST_array(c1), const_CAST_run(c2), + CAST_run(result)); + result = convert_run_to_efficient_container_and_free( + CAST_run(result), result_type); + return result; + + case CONTAINER_PAIR(RUN, ARRAY): + result = run_container_create(); + array_run_container_union(const_CAST_array(c2), const_CAST_run(c1), + CAST_run(result)); + result = convert_run_to_efficient_container_and_free( + CAST_run(result), result_type); + return result; + + default: + assert(false); + roaring_unreachable; + return NULL; // unreached + } +} + +/** + * Compute union between two containers, generate a new container (having type + * result_type), requires a typecode. This allocates new memory, caller + * is responsible for deallocation. + * + * This lazy version delays some operations such as the maintenance of the + * cardinality. It requires repair later on the generated containers. + */ +static inline container_t *container_lazy_or(const container_t *c1, + uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type) { + c1 = container_unwrap_shared(c1, &type1); + c2 = container_unwrap_shared(c2, &type2); + container_t *result = NULL; + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + result = bitset_container_create(); + bitset_container_or_nocard(const_CAST_bitset(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); // is lazy + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, ARRAY): + *result_type = + array_array_container_lazy_union(const_CAST_array(c1), + const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(RUN, RUN): + result = run_container_create(); + run_container_union(const_CAST_run(c1), const_CAST_run(c2), + CAST_run(result)); + *result_type = RUN_CONTAINER_TYPE; + // we are being lazy + result = convert_run_to_efficient_container_and_free( + CAST_run(result), result_type); + return result; + + case CONTAINER_PAIR(BITSET, ARRAY): + result = bitset_container_create(); + array_bitset_container_lazy_union(const_CAST_array(c2), + const_CAST_bitset(c1), + CAST_bitset(result)); // is lazy + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, BITSET): + result = bitset_container_create(); + array_bitset_container_lazy_union(const_CAST_array(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); // is lazy + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(BITSET, RUN): + if (run_container_is_full(const_CAST_run(c2))) { + result = run_container_create(); + *result_type = RUN_CONTAINER_TYPE; + run_container_copy(const_CAST_run(c2), CAST_run(result)); + return result; + } + result = bitset_container_create(); + run_bitset_container_lazy_union(const_CAST_run(c2), + const_CAST_bitset(c1), + CAST_bitset(result)); // is lazy + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(RUN, BITSET): + if (run_container_is_full(const_CAST_run(c1))) { + result = run_container_create(); + *result_type = RUN_CONTAINER_TYPE; + run_container_copy(const_CAST_run(c1), CAST_run(result)); + return result; + } + result = bitset_container_create(); + run_bitset_container_lazy_union(const_CAST_run(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); // is lazy + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, RUN): + result = run_container_create(); + array_run_container_union(const_CAST_array(c1), const_CAST_run(c2), + CAST_run(result)); + *result_type = RUN_CONTAINER_TYPE; + // next line skipped since we are lazy + // result = convert_run_to_efficient_container(result, result_type); + return result; + + case CONTAINER_PAIR(RUN, ARRAY): + result = run_container_create(); + array_run_container_union(const_CAST_array(c2), const_CAST_run(c1), + CAST_run(result)); // TODO make lazy + *result_type = RUN_CONTAINER_TYPE; + // next line skipped since we are lazy + // result = convert_run_to_efficient_container(result, result_type); + return result; + + default: + assert(false); + roaring_unreachable; + return NULL; // unreached + } +} + +/** + * Compute the union between two containers, with result in the first container. + * If the returned pointer is identical to c1, then the container has been + * modified. + * If the returned pointer is different from c1, then a new container has been + * created and the caller is responsible for freeing it. + * The type of the first container may change. Returns the modified + * (and possibly new) container + */ +static inline container_t *container_ior(container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type) { + c1 = get_writable_copy_if_shared(c1, &type1); + c2 = container_unwrap_shared(c2, &type2); + container_t *result = NULL; + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + bitset_container_or(const_CAST_bitset(c1), const_CAST_bitset(c2), + CAST_bitset(c1)); +#ifdef OR_BITSET_CONVERSION_TO_FULL + if (CAST_bitset(c1)->cardinality == (1 << 16)) { // we convert + result = run_container_create_range(0, (1 << 16)); + *result_type = RUN_CONTAINER_TYPE; + return result; + } +#endif + *result_type = BITSET_CONTAINER_TYPE; + return c1; + + case CONTAINER_PAIR(ARRAY, ARRAY): + *result_type = array_array_container_inplace_union( + CAST_array(c1), const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + if ((result == NULL) && (*result_type == ARRAY_CONTAINER_TYPE)) { + return c1; // the computation was done in-place! + } + return result; + + case CONTAINER_PAIR(RUN, RUN): + run_container_union_inplace(CAST_run(c1), const_CAST_run(c2)); + return convert_run_to_efficient_container(CAST_run(c1), + result_type); + + case CONTAINER_PAIR(BITSET, ARRAY): + array_bitset_container_union( + const_CAST_array(c2), const_CAST_bitset(c1), CAST_bitset(c1)); + *result_type = BITSET_CONTAINER_TYPE; // never array + return c1; + + case CONTAINER_PAIR(ARRAY, BITSET): + // c1 is an array, so no in-place possible + result = bitset_container_create(); + *result_type = BITSET_CONTAINER_TYPE; + array_bitset_container_union(const_CAST_array(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); + return result; + + case CONTAINER_PAIR(BITSET, RUN): + if (run_container_is_full(const_CAST_run(c2))) { + result = run_container_create(); + *result_type = RUN_CONTAINER_TYPE; + run_container_copy(const_CAST_run(c2), CAST_run(result)); + return result; + } + run_bitset_container_union(const_CAST_run(c2), + const_CAST_bitset(c1), + CAST_bitset(c1)); // allowed + *result_type = BITSET_CONTAINER_TYPE; + return c1; + + case CONTAINER_PAIR(RUN, BITSET): + if (run_container_is_full(const_CAST_run(c1))) { + *result_type = RUN_CONTAINER_TYPE; + return c1; + } + result = bitset_container_create(); + run_bitset_container_union( + const_CAST_run(c1), const_CAST_bitset(c2), CAST_bitset(result)); + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, RUN): + result = run_container_create(); + array_run_container_union(const_CAST_array(c1), const_CAST_run(c2), + CAST_run(result)); + result = convert_run_to_efficient_container_and_free( + CAST_run(result), result_type); + return result; + + case CONTAINER_PAIR(RUN, ARRAY): + array_run_container_inplace_union(const_CAST_array(c2), + CAST_run(c1)); + c1 = convert_run_to_efficient_container(CAST_run(c1), result_type); + return c1; + + default: + assert(false); + roaring_unreachable; + return NULL; + } +} + +/** + * Compute the union between two containers, with result in the first container. + * If the returned pointer is identical to c1, then the container has been + * modified. + * If the returned pointer is different from c1, then a new container has been + * created and the caller is responsible for freeing it. + * The type of the first container may change. Returns the modified + * (and possibly new) container + * + * This lazy version delays some operations such as the maintenance of the + * cardinality. It requires repair later on the generated containers. + */ +static inline container_t *container_lazy_ior(container_t *c1, uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type) { + assert(type1 != SHARED_CONTAINER_TYPE); + // c1 = get_writable_copy_if_shared(c1,&type1); + c2 = container_unwrap_shared(c2, &type2); + container_t *result = NULL; + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): +#ifdef LAZY_OR_BITSET_CONVERSION_TO_FULL + // if we have two bitsets, we might as well compute the cardinality + bitset_container_or(const_CAST_bitset(c1), const_CAST_bitset(c2), + CAST_bitset(c1)); + // it is possible that two bitsets can lead to a full container + if (CAST_bitset(c1)->cardinality == (1 << 16)) { // we convert + result = run_container_create_range(0, (1 << 16)); + *result_type = RUN_CONTAINER_TYPE; + return result; + } +#else + bitset_container_or_nocard(const_CAST_bitset(c1), + const_CAST_bitset(c2), CAST_bitset(c1)); + +#endif + *result_type = BITSET_CONTAINER_TYPE; + return c1; + + case CONTAINER_PAIR(ARRAY, ARRAY): + *result_type = array_array_container_lazy_inplace_union( + CAST_array(c1), const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + if ((result == NULL) && (*result_type == ARRAY_CONTAINER_TYPE)) { + return c1; // the computation was done in-place! + } + return result; + + case CONTAINER_PAIR(RUN, RUN): + run_container_union_inplace(CAST_run(c1), const_CAST_run(c2)); + *result_type = RUN_CONTAINER_TYPE; + return convert_run_to_efficient_container(CAST_run(c1), + result_type); + + case CONTAINER_PAIR(BITSET, ARRAY): + array_bitset_container_lazy_union(const_CAST_array(c2), + const_CAST_bitset(c1), + CAST_bitset(c1)); // is lazy + *result_type = BITSET_CONTAINER_TYPE; // never array + return c1; + + case CONTAINER_PAIR(ARRAY, BITSET): + // c1 is an array, so no in-place possible + result = bitset_container_create(); + *result_type = BITSET_CONTAINER_TYPE; + array_bitset_container_lazy_union(const_CAST_array(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); // is lazy + return result; + + case CONTAINER_PAIR(BITSET, RUN): + if (run_container_is_full(const_CAST_run(c2))) { + result = run_container_create(); + *result_type = RUN_CONTAINER_TYPE; + run_container_copy(const_CAST_run(c2), CAST_run(result)); + return result; + } + run_bitset_container_lazy_union( + const_CAST_run(c2), const_CAST_bitset(c1), + CAST_bitset(c1)); // allowed // lazy + *result_type = BITSET_CONTAINER_TYPE; + return c1; + + case CONTAINER_PAIR(RUN, BITSET): + if (run_container_is_full(const_CAST_run(c1))) { + *result_type = RUN_CONTAINER_TYPE; + return c1; + } + result = bitset_container_create(); + run_bitset_container_lazy_union(const_CAST_run(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); // lazy + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, RUN): + result = run_container_create(); + array_run_container_union(const_CAST_array(c1), const_CAST_run(c2), + CAST_run(result)); + *result_type = RUN_CONTAINER_TYPE; + // next line skipped since we are lazy + // result = convert_run_to_efficient_container_and_free(result, + // result_type); + return result; + + case CONTAINER_PAIR(RUN, ARRAY): + array_run_container_inplace_union(const_CAST_array(c2), + CAST_run(c1)); + *result_type = RUN_CONTAINER_TYPE; + // next line skipped since we are lazy + // result = convert_run_to_efficient_container_and_free(result, + // result_type); + return c1; + + default: + assert(false); + roaring_unreachable; + return NULL; + } +} + +/** + * Compute symmetric difference (xor) between two containers, generate a new + * container (having type result_type), requires a typecode. This allocates new + * memory, caller is responsible for deallocation. + */ +static inline container_t *container_xor(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type) { + c1 = container_unwrap_shared(c1, &type1); + c2 = container_unwrap_shared(c2, &type2); + container_t *result = NULL; + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + *result_type = + bitset_bitset_container_xor(const_CAST_bitset(c1), + const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, ARRAY): + *result_type = + array_array_container_xor(const_CAST_array(c1), + const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(RUN, RUN): + *result_type = (uint8_t)run_run_container_xor( + const_CAST_run(c1), const_CAST_run(c2), &result); + return result; + + case CONTAINER_PAIR(BITSET, ARRAY): + *result_type = + array_bitset_container_xor(const_CAST_array(c2), + const_CAST_bitset(c1), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, BITSET): + *result_type = + array_bitset_container_xor(const_CAST_array(c1), + const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(BITSET, RUN): + *result_type = + run_bitset_container_xor(const_CAST_run(c2), + const_CAST_bitset(c1), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(RUN, BITSET): + *result_type = + run_bitset_container_xor(const_CAST_run(c1), + const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, RUN): + *result_type = (uint8_t)array_run_container_xor( + const_CAST_array(c1), const_CAST_run(c2), &result); + return result; + + case CONTAINER_PAIR(RUN, ARRAY): + *result_type = (uint8_t)array_run_container_xor( + const_CAST_array(c2), const_CAST_run(c1), &result); + return result; + + default: + assert(false); + roaring_unreachable; + return NULL; // unreached + } +} + +/* Applies an offset to the non-empty container 'c'. + * The results are stored in new containers returned via 'lo' and 'hi', for the + * low and high halves of the result (where the low half matches the original + * key and the high one corresponds to values for the following key). Either one + * of 'lo' and 'hi' are allowed to be 'NULL', but not both. Whenever one of them + * is not 'NULL', it should point to a 'NULL' container. Whenever one of them is + * 'NULL' the shifted elements for that part will not be computed. If either of + * the resulting containers turns out to be empty, the pointed container will + * remain 'NULL'. + */ +static inline void container_add_offset(const container_t *c, uint8_t type, + container_t **lo, container_t **hi, + uint16_t offset) { + assert(offset != 0); + assert(container_nonzero_cardinality(c, type)); + assert(lo != NULL || hi != NULL); + assert(lo == NULL || *lo == NULL); + assert(hi == NULL || *hi == NULL); + + switch (type) { + case BITSET_CONTAINER_TYPE: + bitset_container_offset(const_CAST_bitset(c), lo, hi, offset); + break; + case ARRAY_CONTAINER_TYPE: + array_container_offset(const_CAST_array(c), lo, hi, offset); + break; + case RUN_CONTAINER_TYPE: + run_container_offset(const_CAST_run(c), lo, hi, offset); + break; + default: + assert(false); + roaring_unreachable; + break; + } +} + +/** + * Compute xor between two containers, generate a new container (having type + * result_type), requires a typecode. This allocates new memory, caller + * is responsible for deallocation. + * + * This lazy version delays some operations such as the maintenance of the + * cardinality. It requires repair later on the generated containers. + */ +static inline container_t *container_lazy_xor(const container_t *c1, + uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type) { + c1 = container_unwrap_shared(c1, &type1); + c2 = container_unwrap_shared(c2, &type2); + container_t *result = NULL; + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + result = bitset_container_create(); + bitset_container_xor_nocard(const_CAST_bitset(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); // is lazy + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, ARRAY): + *result_type = + array_array_container_lazy_xor(const_CAST_array(c1), + const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(RUN, RUN): + // nothing special done yet. + *result_type = (uint8_t)run_run_container_xor( + const_CAST_run(c1), const_CAST_run(c2), &result); + return result; + + case CONTAINER_PAIR(BITSET, ARRAY): + result = bitset_container_create(); + *result_type = BITSET_CONTAINER_TYPE; + array_bitset_container_lazy_xor(const_CAST_array(c2), + const_CAST_bitset(c1), + CAST_bitset(result)); + return result; + + case CONTAINER_PAIR(ARRAY, BITSET): + result = bitset_container_create(); + *result_type = BITSET_CONTAINER_TYPE; + array_bitset_container_lazy_xor(const_CAST_array(c1), + const_CAST_bitset(c2), + CAST_bitset(result)); + return result; + + case CONTAINER_PAIR(BITSET, RUN): + result = bitset_container_create(); + run_bitset_container_lazy_xor( + const_CAST_run(c2), const_CAST_bitset(c1), CAST_bitset(result)); + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(RUN, BITSET): + result = bitset_container_create(); + run_bitset_container_lazy_xor( + const_CAST_run(c1), const_CAST_bitset(c2), CAST_bitset(result)); + *result_type = BITSET_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, RUN): + result = run_container_create(); + array_run_container_lazy_xor(const_CAST_array(c1), + const_CAST_run(c2), CAST_run(result)); + *result_type = RUN_CONTAINER_TYPE; + // next line skipped since we are lazy + // result = convert_run_to_efficient_container(result, result_type); + return result; + + case CONTAINER_PAIR(RUN, ARRAY): + result = run_container_create(); + array_run_container_lazy_xor(const_CAST_array(c2), + const_CAST_run(c1), CAST_run(result)); + *result_type = RUN_CONTAINER_TYPE; + // next line skipped since we are lazy + // result = convert_run_to_efficient_container(result, result_type); + return result; + + default: + assert(false); + roaring_unreachable; + return NULL; // unreached + } +} + +/** + * Compute the xor between two containers, with result in the first container. + * If the returned pointer is identical to c1, then the container has been + * modified. + * If the returned pointer is different from c1, then a new container has been + * created. The original container is freed by container_ixor. + * The type of the first container may change. Returns the modified (and + * possibly new) container. + */ +static inline container_t *container_ixor(container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type) { + c1 = get_writable_copy_if_shared(c1, &type1); + c2 = container_unwrap_shared(c2, &type2); + container_t *result = NULL; + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + *result_type = bitset_bitset_container_ixor( + CAST_bitset(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, ARRAY): + *result_type = array_array_container_ixor( + CAST_array(c1), const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(RUN, RUN): + *result_type = (uint8_t)run_run_container_ixor( + CAST_run(c1), const_CAST_run(c2), &result); + return result; + + case CONTAINER_PAIR(BITSET, ARRAY): + *result_type = bitset_array_container_ixor( + CAST_bitset(c1), const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, BITSET): + *result_type = array_bitset_container_ixor( + CAST_array(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(BITSET, RUN): + *result_type = bitset_run_container_ixor( + CAST_bitset(c1), const_CAST_run(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + + return result; + + case CONTAINER_PAIR(RUN, BITSET): + *result_type = run_bitset_container_ixor( + CAST_run(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, RUN): + *result_type = (uint8_t)array_run_container_ixor( + CAST_array(c1), const_CAST_run(c2), &result); + return result; + + case CONTAINER_PAIR(RUN, ARRAY): + *result_type = (uint8_t)run_array_container_ixor( + CAST_run(c1), const_CAST_array(c2), &result); + return result; + + default: + assert(false); + roaring_unreachable; + return NULL; + } +} + +/** + * Compute the xor between two containers, with result in the first container. + * If the returned pointer is identical to c1, then the container has been + * modified. + * If the returned pointer is different from c1, then a new container has been + * created and the caller is responsible for freeing it. + * The type of the first container may change. Returns the modified + * (and possibly new) container + * + * This lazy version delays some operations such as the maintenance of the + * cardinality. It requires repair later on the generated containers. + */ +static inline container_t *container_lazy_ixor(container_t *c1, uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type) { + assert(type1 != SHARED_CONTAINER_TYPE); + // c1 = get_writable_copy_if_shared(c1,&type1); + c2 = container_unwrap_shared(c2, &type2); + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + bitset_container_xor_nocard(CAST_bitset(c1), const_CAST_bitset(c2), + CAST_bitset(c1)); // is lazy + *result_type = BITSET_CONTAINER_TYPE; + return c1; + + // TODO: other cases being lazy, esp. when we know inplace not likely + // could see the corresponding code for union + default: + // we may have a dirty bitset (without a precomputed cardinality) + // and calling container_ixor on it might be unsafe. + if (type1 == BITSET_CONTAINER_TYPE) { + bitset_container_t *bc = CAST_bitset(c1); + if (bc->cardinality == BITSET_UNKNOWN_CARDINALITY) { + bc->cardinality = bitset_container_compute_cardinality(bc); + } + } + return container_ixor(c1, type1, c2, type2, result_type); + } +} + +/** + * Compute difference (andnot) between two containers, generate a new + * container (having type result_type), requires a typecode. This allocates new + * memory, caller is responsible for deallocation. + */ +static inline container_t *container_andnot(const container_t *c1, + uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type) { + c1 = container_unwrap_shared(c1, &type1); + c2 = container_unwrap_shared(c2, &type2); + container_t *result = NULL; + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + *result_type = + bitset_bitset_container_andnot(const_CAST_bitset(c1), + const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, ARRAY): + result = array_container_create(); + array_array_container_andnot( + const_CAST_array(c1), const_CAST_array(c2), CAST_array(result)); + *result_type = ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(RUN, RUN): + if (run_container_is_full(const_CAST_run(c2))) { + result = array_container_create(); + *result_type = ARRAY_CONTAINER_TYPE; + return result; + } + *result_type = (uint8_t)run_run_container_andnot( + const_CAST_run(c1), const_CAST_run(c2), &result); + return result; + + case CONTAINER_PAIR(BITSET, ARRAY): + *result_type = + bitset_array_container_andnot(const_CAST_bitset(c1), + const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, BITSET): + result = array_container_create(); + array_bitset_container_andnot(const_CAST_array(c1), + const_CAST_bitset(c2), + CAST_array(result)); + *result_type = ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(BITSET, RUN): + if (run_container_is_full(const_CAST_run(c2))) { + result = array_container_create(); + *result_type = ARRAY_CONTAINER_TYPE; + return result; + } + *result_type = + bitset_run_container_andnot(const_CAST_bitset(c1), + const_CAST_run(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(RUN, BITSET): + *result_type = + run_bitset_container_andnot(const_CAST_run(c1), + const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, RUN): + if (run_container_is_full(const_CAST_run(c2))) { + result = array_container_create(); + *result_type = ARRAY_CONTAINER_TYPE; + return result; + } + result = array_container_create(); + array_run_container_andnot(const_CAST_array(c1), const_CAST_run(c2), + CAST_array(result)); + *result_type = ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(RUN, ARRAY): + *result_type = (uint8_t)run_array_container_andnot( + const_CAST_run(c1), const_CAST_array(c2), &result); + return result; + + default: + assert(false); + roaring_unreachable; + return NULL; // unreached + } +} + +/** + * Compute the andnot between two containers, with result in the first + * container. + * If the returned pointer is identical to c1, then the container has been + * modified. + * If the returned pointer is different from c1, then a new container has been + * created. The original container is freed by container_iandnot. + * The type of the first container may change. Returns the modified (and + * possibly new) container. + */ +static inline container_t *container_iandnot(container_t *c1, uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type) { + c1 = get_writable_copy_if_shared(c1, &type1); + c2 = container_unwrap_shared(c2, &type2); + container_t *result = NULL; + switch (PAIR_CONTAINER_TYPES(type1, type2)) { + case CONTAINER_PAIR(BITSET, BITSET): + *result_type = bitset_bitset_container_iandnot( + CAST_bitset(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, ARRAY): + array_array_container_iandnot(CAST_array(c1), const_CAST_array(c2)); + *result_type = ARRAY_CONTAINER_TYPE; + return c1; + + case CONTAINER_PAIR(RUN, RUN): + *result_type = (uint8_t)run_run_container_iandnot( + CAST_run(c1), const_CAST_run(c2), &result); + return result; + + case CONTAINER_PAIR(BITSET, ARRAY): + *result_type = bitset_array_container_iandnot( + CAST_bitset(c1), const_CAST_array(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, BITSET): + *result_type = ARRAY_CONTAINER_TYPE; + array_bitset_container_iandnot(CAST_array(c1), + const_CAST_bitset(c2)); + return c1; + + case CONTAINER_PAIR(BITSET, RUN): + *result_type = bitset_run_container_iandnot( + CAST_bitset(c1), const_CAST_run(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(RUN, BITSET): + *result_type = run_bitset_container_iandnot( + CAST_run(c1), const_CAST_bitset(c2), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + + case CONTAINER_PAIR(ARRAY, RUN): + *result_type = ARRAY_CONTAINER_TYPE; + array_run_container_iandnot(CAST_array(c1), const_CAST_run(c2)); + return c1; + + case CONTAINER_PAIR(RUN, ARRAY): + *result_type = (uint8_t)run_array_container_iandnot( + CAST_run(c1), const_CAST_array(c2), &result); + return result; + + default: + assert(false); + roaring_unreachable; + return NULL; + } +} + +/** + * Visit all values x of the container once, passing (base+x,ptr) + * to iterator. You need to specify a container and its type. + * Returns true if the iteration should continue. + */ +static inline bool container_iterate(const container_t *c, uint8_t type, + uint32_t base, roaring_iterator iterator, + void *ptr) { + c = container_unwrap_shared(c, &type); + switch (type) { + case BITSET_CONTAINER_TYPE: + return bitset_container_iterate(const_CAST_bitset(c), base, + iterator, ptr); + case ARRAY_CONTAINER_TYPE: + return array_container_iterate(const_CAST_array(c), base, iterator, + ptr); + case RUN_CONTAINER_TYPE: + return run_container_iterate(const_CAST_run(c), base, iterator, + ptr); + default: + assert(false); + roaring_unreachable; + } + assert(false); + roaring_unreachable; + return false; +} + +static inline bool container_iterate64(const container_t *c, uint8_t type, + uint32_t base, + roaring_iterator64 iterator, + uint64_t high_bits, void *ptr) { + c = container_unwrap_shared(c, &type); + switch (type) { + case BITSET_CONTAINER_TYPE: + return bitset_container_iterate64(const_CAST_bitset(c), base, + iterator, high_bits, ptr); + case ARRAY_CONTAINER_TYPE: + return array_container_iterate64(const_CAST_array(c), base, + iterator, high_bits, ptr); + case RUN_CONTAINER_TYPE: + return run_container_iterate64(const_CAST_run(c), base, iterator, + high_bits, ptr); + default: + assert(false); + roaring_unreachable; + } + assert(false); + roaring_unreachable; + return false; +} + +static inline container_t *container_not(const container_t *c, uint8_t type, + uint8_t *result_type) { + c = container_unwrap_shared(c, &type); + container_t *result = NULL; + switch (type) { + case BITSET_CONTAINER_TYPE: + *result_type = + bitset_container_negation(const_CAST_bitset(c), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + case ARRAY_CONTAINER_TYPE: + result = bitset_container_create(); + *result_type = BITSET_CONTAINER_TYPE; + array_container_negation(const_CAST_array(c), CAST_bitset(result)); + return result; + case RUN_CONTAINER_TYPE: + *result_type = + (uint8_t)run_container_negation(const_CAST_run(c), &result); + return result; + + default: + assert(false); + roaring_unreachable; + } + assert(false); + roaring_unreachable; + return NULL; +} + +static inline container_t *container_not_range(const container_t *c, + uint8_t type, + uint32_t range_start, + uint32_t range_end, + uint8_t *result_type) { + c = container_unwrap_shared(c, &type); + container_t *result = NULL; + switch (type) { + case BITSET_CONTAINER_TYPE: + *result_type = + bitset_container_negation_range(const_CAST_bitset(c), + range_start, range_end, &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + case ARRAY_CONTAINER_TYPE: + *result_type = + array_container_negation_range(const_CAST_array(c), range_start, + range_end, &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + case RUN_CONTAINER_TYPE: + *result_type = (uint8_t)run_container_negation_range( + const_CAST_run(c), range_start, range_end, &result); + return result; + + default: + assert(false); + roaring_unreachable; + } + assert(false); + roaring_unreachable; + return NULL; +} + +static inline container_t *container_inot(container_t *c, uint8_t type, + uint8_t *result_type) { + c = get_writable_copy_if_shared(c, &type); + container_t *result = NULL; + switch (type) { + case BITSET_CONTAINER_TYPE: + *result_type = + bitset_container_negation_inplace(CAST_bitset(c), &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + case ARRAY_CONTAINER_TYPE: + // will never be inplace + result = bitset_container_create(); + *result_type = BITSET_CONTAINER_TYPE; + array_container_negation(CAST_array(c), CAST_bitset(result)); + array_container_free(CAST_array(c)); + return result; + case RUN_CONTAINER_TYPE: + *result_type = + (uint8_t)run_container_negation_inplace(CAST_run(c), &result); + return result; + + default: + assert(false); + roaring_unreachable; + } + assert(false); + roaring_unreachable; + return NULL; +} + +static inline container_t *container_inot_range(container_t *c, uint8_t type, + uint32_t range_start, + uint32_t range_end, + uint8_t *result_type) { + c = get_writable_copy_if_shared(c, &type); + container_t *result = NULL; + switch (type) { + case BITSET_CONTAINER_TYPE: + *result_type = bitset_container_negation_range_inplace( + CAST_bitset(c), range_start, range_end, &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + case ARRAY_CONTAINER_TYPE: + *result_type = array_container_negation_range_inplace( + CAST_array(c), range_start, range_end, &result) + ? BITSET_CONTAINER_TYPE + : ARRAY_CONTAINER_TYPE; + return result; + case RUN_CONTAINER_TYPE: + *result_type = (uint8_t)run_container_negation_range_inplace( + CAST_run(c), range_start, range_end, &result); + return result; + + default: + assert(false); + roaring_unreachable; + } + assert(false); + roaring_unreachable; + return NULL; +} + +/** + * If the element of given rank is in this container, supposing that + * the first + * element has rank start_rank, then the function returns true and + * sets element + * accordingly. + * Otherwise, it returns false and update start_rank. + */ +static inline bool container_select(const container_t *c, uint8_t type, + uint32_t *start_rank, uint32_t rank, + uint32_t *element) { + c = container_unwrap_shared(c, &type); + switch (type) { + case BITSET_CONTAINER_TYPE: + return bitset_container_select(const_CAST_bitset(c), start_rank, + rank, element); + case ARRAY_CONTAINER_TYPE: + return array_container_select(const_CAST_array(c), start_rank, rank, + element); + case RUN_CONTAINER_TYPE: + return run_container_select(const_CAST_run(c), start_rank, rank, + element); + default: + assert(false); + roaring_unreachable; + } + assert(false); + roaring_unreachable; + return false; +} + +static inline uint16_t container_maximum(const container_t *c, uint8_t type) { + c = container_unwrap_shared(c, &type); + switch (type) { + case BITSET_CONTAINER_TYPE: + return bitset_container_maximum(const_CAST_bitset(c)); + case ARRAY_CONTAINER_TYPE: + return array_container_maximum(const_CAST_array(c)); + case RUN_CONTAINER_TYPE: + return run_container_maximum(const_CAST_run(c)); + default: + assert(false); + roaring_unreachable; + } + assert(false); + roaring_unreachable; + return false; +} + +static inline uint16_t container_minimum(const container_t *c, uint8_t type) { + c = container_unwrap_shared(c, &type); + switch (type) { + case BITSET_CONTAINER_TYPE: + return bitset_container_minimum(const_CAST_bitset(c)); + case ARRAY_CONTAINER_TYPE: + return array_container_minimum(const_CAST_array(c)); + case RUN_CONTAINER_TYPE: + return run_container_minimum(const_CAST_run(c)); + default: + assert(false); + roaring_unreachable; + } + assert(false); + roaring_unreachable; + return false; +} + +// number of values smaller or equal to x +static inline int container_rank(const container_t *c, uint8_t type, + uint16_t x) { + c = container_unwrap_shared(c, &type); + switch (type) { + case BITSET_CONTAINER_TYPE: + return bitset_container_rank(const_CAST_bitset(c), x); + case ARRAY_CONTAINER_TYPE: + return array_container_rank(const_CAST_array(c), x); + case RUN_CONTAINER_TYPE: + return run_container_rank(const_CAST_run(c), x); + default: + assert(false); + roaring_unreachable; + } + assert(false); + roaring_unreachable; + return false; +} + +// bulk version of container_rank(); return number of consumed elements +static inline uint32_t container_rank_many(const container_t *c, uint8_t type, + uint64_t start_rank, + const uint32_t *begin, + const uint32_t *end, uint64_t *ans) { + c = container_unwrap_shared(c, &type); + switch (type) { + case BITSET_CONTAINER_TYPE: + return bitset_container_rank_many(const_CAST_bitset(c), start_rank, + begin, end, ans); + case ARRAY_CONTAINER_TYPE: + return array_container_rank_many(const_CAST_array(c), start_rank, + begin, end, ans); + case RUN_CONTAINER_TYPE: + return run_container_rank_many(const_CAST_run(c), start_rank, begin, + end, ans); + default: + assert(false); + roaring_unreachable; + } + assert(false); + roaring_unreachable; + return 0; +} + +// return the index of x, if not exsist return -1 +static inline int container_get_index(const container_t *c, uint8_t type, + uint16_t x) { + c = container_unwrap_shared(c, &type); + switch (type) { + case BITSET_CONTAINER_TYPE: + return bitset_container_get_index(const_CAST_bitset(c), x); + case ARRAY_CONTAINER_TYPE: + return array_container_get_index(const_CAST_array(c), x); + case RUN_CONTAINER_TYPE: + return run_container_get_index(const_CAST_run(c), x); + default: + assert(false); + roaring_unreachable; + } + assert(false); + roaring_unreachable; + return false; +} + +/** + * Add all values in range [min, max] to a given container. + * + * If the returned pointer is different from $container, then a new container + * has been created and the caller is responsible for freeing it. + * The type of the first container may change. Returns the modified + * (and possibly new) container. + */ +static inline container_t *container_add_range(container_t *c, uint8_t type, + uint32_t min, uint32_t max, + uint8_t *result_type) { + // NB: when selecting new container type, we perform only inexpensive checks + switch (type) { + case BITSET_CONTAINER_TYPE: { + bitset_container_t *bitset = CAST_bitset(c); + + int32_t union_cardinality = 0; + union_cardinality += bitset->cardinality; + union_cardinality += max - min + 1; + union_cardinality -= + bitset_lenrange_cardinality(bitset->words, min, max - min); + + if (union_cardinality == INT32_C(0x10000)) { + *result_type = RUN_CONTAINER_TYPE; + return run_container_create_range(0, INT32_C(0x10000)); + } else { + *result_type = BITSET_CONTAINER_TYPE; + bitset_set_lenrange(bitset->words, min, max - min); + bitset->cardinality = union_cardinality; + return bitset; + } + } + case ARRAY_CONTAINER_TYPE: { + array_container_t *array = CAST_array(c); + + int32_t nvals_greater = + count_greater(array->array, array->cardinality, (uint16_t)max); + int32_t nvals_less = + count_less(array->array, array->cardinality - nvals_greater, + (uint16_t)min); + int32_t union_cardinality = + nvals_less + (max - min + 1) + nvals_greater; + + if (union_cardinality == INT32_C(0x10000)) { + *result_type = RUN_CONTAINER_TYPE; + return run_container_create_range(0, INT32_C(0x10000)); + } else if (union_cardinality <= DEFAULT_MAX_SIZE) { + *result_type = ARRAY_CONTAINER_TYPE; + array_container_add_range_nvals(array, min, max, nvals_less, + nvals_greater); + return array; + } else { + *result_type = BITSET_CONTAINER_TYPE; + bitset_container_t *bitset = bitset_container_from_array(array); + bitset_set_lenrange(bitset->words, min, max - min); + bitset->cardinality = union_cardinality; + return bitset; + } + } + case RUN_CONTAINER_TYPE: { + run_container_t *run = CAST_run(c); + + int32_t nruns_greater = + rle16_count_greater(run->runs, run->n_runs, (uint16_t)max); + int32_t nruns_less = rle16_count_less( + run->runs, run->n_runs - nruns_greater, (uint16_t)min); + + int32_t run_size_bytes = + (nruns_less + 1 + nruns_greater) * sizeof(rle16_t); + int32_t bitset_size_bytes = + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + + if (run_size_bytes <= bitset_size_bytes) { + run_container_add_range_nruns(run, min, max, nruns_less, + nruns_greater); + *result_type = RUN_CONTAINER_TYPE; + return run; + } else { + return container_from_run_range(run, min, max, result_type); + } + } + default: + roaring_unreachable; + } +} + +/* + * Removes all elements in range [min, max]. + * Returns one of: + * - NULL if no elements left + * - pointer to the original container + * - pointer to a newly-allocated container (if it is more efficient) + * + * If the returned pointer is different from $container, then a new container + * has been created and the caller is responsible for freeing the original + * container. + */ +static inline container_t *container_remove_range(container_t *c, uint8_t type, + uint32_t min, uint32_t max, + uint8_t *result_type) { + switch (type) { + case BITSET_CONTAINER_TYPE: { + bitset_container_t *bitset = CAST_bitset(c); + + int32_t result_cardinality = + bitset->cardinality - + bitset_lenrange_cardinality(bitset->words, min, max - min); + + if (result_cardinality == 0) { + return NULL; + } else if (result_cardinality <= DEFAULT_MAX_SIZE) { + *result_type = ARRAY_CONTAINER_TYPE; + bitset_reset_range(bitset->words, min, max + 1); + bitset->cardinality = result_cardinality; + return array_container_from_bitset(bitset); + } else { + *result_type = BITSET_CONTAINER_TYPE; + bitset_reset_range(bitset->words, min, max + 1); + bitset->cardinality = result_cardinality; + return bitset; + } + } + case ARRAY_CONTAINER_TYPE: { + array_container_t *array = CAST_array(c); + + int32_t nvals_greater = + count_greater(array->array, array->cardinality, (uint16_t)max); + int32_t nvals_less = + count_less(array->array, array->cardinality - nvals_greater, + (uint16_t)min); + int32_t result_cardinality = nvals_less + nvals_greater; + + if (result_cardinality == 0) { + return NULL; + } else { + *result_type = ARRAY_CONTAINER_TYPE; + array_container_remove_range( + array, nvals_less, array->cardinality - result_cardinality); + return array; + } + } + case RUN_CONTAINER_TYPE: { + run_container_t *run = CAST_run(c); + + if (run->n_runs == 0) { + return NULL; + } + if (min <= run_container_minimum(run) && + max >= run_container_maximum(run)) { + return NULL; + } + + run_container_remove_range(run, min, max); + return convert_run_to_efficient_container(run, result_type); + } + default: + roaring_unreachable; + } +} + +#ifdef __cplusplus +using api::roaring_container_iterator_t; +#endif + +/** + * Initializes the iterator at the first entry in the container. + */ +roaring_container_iterator_t container_init_iterator(const container_t *c, + uint8_t typecode, + uint16_t *value); + +/** + * Initializes the iterator at the last entry in the container. + */ +roaring_container_iterator_t container_init_iterator_last(const container_t *c, + uint8_t typecode, + uint16_t *value); + +/** + * Moves the iterator to the next entry. Returns true and sets `value` if a + * value is present. + */ +bool container_iterator_next(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, uint16_t *value); + +/** + * Moves the iterator to the previous entry. Returns true and sets `value` if a + * value is present. + */ +bool container_iterator_prev(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, uint16_t *value); + +/** + * Moves the iterator to the smallest entry that is greater than or equal to + * `val`. Returns true and sets `value_out` if a value is present. `value_out` + * should be initialized to a value. + */ +bool container_iterator_lower_bound(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value_out, uint16_t val); + +/** + * Reads up to `count` entries from the container, and writes them into `buf` + * as `high16 | entry`. Returns true and sets `value_out` if a value is present + * after reading the entries. Sets `consumed` to the number of values read. + * `count` should be greater than zero. + */ +bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint32_t high16, uint32_t *buf, + uint32_t count, uint32_t *consumed, + uint16_t *value_out); + +/** + * Reads up to `count` entries from the container, and writes them into `buf` + * as `high48 | entry`. Returns true and sets `value_out` if a value is present + * after reading the entries. Sets `consumed` to the number of values read. + * `count` should be greater than zero. + */ +bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint64_t high48, uint64_t *buf, + uint32_t count, uint32_t *consumed, + uint16_t *value_out); + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#endif diff --git a/contrib/libs/croaring/include/roaring/containers/convert.h b/contrib/libs/croaring/include/roaring/containers/convert.h new file mode 100644 index 00000000000..923c0c218f0 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/containers/convert.h @@ -0,0 +1,73 @@ +/* + * convert.h + * + */ + +#ifndef INCLUDE_CONTAINERS_CONVERT_H_ +#define INCLUDE_CONTAINERS_CONVERT_H_ + +#include <roaring/containers/array.h> +#include <roaring/containers/bitset.h> +#include <roaring/containers/run.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/* Convert an array into a bitset. The input container is not freed or modified. + */ +bitset_container_t *bitset_container_from_array(const array_container_t *arr); + +/* Convert a run into a bitset. The input container is not freed or modified. */ +bitset_container_t *bitset_container_from_run(const run_container_t *arr); + +/* Convert a run into an array. The input container is not freed or modified. */ +array_container_t *array_container_from_run(const run_container_t *arr); + +/* Convert a bitset into an array. The input container is not freed or modified. + */ +array_container_t *array_container_from_bitset(const bitset_container_t *bits); + +/* Convert an array into a run. The input container is not freed or modified. + */ +run_container_t *run_container_from_array(const array_container_t *c); + +/* convert a run into either an array or a bitset + * might free the container. This does not free the input run container. */ +container_t *convert_to_bitset_or_array_container(run_container_t *rc, + int32_t card, + uint8_t *resulttype); + +/* convert containers to and from runcontainers, as is most space efficient. + * The container might be freed. */ +container_t *convert_run_optimize(container_t *c, uint8_t typecode_original, + uint8_t *typecode_after); + +/* converts a run container to either an array or a bitset, IF it saves space. + */ +/* If a conversion occurs, the caller is responsible to free the original + * container and + * he becomes reponsible to free the new one. */ +container_t *convert_run_to_efficient_container(run_container_t *c, + uint8_t *typecode_after); + +// like convert_run_to_efficient_container but frees the old result if needed +container_t *convert_run_to_efficient_container_and_free( + run_container_t *c, uint8_t *typecode_after); + +/** + * Create new container which is a union of run container and + * range [min, max]. Caller is responsible for freeing run container. + */ +container_t *container_from_run_range(const run_container_t *run, uint32_t min, + uint32_t max, uint8_t *typecode_after); + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#endif /* INCLUDE_CONTAINERS_CONVERT_H_ */ diff --git a/contrib/libs/croaring/include/roaring/containers/mixed_andnot.h b/contrib/libs/croaring/include/roaring/containers/mixed_andnot.h new file mode 100644 index 00000000000..f7ba19de9ab --- /dev/null +++ b/contrib/libs/croaring/include/roaring/containers/mixed_andnot.h @@ -0,0 +1,181 @@ +/* + * mixed_andnot.h + */ +#ifndef INCLUDE_CONTAINERS_MIXED_ANDNOT_H_ +#define INCLUDE_CONTAINERS_MIXED_ANDNOT_H_ + +#include <roaring/containers/array.h> +#include <roaring/containers/bitset.h> +#include <roaring/containers/run.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst, a valid array container that could be the same as dst.*/ +void array_bitset_container_andnot(const array_container_t *src_1, + const bitset_container_t *src_2, + array_container_t *dst); + +/* Compute the andnot of src_1 and src_2 and write the result to + * src_1 */ + +void array_bitset_container_iandnot(array_container_t *src_1, + const bitset_container_t *src_2); + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst, which does not initially have a valid container. + * Return true for a bitset result; false for array + */ + +bool bitset_array_container_andnot(const bitset_container_t *src_1, + const array_container_t *src_2, + container_t **dst); + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst (which has no container initially). It will modify src_1 + * to be dst if the result is a bitset. Otherwise, it will + * free src_1 and dst will be a new array container. In both + * cases, the caller is responsible for deallocating dst. + * Returns true iff dst is a bitset */ + +bool bitset_array_container_iandnot(bitset_container_t *src_1, + const array_container_t *src_2, + container_t **dst); + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst. Result may be either a bitset or an array container + * (returns "result is bitset"). dst does not initially have + * any container, but becomes either a bitset container (return + * result true) or an array container. + */ + +bool run_bitset_container_andnot(const run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst. Result may be either a bitset or an array container + * (returns "result is bitset"). dst does not initially have + * any container, but becomes either a bitset container (return + * result true) or an array container. + */ + +bool run_bitset_container_iandnot(run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst. Result may be either a bitset or an array container + * (returns "result is bitset"). dst does not initially have + * any container, but becomes either a bitset container (return + * result true) or an array container. + */ + +bool bitset_run_container_andnot(const bitset_container_t *src_1, + const run_container_t *src_2, + container_t **dst); + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst (which has no container initially). It will modify src_1 + * to be dst if the result is a bitset. Otherwise, it will + * free src_1 and dst will be a new array container. In both + * cases, the caller is responsible for deallocating dst. + * Returns true iff dst is a bitset */ + +bool bitset_run_container_iandnot(bitset_container_t *src_1, + const run_container_t *src_2, + container_t **dst); + +/* dst does not indicate a valid container initially. Eventually it + * can become any type of container. + */ + +int run_array_container_andnot(const run_container_t *src_1, + const array_container_t *src_2, + container_t **dst); + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst (which has no container initially). It will modify src_1 + * to be dst if the result is a bitset. Otherwise, it will + * free src_1 and dst will be a new array container. In both + * cases, the caller is responsible for deallocating dst. + * Returns true iff dst is a bitset */ + +int run_array_container_iandnot(run_container_t *src_1, + const array_container_t *src_2, + container_t **dst); + +/* dst must be a valid array container, allowed to be src_1 */ + +void array_run_container_andnot(const array_container_t *src_1, + const run_container_t *src_2, + array_container_t *dst); + +/* dst does not indicate a valid container initially. Eventually it + * can become any kind of container. + */ + +void array_run_container_iandnot(array_container_t *src_1, + const run_container_t *src_2); + +/* dst does not indicate a valid container initially. Eventually it + * can become any kind of container. + */ + +int run_run_container_andnot(const run_container_t *src_1, + const run_container_t *src_2, container_t **dst); + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst (which has no container initially). It will modify src_1 + * to be dst if the result is a bitset. Otherwise, it will + * free src_1 and dst will be a new array container. In both + * cases, the caller is responsible for deallocating dst. + * Returns true iff dst is a bitset */ + +int run_run_container_iandnot(run_container_t *src_1, + const run_container_t *src_2, container_t **dst); + +/* + * dst is a valid array container and may be the same as src_1 + */ + +void array_array_container_andnot(const array_container_t *src_1, + const array_container_t *src_2, + array_container_t *dst); + +/* inplace array-array andnot will always be able to reuse the space of + * src_1 */ +void array_array_container_iandnot(array_container_t *src_1, + const array_container_t *src_2); + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst (which has no container initially). Return value is + * "dst is a bitset" + */ + +bool bitset_bitset_container_andnot(const bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst (which has no container initially). It will modify src_1 + * to be dst if the result is a bitset. Otherwise, it will + * free src_1 and dst will be a new array container. In both + * cases, the caller is responsible for deallocating dst. + * Returns true iff dst is a bitset */ + +bool bitset_bitset_container_iandnot(bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#endif diff --git a/contrib/libs/croaring/include/roaring/containers/mixed_equal.h b/contrib/libs/croaring/include/roaring/containers/mixed_equal.h new file mode 100644 index 00000000000..f1b236e0253 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/containers/mixed_equal.h @@ -0,0 +1,42 @@ +/* + * mixed_equal.h + * + */ + +#ifndef CONTAINERS_MIXED_EQUAL_H_ +#define CONTAINERS_MIXED_EQUAL_H_ + +#include <roaring/containers/array.h> +#include <roaring/containers/bitset.h> +#include <roaring/containers/run.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/** + * Return true if the two containers have the same content. + */ +bool array_container_equal_bitset(const array_container_t* container1, + const bitset_container_t* container2); + +/** + * Return true if the two containers have the same content. + */ +bool run_container_equals_array(const run_container_t* container1, + const array_container_t* container2); +/** + * Return true if the two containers have the same content. + */ +bool run_container_equals_bitset(const run_container_t* container1, + const bitset_container_t* container2); + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#endif /* CONTAINERS_MIXED_EQUAL_H_ */ diff --git a/contrib/libs/croaring/include/roaring/containers/mixed_intersection.h b/contrib/libs/croaring/include/roaring/containers/mixed_intersection.h new file mode 100644 index 00000000000..53c85298d9a --- /dev/null +++ b/contrib/libs/croaring/include/roaring/containers/mixed_intersection.h @@ -0,0 +1,101 @@ +/* + * mixed_intersection.h + * + */ + +#ifndef INCLUDE_CONTAINERS_MIXED_INTERSECTION_H_ +#define INCLUDE_CONTAINERS_MIXED_INTERSECTION_H_ + +/* These functions appear to exclude cases where the + * inputs have the same type and the output is guaranteed + * to have the same type as the inputs. Eg, array intersection + */ + +#include <roaring/containers/array.h> +#include <roaring/containers/bitset.h> +#include <roaring/containers/run.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/* Compute the intersection of src_1 and src_2 and write the result to + * dst. It is allowed for dst to be equal to src_1. We assume that dst is a + * valid container. */ +void array_bitset_container_intersection(const array_container_t *src_1, + const bitset_container_t *src_2, + array_container_t *dst); + +/* Compute the size of the intersection of src_1 and src_2. */ +int array_bitset_container_intersection_cardinality( + const array_container_t *src_1, const bitset_container_t *src_2); + +/* Checking whether src_1 and src_2 intersect. */ +bool array_bitset_container_intersect(const array_container_t *src_1, + const bitset_container_t *src_2); + +/* + * Compute the intersection between src_1 and src_2 and write the result + * to *dst. If the return function is true, the result is a bitset_container_t + * otherwise is a array_container_t. We assume that dst is not pre-allocated. In + * case of failure, *dst will be NULL. + */ +bool bitset_bitset_container_intersection(const bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); + +/* Compute the intersection between src_1 and src_2 and write the result to + * dst. It is allowed for dst to be equal to src_1. We assume that dst is a + * valid container. */ +void array_run_container_intersection(const array_container_t *src_1, + const run_container_t *src_2, + array_container_t *dst); + +/* Compute the intersection between src_1 and src_2 and write the result to + * *dst. If the result is true then the result is a bitset_container_t + * otherwise is a array_container_t. + * If *dst == src_2, then an in-place intersection is attempted + **/ +bool run_bitset_container_intersection(const run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); + +/* Compute the size of the intersection between src_1 and src_2 . */ +int array_run_container_intersection_cardinality(const array_container_t *src_1, + const run_container_t *src_2); + +/* Compute the size of the intersection between src_1 and src_2 + **/ +int run_bitset_container_intersection_cardinality( + const run_container_t *src_1, const bitset_container_t *src_2); + +/* Check that src_1 and src_2 intersect. */ +bool array_run_container_intersect(const array_container_t *src_1, + const run_container_t *src_2); + +/* Check that src_1 and src_2 intersect. + **/ +bool run_bitset_container_intersect(const run_container_t *src_1, + const bitset_container_t *src_2); + +/* + * Same as bitset_bitset_container_intersection except that if the output is to + * be a + * bitset_container_t, then src_1 is modified and no allocation is made. + * If the output is to be an array_container_t, then caller is responsible + * to free the container. + * In all cases, the result is in *dst. + */ +bool bitset_bitset_container_intersection_inplace( + bitset_container_t *src_1, const bitset_container_t *src_2, + container_t **dst); + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#endif /* INCLUDE_CONTAINERS_MIXED_INTERSECTION_H_ */ diff --git a/contrib/libs/croaring/include/roaring/containers/mixed_negation.h b/contrib/libs/croaring/include/roaring/containers/mixed_negation.h new file mode 100644 index 00000000000..3788ca413b5 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/containers/mixed_negation.h @@ -0,0 +1,140 @@ +/* + * mixed_negation.h + * + */ + +#ifndef INCLUDE_CONTAINERS_MIXED_NEGATION_H_ +#define INCLUDE_CONTAINERS_MIXED_NEGATION_H_ + +#include <roaring/containers/array.h> +#include <roaring/containers/bitset.h> +#include <roaring/containers/run.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/* Negation across the entire range of the container. + * Compute the negation of src and write the result + * to *dst. The complement of a + * sufficiently sparse set will always be dense and a hence a bitmap + * We assume that dst is pre-allocated and a valid bitset container + * There can be no in-place version. + */ +void array_container_negation(const array_container_t *src, + bitset_container_t *dst); + +/* Negation across the entire range of the container + * Compute the negation of src and write the result + * to *dst. A true return value indicates a bitset result, + * otherwise the result is an array container. + * We assume that dst is not pre-allocated. In + * case of failure, *dst will be NULL. + */ +bool bitset_container_negation(const bitset_container_t *src, + container_t **dst); + +/* inplace version */ +/* + * Same as bitset_container_negation except that if the output is to + * be a + * bitset_container_t, then src is modified and no allocation is made. + * If the output is to be an array_container_t, then caller is responsible + * to free the container. + * In all cases, the result is in *dst. + */ +bool bitset_container_negation_inplace(bitset_container_t *src, + container_t **dst); + +/* Negation across the entire range of container + * Compute the negation of src and write the result + * to *dst. + * Return values are the *_TYPECODES as defined * in containers.h + * We assume that dst is not pre-allocated. In + * case of failure, *dst will be NULL. + */ +int run_container_negation(const run_container_t *src, container_t **dst); + +/* + * Same as run_container_negation except that if the output is to + * be a + * run_container_t, and has the capacity to hold the result, + * then src is modified and no allocation is made. + * In all cases, the result is in *dst. + */ +int run_container_negation_inplace(run_container_t *src, container_t **dst); + +/* Negation across a range of the container. + * Compute the negation of src and write the result + * to *dst. Returns true if the result is a bitset container + * and false for an array container. *dst is not preallocated. + */ +bool array_container_negation_range(const array_container_t *src, + const int range_start, const int range_end, + container_t **dst); + +/* Even when the result would fit, it is unclear how to make an + * inplace version without inefficient copying. Thus this routine + * may be a wrapper for the non-in-place version + */ +bool array_container_negation_range_inplace(array_container_t *src, + const int range_start, + const int range_end, + container_t **dst); + +/* Negation across a range of the container + * Compute the negation of src and write the result + * to *dst. A true return value indicates a bitset result, + * otherwise the result is an array container. + * We assume that dst is not pre-allocated. In + * case of failure, *dst will be NULL. + */ +bool bitset_container_negation_range(const bitset_container_t *src, + const int range_start, const int range_end, + container_t **dst); + +/* inplace version */ +/* + * Same as bitset_container_negation except that if the output is to + * be a + * bitset_container_t, then src is modified and no allocation is made. + * If the output is to be an array_container_t, then caller is responsible + * to free the container. + * In all cases, the result is in *dst. + */ +bool bitset_container_negation_range_inplace(bitset_container_t *src, + const int range_start, + const int range_end, + container_t **dst); + +/* Negation across a range of container + * Compute the negation of src and write the result + * to *dst. Return values are the *_TYPECODES as defined * in containers.h + * We assume that dst is not pre-allocated. In + * case of failure, *dst will be NULL. + */ +int run_container_negation_range(const run_container_t *src, + const int range_start, const int range_end, + container_t **dst); + +/* + * Same as run_container_negation except that if the output is to + * be a + * run_container_t, and has the capacity to hold the result, + * then src is modified and no allocation is made. + * In all cases, the result is in *dst. + */ +int run_container_negation_range_inplace(run_container_t *src, + const int range_start, + const int range_end, + container_t **dst); + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#endif /* INCLUDE_CONTAINERS_MIXED_NEGATION_H_ */ diff --git a/contrib/libs/croaring/include/roaring/containers/mixed_subset.h b/contrib/libs/croaring/include/roaring/containers/mixed_subset.h new file mode 100644 index 00000000000..e7ce1f866cd --- /dev/null +++ b/contrib/libs/croaring/include/roaring/containers/mixed_subset.h @@ -0,0 +1,55 @@ +/* + * mixed_subset.h + * + */ + +#ifndef CONTAINERS_MIXED_SUBSET_H_ +#define CONTAINERS_MIXED_SUBSET_H_ + +#include <roaring/containers/array.h> +#include <roaring/containers/bitset.h> +#include <roaring/containers/run.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/** + * Return true if container1 is a subset of container2. + */ +bool array_container_is_subset_bitset(const array_container_t* container1, + const bitset_container_t* container2); + +/** + * Return true if container1 is a subset of container2. + */ +bool run_container_is_subset_array(const run_container_t* container1, + const array_container_t* container2); + +/** + * Return true if container1 is a subset of container2. + */ +bool array_container_is_subset_run(const array_container_t* container1, + const run_container_t* container2); + +/** + * Return true if container1 is a subset of container2. + */ +bool run_container_is_subset_bitset(const run_container_t* container1, + const bitset_container_t* container2); + +/** + * Return true if container1 is a subset of container2. + */ +bool bitset_container_is_subset_run(const bitset_container_t* container1, + const run_container_t* container2); + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#endif /* CONTAINERS_MIXED_SUBSET_H_ */ diff --git a/contrib/libs/croaring/include/roaring/containers/mixed_union.h b/contrib/libs/croaring/include/roaring/containers/mixed_union.h new file mode 100644 index 00000000000..0ff439217e5 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/containers/mixed_union.h @@ -0,0 +1,117 @@ +/* + * mixed_intersection.h + * + */ + +#ifndef INCLUDE_CONTAINERS_MIXED_UNION_H_ +#define INCLUDE_CONTAINERS_MIXED_UNION_H_ + +/* These functions appear to exclude cases where the + * inputs have the same type and the output is guaranteed + * to have the same type as the inputs. Eg, bitset unions + */ + +#include <roaring/containers/array.h> +#include <roaring/containers/bitset.h> +#include <roaring/containers/run.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/* Compute the union of src_1 and src_2 and write the result to + * dst. It is allowed for src_2 to be dst. */ +void array_bitset_container_union(const array_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* Compute the union of src_1 and src_2 and write the result to + * dst. It is allowed for src_2 to be dst. This version does not + * update the cardinality of dst (it is set to BITSET_UNKNOWN_CARDINALITY). */ +void array_bitset_container_lazy_union(const array_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* + * Compute the union between src_1 and src_2 and write the result + * to *dst. If the return function is true, the result is a bitset_container_t + * otherwise is a array_container_t. We assume that dst is not pre-allocated. In + * case of failure, *dst will be NULL. + */ +bool array_array_container_union(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst); + +/* + * Compute the union between src_1 and src_2 and write the result + * to *dst if it cannot be written to src_1. If the return function is true, + * the result is a bitset_container_t + * otherwise is a array_container_t. When the result is an array_container_t, it + * it either written to src_1 (if *dst is null) or to *dst. + * If the result is a bitset_container_t and *dst is null, then there was a + * failure. + */ +bool array_array_container_inplace_union(array_container_t *src_1, + const array_container_t *src_2, + container_t **dst); + +/* + * Same as array_array_container_union except that it will more eagerly produce + * a bitset. + */ +bool array_array_container_lazy_union(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst); + +/* + * Same as array_array_container_inplace_union except that it will more eagerly + * produce a bitset. + */ +bool array_array_container_lazy_inplace_union(array_container_t *src_1, + const array_container_t *src_2, + container_t **dst); + +/* Compute the union of src_1 and src_2 and write the result to + * dst. We assume that dst is a + * valid container. The result might need to be further converted to array or + * bitset container, + * the caller is responsible for the eventual conversion. */ +void array_run_container_union(const array_container_t *src_1, + const run_container_t *src_2, + run_container_t *dst); + +/* Compute the union of src_1 and src_2 and write the result to + * src2. The result might need to be further converted to array or + * bitset container, + * the caller is responsible for the eventual conversion. */ +void array_run_container_inplace_union(const array_container_t *src_1, + run_container_t *src_2); + +/* Compute the union of src_1 and src_2 and write the result to + * dst. It is allowed for dst to be src_2. + * If run_container_is_full(src_1) is true, you must not be calling this + *function. + **/ +void run_bitset_container_union(const run_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* Compute the union of src_1 and src_2 and write the result to + * dst. It is allowed for dst to be src_2. This version does not + * update the cardinality of dst (it is set to BITSET_UNKNOWN_CARDINALITY). + * If run_container_is_full(src_1) is true, you must not be calling this + * function. + * */ +void run_bitset_container_lazy_union(const run_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#endif /* INCLUDE_CONTAINERS_MIXED_UNION_H_ */ diff --git a/contrib/libs/croaring/include/roaring/containers/mixed_xor.h b/contrib/libs/croaring/include/roaring/containers/mixed_xor.h new file mode 100644 index 00000000000..8000898bcf3 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/containers/mixed_xor.h @@ -0,0 +1,177 @@ +/* + * mixed_xor.h + * + */ + +#ifndef INCLUDE_CONTAINERS_MIXED_XOR_H_ +#define INCLUDE_CONTAINERS_MIXED_XOR_H_ + +/* These functions appear to exclude cases where the + * inputs have the same type and the output is guaranteed + * to have the same type as the inputs. Eg, bitset unions + */ + +/* + * Java implementation (as of May 2016) for array_run, run_run + * and bitset_run don't do anything different for inplace. + * (They are not truly in place.) + */ + +#include <roaring/containers/array.h> +#include <roaring/containers/bitset.h> +#include <roaring/containers/run.h> + +// #include "containers.h" + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/* Compute the xor of src_1 and src_2 and write the result to + * dst (which has no container initially). + * Result is true iff dst is a bitset */ +bool array_bitset_container_xor(const array_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); + +/* Compute the xor of src_1 and src_2 and write the result to + * dst. It is allowed for src_2 to be dst. This version does not + * update the cardinality of dst (it is set to BITSET_UNKNOWN_CARDINALITY). + */ + +void array_bitset_container_lazy_xor(const array_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); +/* Compute the xor of src_1 and src_2 and write the result to + * dst (which has no container initially). Return value is + * "dst is a bitset" + */ + +bool bitset_bitset_container_xor(const bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); + +/* Compute the xor of src_1 and src_2 and write the result to + * dst. Result may be either a bitset or an array container + * (returns "result is bitset"). dst does not initially have + * any container, but becomes either a bitset container (return + * result true) or an array container. + */ + +bool run_bitset_container_xor(const run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); + +/* lazy xor. Dst is initialized and may be equal to src_2. + * Result is left as a bitset container, even if actual + * cardinality would dictate an array container. + */ + +void run_bitset_container_lazy_xor(const run_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst); + +/* dst does not indicate a valid container initially. Eventually it + * can become any kind of container. + */ + +int array_run_container_xor(const array_container_t *src_1, + const run_container_t *src_2, container_t **dst); + +/* dst does not initially have a valid container. Creates either + * an array or a bitset container, indicated by return code + */ + +bool array_array_container_xor(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst); + +/* dst does not initially have a valid container. Creates either + * an array or a bitset container, indicated by return code. + * A bitset container will not have a valid cardinality and the + * container type might not be correct for the actual cardinality + */ + +bool array_array_container_lazy_xor(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst); + +/* Dst is a valid run container. (Can it be src_2? Let's say not.) + * Leaves result as run container, even if other options are + * smaller. + */ + +void array_run_container_lazy_xor(const array_container_t *src_1, + const run_container_t *src_2, + run_container_t *dst); + +/* dst does not indicate a valid container initially. Eventually it + * can become any kind of container. + */ + +int run_run_container_xor(const run_container_t *src_1, + const run_container_t *src_2, container_t **dst); + +/* INPLACE versions (initial implementation may not exploit all inplace + * opportunities (if any...) + */ + +/* Compute the xor of src_1 and src_2 and write the result to + * dst (which has no container initially). It will modify src_1 + * to be dst if the result is a bitset. Otherwise, it will + * free src_1 and dst will be a new array container. In both + * cases, the caller is responsible for deallocating dst. + * Returns true iff dst is a bitset */ + +bool bitset_array_container_ixor(bitset_container_t *src_1, + const array_container_t *src_2, + container_t **dst); + +bool bitset_bitset_container_ixor(bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); + +bool array_bitset_container_ixor(array_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); + +/* Compute the xor of src_1 and src_2 and write the result to + * dst. Result may be either a bitset or an array container + * (returns "result is bitset"). dst does not initially have + * any container, but becomes either a bitset container (return + * result true) or an array container. + */ + +bool run_bitset_container_ixor(run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst); + +bool bitset_run_container_ixor(bitset_container_t *src_1, + const run_container_t *src_2, container_t **dst); + +/* dst does not indicate a valid container initially. Eventually it + * can become any kind of container. + */ + +int array_run_container_ixor(array_container_t *src_1, + const run_container_t *src_2, container_t **dst); + +int run_array_container_ixor(run_container_t *src_1, + const array_container_t *src_2, container_t **dst); + +bool array_array_container_ixor(array_container_t *src_1, + const array_container_t *src_2, + container_t **dst); + +int run_run_container_ixor(run_container_t *src_1, const run_container_t *src_2, + container_t **dst); + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#endif diff --git a/contrib/libs/croaring/include/roaring/containers/perfparameters.h b/contrib/libs/croaring/include/roaring/containers/perfparameters.h new file mode 100644 index 00000000000..f42a8741095 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/containers/perfparameters.h @@ -0,0 +1,49 @@ +#ifndef PERFPARAMETERS_H_ +#define PERFPARAMETERS_H_ + +#include <stdbool.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/** +During lazy computations, we can transform array containers into bitset +containers as +long as we can expect them to have ARRAY_LAZY_LOWERBOUND values. +*/ +enum { ARRAY_LAZY_LOWERBOUND = 1024 }; + +/* default initial size of a run container + setting it to zero delays the malloc.*/ +enum { RUN_DEFAULT_INIT_SIZE = 0 }; + +/* default initial size of an array container + setting it to zero delays the malloc */ +enum { ARRAY_DEFAULT_INIT_SIZE = 0 }; + +/* automatic bitset conversion during lazy or */ +#ifndef LAZY_OR_BITSET_CONVERSION +#define LAZY_OR_BITSET_CONVERSION true +#endif + +/* automatically attempt to convert a bitset to a full run during lazy + * evaluation */ +#ifndef LAZY_OR_BITSET_CONVERSION_TO_FULL +#define LAZY_OR_BITSET_CONVERSION_TO_FULL true +#endif + +/* automatically attempt to convert a bitset to a full run */ +#ifndef OR_BITSET_CONVERSION_TO_FULL +#define OR_BITSET_CONVERSION_TO_FULL true +#endif + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#endif diff --git a/contrib/libs/croaring/include/roaring/containers/run.h b/contrib/libs/croaring/include/roaring/containers/run.h new file mode 100644 index 00000000000..1c7e07b2d5a --- /dev/null +++ b/contrib/libs/croaring/include/roaring/containers/run.h @@ -0,0 +1,718 @@ +/* + * run.h + * + */ + +#ifndef INCLUDE_CONTAINERS_RUN_H_ +#define INCLUDE_CONTAINERS_RUN_H_ + +#include <roaring/roaring_types.h> // roaring_iterator + +// Include other headers after roaring_types.h +#include <assert.h> +#include <stdbool.h> +#include <stdint.h> +#include <string.h> + +#include <roaring/array_util.h> // binarySearch()/memequals() for inlining +#include <roaring/containers/container_defs.h> // container_t, perfparameters +#include <roaring/portability.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { + +// Note: in pure C++ code, you should avoid putting `using` in header files +using api::roaring_iterator; +using api::roaring_iterator64; + +namespace internal { +#endif + +/* struct rle16_s - run length pair + * + * @value: start position of the run + * @length: length of the run is `length + 1` + * + * An RLE pair {v, l} would represent the integers between the interval + * [v, v+l+1], e.g. {3, 2} = [3, 4, 5]. + */ +struct rle16_s { + uint16_t value; + uint16_t length; +}; + +typedef struct rle16_s rle16_t; + +#ifdef __cplusplus +#define MAKE_RLE16(val, len) \ + { (uint16_t)(val), (uint16_t)(len) } // no tagged structs until c++20 +#else +#define MAKE_RLE16(val, len) \ + (rle16_t) { .value = (uint16_t)(val), .length = (uint16_t)(len) } +#endif + +/* struct run_container_s - run container bitmap + * + * @n_runs: number of rle_t pairs in `runs`. + * @capacity: capacity in rle_t pairs `runs` can hold. + * @runs: pairs of rle_t. + */ +STRUCT_CONTAINER(run_container_s) { + int32_t n_runs; + int32_t capacity; + rle16_t *runs; +}; + +typedef struct run_container_s run_container_t; + +#define CAST_run(c) CAST(run_container_t *, c) // safer downcast +#define const_CAST_run(c) CAST(const run_container_t *, c) +#define movable_CAST_run(c) movable_CAST(run_container_t **, c) + +/* Create a new run container. Return NULL in case of failure. */ +run_container_t *run_container_create(void); + +/* Create a new run container with given capacity. Return NULL in case of + * failure. */ +run_container_t *run_container_create_given_capacity(int32_t size); + +/* + * Shrink the capacity to the actual size, return the number of bytes saved. + */ +int run_container_shrink_to_fit(run_container_t *src); + +/* Free memory owned by `run'. */ +void run_container_free(run_container_t *run); + +/* Duplicate container */ +run_container_t *run_container_clone(const run_container_t *src); + +/* + * Effectively deletes the value at index index, repacking data. + */ +static inline void recoverRoomAtIndex(run_container_t *run, uint16_t index) { + memmove(run->runs + index, run->runs + (1 + index), + (run->n_runs - index - 1) * sizeof(rle16_t)); + run->n_runs--; +} + +/** + * Good old binary search through rle data + */ +inline int32_t interleavedBinarySearch(const rle16_t *array, int32_t lenarray, + uint16_t ikey) { + int32_t low = 0; + int32_t high = lenarray - 1; + while (low <= high) { + int32_t middleIndex = (low + high) >> 1; + uint16_t middleValue = array[middleIndex].value; + if (middleValue < ikey) { + low = middleIndex + 1; + } else if (middleValue > ikey) { + high = middleIndex - 1; + } else { + return middleIndex; + } + } + return -(low + 1); +} + +/* + * Returns index of the run which contains $ikey + */ +static inline int32_t rle16_find_run(const rle16_t *array, int32_t lenarray, + uint16_t ikey) { + int32_t low = 0; + int32_t high = lenarray - 1; + while (low <= high) { + int32_t middleIndex = (low + high) >> 1; + uint16_t min = array[middleIndex].value; + uint16_t max = array[middleIndex].value + array[middleIndex].length; + if (ikey > max) { + low = middleIndex + 1; + } else if (ikey < min) { + high = middleIndex - 1; + } else { + return middleIndex; + } + } + return -(low + 1); +} + +/** + * Returns number of runs which can'be be merged with the key because they + * are less than the key. + * Note that [5,6,7,8] can be merged with the key 9 and won't be counted. + */ +static inline int32_t rle16_count_less(const rle16_t *array, int32_t lenarray, + uint16_t key) { + if (lenarray == 0) return 0; + int32_t low = 0; + int32_t high = lenarray - 1; + while (low <= high) { + int32_t middleIndex = (low + high) >> 1; + uint16_t min_value = array[middleIndex].value; + uint16_t max_value = + array[middleIndex].value + array[middleIndex].length; + if (max_value + UINT32_C(1) < key) { // uint32 arithmetic + low = middleIndex + 1; + } else if (key < min_value) { + high = middleIndex - 1; + } else { + return middleIndex; + } + } + return low; +} + +static inline int32_t rle16_count_greater(const rle16_t *array, + int32_t lenarray, uint16_t key) { + if (lenarray == 0) return 0; + int32_t low = 0; + int32_t high = lenarray - 1; + while (low <= high) { + int32_t middleIndex = (low + high) >> 1; + uint16_t min_value = array[middleIndex].value; + uint16_t max_value = + array[middleIndex].value + array[middleIndex].length; + if (max_value < key) { + low = middleIndex + 1; + } else if (key + UINT32_C(1) < min_value) { // uint32 arithmetic + high = middleIndex - 1; + } else { + return lenarray - (middleIndex + 1); + } + } + return lenarray - low; +} + +/** + * increase capacity to at least min. Whether the + * existing data needs to be copied over depends on copy. If "copy" is false, + * then the new content will be uninitialized, otherwise a copy is made. + */ +void run_container_grow(run_container_t *run, int32_t min, bool copy); + +/** + * Moves the data so that we can write data at index + */ +static inline void makeRoomAtIndex(run_container_t *run, uint16_t index) { + /* This function calls realloc + memmove sequentially to move by one index. + * Potentially copying twice the array. + */ + if (run->n_runs + 1 > run->capacity) + run_container_grow(run, run->n_runs + 1, true); + memmove(run->runs + 1 + index, run->runs + index, + (run->n_runs - index) * sizeof(rle16_t)); + run->n_runs++; +} + +/* Add `pos' to `run'. Returns true if `pos' was not present. */ +bool run_container_add(run_container_t *run, uint16_t pos); + +/* Remove `pos' from `run'. Returns true if `pos' was present. */ +static inline bool run_container_remove(run_container_t *run, uint16_t pos) { + int32_t index = interleavedBinarySearch(run->runs, run->n_runs, pos); + if (index >= 0) { + int32_t le = run->runs[index].length; + if (le == 0) { + recoverRoomAtIndex(run, (uint16_t)index); + } else { + run->runs[index].value++; + run->runs[index].length--; + } + return true; + } + index = -index - 2; // points to preceding value, possibly -1 + if (index >= 0) { // possible match + int32_t offset = pos - run->runs[index].value; + int32_t le = run->runs[index].length; + if (offset < le) { + // need to break in two + run->runs[index].length = (uint16_t)(offset - 1); + // need to insert + uint16_t newvalue = pos + 1; + int32_t newlength = le - offset - 1; + makeRoomAtIndex(run, (uint16_t)(index + 1)); + run->runs[index + 1].value = newvalue; + run->runs[index + 1].length = (uint16_t)newlength; + return true; + + } else if (offset == le) { + run->runs[index].length--; + return true; + } + } + // no match + return false; +} + +/* Check whether `pos' is present in `run'. */ +inline bool run_container_contains(const run_container_t *run, uint16_t pos) { + int32_t index = interleavedBinarySearch(run->runs, run->n_runs, pos); + if (index >= 0) return true; + index = -index - 2; // points to preceding value, possibly -1 + if (index != -1) { // possible match + int32_t offset = pos - run->runs[index].value; + int32_t le = run->runs[index].length; + if (offset <= le) return true; + } + return false; +} + +/* + * Check whether all positions in a range of positions from pos_start (included) + * to pos_end (excluded) is present in `run'. + */ +static inline bool run_container_contains_range(const run_container_t *run, + uint32_t pos_start, + uint32_t pos_end) { + uint32_t count = 0; + int32_t index = + interleavedBinarySearch(run->runs, run->n_runs, (uint16_t)pos_start); + if (index < 0) { + index = -index - 2; + if ((index == -1) || + ((pos_start - run->runs[index].value) > run->runs[index].length)) { + return false; + } + } + for (int32_t i = index; i < run->n_runs; ++i) { + const uint32_t stop = run->runs[i].value + run->runs[i].length; + if (run->runs[i].value >= pos_end) break; + if (stop >= pos_end) { + count += (((pos_end - run->runs[i].value) > 0) + ? (pos_end - run->runs[i].value) + : 0); + break; + } + const uint32_t min = (stop - pos_start) > 0 ? (stop - pos_start) : 0; + count += (min < run->runs[i].length) ? min : run->runs[i].length; + } + return count >= (pos_end - pos_start - 1); +} + +/* Get the cardinality of `run'. Requires an actual computation. */ +int run_container_cardinality(const run_container_t *run); + +/* Card > 0?, see run_container_empty for the reverse */ +static inline bool run_container_nonzero_cardinality( + const run_container_t *run) { + return run->n_runs > 0; // runs never empty +} + +/* Card == 0?, see run_container_nonzero_cardinality for the reverse */ +static inline bool run_container_empty(const run_container_t *run) { + return run->n_runs == 0; // runs never empty +} + +/* Copy one container into another. We assume that they are distinct. */ +void run_container_copy(const run_container_t *src, run_container_t *dst); + +/** + * Append run described by vl to the run container, possibly merging. + * It is assumed that the run would be inserted at the end of the container, no + * check is made. + * It is assumed that the run container has the necessary capacity: caller is + * responsible for checking memory capacity. + * + * + * This is not a safe function, it is meant for performance: use with care. + */ +static inline void run_container_append(run_container_t *run, rle16_t vl, + rle16_t *previousrl) { + const uint32_t previousend = previousrl->value + previousrl->length; + if (vl.value > previousend + 1) { // we add a new one + run->runs[run->n_runs] = vl; + run->n_runs++; + *previousrl = vl; + } else { + uint32_t newend = vl.value + vl.length + UINT32_C(1); + if (newend > previousend) { // we merge + previousrl->length = (uint16_t)(newend - 1 - previousrl->value); + run->runs[run->n_runs - 1] = *previousrl; + } + } +} + +/** + * Like run_container_append but it is assumed that the content of run is empty. + */ +static inline rle16_t run_container_append_first(run_container_t *run, + rle16_t vl) { + run->runs[run->n_runs] = vl; + run->n_runs++; + return vl; +} + +/** + * append a single value given by val to the run container, possibly merging. + * It is assumed that the value would be inserted at the end of the container, + * no check is made. + * It is assumed that the run container has the necessary capacity: caller is + * responsible for checking memory capacity. + * + * This is not a safe function, it is meant for performance: use with care. + */ +static inline void run_container_append_value(run_container_t *run, + uint16_t val, + rle16_t *previousrl) { + const uint32_t previousend = previousrl->value + previousrl->length; + if (val > previousend + 1) { // we add a new one + *previousrl = MAKE_RLE16(val, 0); + run->runs[run->n_runs] = *previousrl; + run->n_runs++; + } else if (val == previousend + 1) { // we merge + previousrl->length++; + run->runs[run->n_runs - 1] = *previousrl; + } +} + +/** + * Like run_container_append_value but it is assumed that the content of run is + * empty. + */ +static inline rle16_t run_container_append_value_first(run_container_t *run, + uint16_t val) { + rle16_t newrle = MAKE_RLE16(val, 0); + run->runs[run->n_runs] = newrle; + run->n_runs++; + return newrle; +} + +/* Check whether the container spans the whole chunk (cardinality = 1<<16). + * This check can be done in constant time (inexpensive). */ +static inline bool run_container_is_full(const run_container_t *run) { + rle16_t vl = run->runs[0]; + return (run->n_runs == 1) && (vl.value == 0) && (vl.length == 0xFFFF); +} + +/* Compute the union of `src_1' and `src_2' and write the result to `dst' + * It is assumed that `dst' is distinct from both `src_1' and `src_2'. */ +void run_container_union(const run_container_t *src_1, + const run_container_t *src_2, run_container_t *dst); + +/* Compute the union of `src_1' and `src_2' and write the result to `src_1' */ +void run_container_union_inplace(run_container_t *src_1, + const run_container_t *src_2); + +/* Compute the intersection of src_1 and src_2 and write the result to + * dst. It is assumed that dst is distinct from both src_1 and src_2. */ +void run_container_intersection(const run_container_t *src_1, + const run_container_t *src_2, + run_container_t *dst); + +/* Compute the size of the intersection of src_1 and src_2 . */ +int run_container_intersection_cardinality(const run_container_t *src_1, + const run_container_t *src_2); + +/* Check whether src_1 and src_2 intersect. */ +bool run_container_intersect(const run_container_t *src_1, + const run_container_t *src_2); + +/* Compute the symmetric difference of `src_1' and `src_2' and write the result + * to `dst' + * It is assumed that `dst' is distinct from both `src_1' and `src_2'. */ +void run_container_xor(const run_container_t *src_1, + const run_container_t *src_2, run_container_t *dst); + +/* + * Write out the 16-bit integers contained in this container as a list of 32-bit + * integers using base + * as the starting value (it might be expected that base has zeros in its 16 + * least significant bits). + * The function returns the number of values written. + * The caller is responsible for allocating enough memory in out. + */ +int run_container_to_uint32_array(void *vout, const run_container_t *cont, + uint32_t base); + +/* + * Print this container using printf (useful for debugging). + */ +void run_container_printf(const run_container_t *v); + +/* + * Print this container using printf as a comma-separated list of 32-bit + * integers starting at base. + */ +void run_container_printf_as_uint32_array(const run_container_t *v, + uint32_t base); + +bool run_container_validate(const run_container_t *run, const char **reason); + +/** + * Return the serialized size in bytes of a container having "num_runs" runs. + */ +static inline int32_t run_container_serialized_size_in_bytes(int32_t num_runs) { + return sizeof(uint16_t) + + sizeof(rle16_t) * num_runs; // each run requires 2 2-byte entries. +} + +bool run_container_iterate(const run_container_t *cont, uint32_t base, + roaring_iterator iterator, void *ptr); +bool run_container_iterate64(const run_container_t *cont, uint32_t base, + roaring_iterator64 iterator, uint64_t high_bits, + void *ptr); + +/** + * Writes the underlying array to buf, outputs how many bytes were written. + * This is meant to be byte-by-byte compatible with the Java and Go versions of + * Roaring. + * The number of bytes written should be run_container_size_in_bytes(container). + */ +int32_t run_container_write(const run_container_t *container, char *buf); + +/** + * Reads the instance from buf, outputs how many bytes were read. + * This is meant to be byte-by-byte compatible with the Java and Go versions of + * Roaring. + * The number of bytes read should be bitset_container_size_in_bytes(container). + * The cardinality parameter is provided for consistency with other containers, + * but + * it might be effectively ignored.. + */ +int32_t run_container_read(int32_t cardinality, run_container_t *container, + const char *buf); + +/** + * Return the serialized size in bytes of a container (see run_container_write). + * This is meant to be compatible with the Java and Go versions of Roaring. + */ +ALLOW_UNALIGNED +static inline int32_t run_container_size_in_bytes( + const run_container_t *container) { + return run_container_serialized_size_in_bytes(container->n_runs); +} + +/** + * Return true if the two containers have the same content. + */ +ALLOW_UNALIGNED +static inline bool run_container_equals(const run_container_t *container1, + const run_container_t *container2) { + if (container1->n_runs != container2->n_runs) { + return false; + } + return memequals(container1->runs, container2->runs, + container1->n_runs * sizeof(rle16_t)); +} + +/** + * Return true if container1 is a subset of container2. + */ +bool run_container_is_subset(const run_container_t *container1, + const run_container_t *container2); + +/** + * Used in a start-finish scan that appends segments, for XOR and NOT + */ + +void run_container_smart_append_exclusive(run_container_t *src, + const uint16_t start, + const uint16_t length); + +/** + * The new container consists of a single run [start,stop). + * It is required that stop>start, the caller is responsability for this check. + * It is required that stop <= (1<<16), the caller is responsability for this + * check. The cardinality of the created container is stop - start. Returns NULL + * on failure + */ +static inline run_container_t *run_container_create_range(uint32_t start, + uint32_t stop) { + run_container_t *rc = run_container_create_given_capacity(1); + if (rc) { + rle16_t r; + r.value = (uint16_t)start; + r.length = (uint16_t)(stop - start - 1); + run_container_append_first(rc, r); + } + return rc; +} + +/** + * If the element of given rank is in this container, supposing that the first + * element has rank start_rank, then the function returns true and sets element + * accordingly. + * Otherwise, it returns false and update start_rank. + */ +bool run_container_select(const run_container_t *container, + uint32_t *start_rank, uint32_t rank, + uint32_t *element); + +/* Compute the difference of src_1 and src_2 and write the result to + * dst. It is assumed that dst is distinct from both src_1 and src_2. */ + +void run_container_andnot(const run_container_t *src_1, + const run_container_t *src_2, run_container_t *dst); + +void run_container_offset(const run_container_t *c, container_t **loc, + container_t **hic, uint16_t offset); + +/* Returns the smallest value (assumes not empty) */ +inline uint16_t run_container_minimum(const run_container_t *run) { + if (run->n_runs == 0) return 0; + return run->runs[0].value; +} + +/* Returns the largest value (assumes not empty) */ +inline uint16_t run_container_maximum(const run_container_t *run) { + if (run->n_runs == 0) return 0; + return run->runs[run->n_runs - 1].value + run->runs[run->n_runs - 1].length; +} + +/* Returns the number of values equal or smaller than x */ +int run_container_rank(const run_container_t *arr, uint16_t x); + +/* bulk version of run_container_rank(); return number of consumed elements */ +uint32_t run_container_rank_many(const run_container_t *arr, + uint64_t start_rank, const uint32_t *begin, + const uint32_t *end, uint64_t *ans); + +/* Returns the index of x, if not exsist return -1 */ +int run_container_get_index(const run_container_t *arr, uint16_t x); + +/* Returns the index of the first run containing a value at least as large as x, + * or -1 */ +inline int run_container_index_equalorlarger(const run_container_t *arr, + uint16_t x) { + int32_t index = interleavedBinarySearch(arr->runs, arr->n_runs, x); + if (index >= 0) return index; + index = -index - 2; // points to preceding run, possibly -1 + if (index != -1) { // possible match + int32_t offset = x - arr->runs[index].value; + int32_t le = arr->runs[index].length; + if (offset <= le) return index; + } + index += 1; + if (index < arr->n_runs) { + return index; + } + return -1; +} + +/* + * Add all values in range [min, max] using hint. + */ +static inline void run_container_add_range_nruns(run_container_t *run, + uint32_t min, uint32_t max, + int32_t nruns_less, + int32_t nruns_greater) { + int32_t nruns_common = run->n_runs - nruns_less - nruns_greater; + if (nruns_common == 0) { + makeRoomAtIndex(run, (uint16_t)nruns_less); + run->runs[nruns_less].value = (uint16_t)min; + run->runs[nruns_less].length = (uint16_t)(max - min); + } else { + uint32_t common_min = run->runs[nruns_less].value; + uint32_t common_max = run->runs[nruns_less + nruns_common - 1].value + + run->runs[nruns_less + nruns_common - 1].length; + uint32_t result_min = (common_min < min) ? common_min : min; + uint32_t result_max = (common_max > max) ? common_max : max; + + run->runs[nruns_less].value = (uint16_t)result_min; + run->runs[nruns_less].length = (uint16_t)(result_max - result_min); + + memmove(&(run->runs[nruns_less + 1]), + &(run->runs[run->n_runs - nruns_greater]), + nruns_greater * sizeof(rle16_t)); + run->n_runs = nruns_less + 1 + nruns_greater; + } +} + +/** + * Add all values in range [min, max]. This function is currently unused + * and left as documentation. + */ +/*static inline void run_container_add_range(run_container_t* run, + uint32_t min, uint32_t max) { + int32_t nruns_greater = rle16_count_greater(run->runs, run->n_runs, max); + int32_t nruns_less = rle16_count_less(run->runs, run->n_runs - +nruns_greater, min); run_container_add_range_nruns(run, min, max, nruns_less, +nruns_greater); +}*/ + +/** + * Shifts last $count elements either left (distance < 0) or right (distance > + * 0) + */ +static inline void run_container_shift_tail(run_container_t *run, int32_t count, + int32_t distance) { + if (distance > 0) { + if (run->capacity < count + distance) { + run_container_grow(run, count + distance, true); + } + } + int32_t srcpos = run->n_runs - count; + int32_t dstpos = srcpos + distance; + memmove(&(run->runs[dstpos]), &(run->runs[srcpos]), + sizeof(rle16_t) * count); + run->n_runs += distance; +} + +/** + * Remove all elements in range [min, max] + */ +static inline void run_container_remove_range(run_container_t *run, + uint32_t min, uint32_t max) { + int32_t first = rle16_find_run(run->runs, run->n_runs, (uint16_t)min); + int32_t last = rle16_find_run(run->runs, run->n_runs, (uint16_t)max); + + if (first >= 0 && min > run->runs[first].value && + max < ((uint32_t)run->runs[first].value + + (uint32_t)run->runs[first].length)) { + // split this run into two adjacent runs + + // right subinterval + makeRoomAtIndex(run, (uint16_t)(first + 1)); + run->runs[first + 1].value = (uint16_t)(max + 1); + run->runs[first + 1].length = + (uint16_t)((run->runs[first].value + run->runs[first].length) - + (max + 1)); + + // left subinterval + run->runs[first].length = + (uint16_t)((min - 1) - run->runs[first].value); + + return; + } + + // update left-most partial run + if (first >= 0) { + if (min > run->runs[first].value) { + run->runs[first].length = + (uint16_t)((min - 1) - run->runs[first].value); + first++; + } + } else { + first = -first - 1; + } + + // update right-most run + if (last >= 0) { + uint16_t run_max = run->runs[last].value + run->runs[last].length; + if (run_max > max) { + run->runs[last].value = (uint16_t)(max + 1); + run->runs[last].length = (uint16_t)(run_max - (max + 1)); + last--; + } + } else { + last = (-last - 1) - 1; + } + + // remove intermediate runs + if (first <= last) { + run_container_shift_tail(run, run->n_runs - (last + 1), + -(last - first + 1)); + } +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#endif /* INCLUDE_CONTAINERS_RUN_H_ */ diff --git a/contrib/libs/croaring/include/roaring/isadetection.h b/contrib/libs/croaring/include/roaring/isadetection.h new file mode 100644 index 00000000000..189db5f00a9 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/isadetection.h @@ -0,0 +1,42 @@ +#ifndef ROARING_ISADETECTION_H +#define ROARING_ISADETECTION_H +#if defined(__x86_64__) || defined(_M_AMD64) // x64 + +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#ifdef __has_include +// We want to make sure that the AVX-512 functions are only built on compilers +// fully supporting AVX-512. +#if __has_include(<avx512vbmi2intrin.h>) +#define CROARING_COMPILER_SUPPORTS_AVX512 1 +#endif // #if __has_include(<avx512vbmi2intrin.h>) +#endif // #ifdef __has_include + +// Visual Studio 2019 and up support AVX-512 +#ifdef _MSC_VER +#if _MSC_VER >= 1920 +#define CROARING_COMPILER_SUPPORTS_AVX512 1 +#endif // #if _MSC_VER >= 1920 +#endif // #ifdef _MSC_VER + +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#define CROARING_COMPILER_SUPPORTS_AVX512 0 +#endif // #ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#endif // #ifndef CROARING_COMPILER_SUPPORTS_AVX512 + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif +enum { + ROARING_SUPPORTS_AVX2 = 1, + ROARING_SUPPORTS_AVX512 = 2, +}; +int croaring_hardware_support(void); +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif +#endif // x64 +#endif // ROARING_ISADETECTION_H diff --git a/contrib/libs/croaring/include/roaring/memory.h b/contrib/libs/croaring/include/roaring/memory.h new file mode 100644 index 00000000000..ad9a64f5e22 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/memory.h @@ -0,0 +1,39 @@ +#ifndef INCLUDE_ROARING_MEMORY_H_ +#define INCLUDE_ROARING_MEMORY_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <stddef.h> // for size_t + +typedef void* (*roaring_malloc_p)(size_t); +typedef void* (*roaring_realloc_p)(void*, size_t); +typedef void* (*roaring_calloc_p)(size_t, size_t); +typedef void (*roaring_free_p)(void*); +typedef void* (*roaring_aligned_malloc_p)(size_t, size_t); +typedef void (*roaring_aligned_free_p)(void*); + +typedef struct roaring_memory_s { + roaring_malloc_p malloc; + roaring_realloc_p realloc; + roaring_calloc_p calloc; + roaring_free_p free; + roaring_aligned_malloc_p aligned_malloc; + roaring_aligned_free_p aligned_free; +} roaring_memory_t; + +void roaring_init_memory_hook(roaring_memory_t memory_hook); + +void* roaring_malloc(size_t); +void* roaring_realloc(void*, size_t); +void* roaring_calloc(size_t, size_t); +void roaring_free(void*); +void* roaring_aligned_malloc(size_t, size_t); +void roaring_aligned_free(void*); + +#ifdef __cplusplus +} +#endif + +#endif // INCLUDE_ROARING_MEMORY_H_ diff --git a/contrib/libs/croaring/include/roaring/portability.h b/contrib/libs/croaring/include/roaring/portability.h new file mode 100644 index 00000000000..9962aaf4192 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/portability.h @@ -0,0 +1,593 @@ +/* + * portability.h + * + */ + +/** + * All macros should be prefixed with either CROARING or ROARING. + * The library uses both ROARING_... + * as well as CROAIRING_ as prefixes. The ROARING_ prefix is for + * macros that are provided by the build system or that are closely + * related to the format. The header macros may also use ROARING_. + * The CROARING_ prefix is for internal macros that a user is unlikely + * to ever interact with. + */ + +#ifndef INCLUDE_PORTABILITY_H_ +#define INCLUDE_PORTABILITY_H_ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE 1 +#endif // _GNU_SOURCE +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS 1 +#endif // __STDC_FORMAT_MACROS + +#ifdef _MSC_VER +#define CROARING_VISUAL_STUDIO 1 +/** + * We want to differentiate carefully between + * clang under visual studio and regular visual + * studio. + */ +#ifdef __clang__ +// clang under visual studio +#define CROARING_CLANG_VISUAL_STUDIO 1 +#else +// just regular visual studio (best guess) +#define CROARING_REGULAR_VISUAL_STUDIO 1 +#endif // __clang__ +#endif // _MSC_VER +#ifndef CROARING_VISUAL_STUDIO +#define CROARING_VISUAL_STUDIO 0 +#endif +#ifndef CROARING_CLANG_VISUAL_STUDIO +#define CROARING_CLANG_VISUAL_STUDIO 0 +#endif +#ifndef CROARING_REGULAR_VISUAL_STUDIO +#define CROARING_REGULAR_VISUAL_STUDIO 0 +#endif + +#if defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE < 200809L) +#undef _POSIX_C_SOURCE +#endif + +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 200809L +#endif // !(defined(_POSIX_C_SOURCE)) || (_POSIX_C_SOURCE < 200809L) +#if !(defined(_XOPEN_SOURCE)) || (_XOPEN_SOURCE < 700) +#define _XOPEN_SOURCE 700 +#endif // !(defined(_XOPEN_SOURCE)) || (_XOPEN_SOURCE < 700) + +#ifdef __illumos__ +#define __EXTENSIONS__ +#endif + +#include <stdbool.h> +#include <stdint.h> +#include <stdlib.h> // will provide posix_memalign with _POSIX_C_SOURCE as defined above +#ifdef __GLIBC__ +#include <malloc.h> // this should never be needed but there are some reports that it is needed. +#endif + +#ifdef __cplusplus +extern "C" { // portability definitions are in global scope, not a namespace +#endif + +#if defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ != 8 +#error This code assumes 64-bit long longs (by use of the GCC intrinsics). Your system is not currently supported. +#endif + +#if CROARING_REGULAR_VISUAL_STUDIO +#ifndef __restrict__ +#define __restrict__ __restrict +#endif // __restrict__ +#endif // CROARING_REGULAR_VISUAL_STUDIO + +#if defined(__x86_64__) || defined(_M_X64) +// we have an x64 processor +#define CROARING_IS_X64 1 + +#if defined(_MSC_VER) && (_MSC_VER < 1910) +// Old visual studio systems won't support AVX2 well. +#undef CROARING_IS_X64 +#endif + +#if defined(__clang_major__) && (__clang_major__ <= 8) && !defined(__AVX2__) +// Older versions of clang have a bug affecting us +// https://stackoverflow.com/questions/57228537/how-does-one-use-pragma-clang-attribute-push-with-c-namespaces +#undef CROARING_IS_X64 +#endif + +#ifdef ROARING_DISABLE_X64 +#undef CROARING_IS_X64 +#endif +// we include the intrinsic header +#if !CROARING_REGULAR_VISUAL_STUDIO +/* Non-Microsoft C/C++-compatible compiler */ +#include <x86intrin.h> // on some recent GCC, this will declare posix_memalign + +#if CROARING_CLANG_VISUAL_STUDIO + +/** + * You are not supposed, normally, to include these + * headers directly. Instead you should either include intrin.h + * or x86intrin.h. However, when compiling with clang + * under Windows (i.e., when _MSC_VER is set), these headers + * only get included *if* the corresponding features are detected + * from macros: + * e.g., if __AVX2__ is set... in turn, we normally set these + * macros by compiling against the corresponding architecture + * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole + * software with these advanced instructions. These headers would + * normally guard against such usage, but we carefully included + * <x86intrin.h> (or <intrin.h>) before, so the headers + * are fooled. + */ +// To avoid reordering imports: +// clang-format off +#include <bmiintrin.h> // for _blsr_u64 +#include <lzcntintrin.h> // for __lzcnt64 +#include <immintrin.h> // for most things (AVX2, AVX512, _popcnt64) +#include <smmintrin.h> +#include <tmmintrin.h> +#include <avxintrin.h> +#include <avx2intrin.h> +#include <wmmintrin.h> +#if _MSC_VER >= 1920 +// Important: we need the AVX-512 headers: +#include <avx512fintrin.h> +#include <avx512dqintrin.h> +#include <avx512cdintrin.h> +#include <avx512bwintrin.h> +#include <avx512vlintrin.h> +#include <avx512vbmiintrin.h> +#include <avx512vbmi2intrin.h> +#include <avx512vpopcntdqintrin.h> +// clang-format on +#endif // _MSC_VER >= 1920 +// unfortunately, we may not get _blsr_u64, but, thankfully, clang +// has it as a macro. +#ifndef _blsr_u64 +// we roll our own +#define _blsr_u64(n) ((n - 1) & n) +#endif // _blsr_u64 +#endif // SIMDJSON_CLANG_VISUAL_STUDIO + +#endif // CROARING_REGULAR_VISUAL_STUDIO +#endif // defined(__x86_64__) || defined(_M_X64) + +#if !defined(CROARING_USENEON) && !defined(DISABLENEON) && defined(__ARM_NEON) +#define CROARING_USENEON +#endif +#if defined(CROARING_USENEON) +#include <arm_neon.h> +#endif + +#if !CROARING_REGULAR_VISUAL_STUDIO +/* Non-Microsoft C/C++-compatible compiler, assumes that it supports inline + * assembly */ +#define CROARING_INLINE_ASM 1 +#endif // _MSC_VER + +#if CROARING_REGULAR_VISUAL_STUDIO +/* Microsoft C/C++-compatible compiler */ +#include <intrin.h> + +#ifndef __clang__ // if one compiles with MSVC *with* clang, then these + // intrinsics are defined!!! +#define CROARING_INTRINSICS 1 +// sadly there is no way to check whether we are missing these intrinsics +// specifically. + +/* wrappers for Visual Studio built-ins that look like gcc built-ins + * __builtin_ctzll */ +/** result might be undefined when input_num is zero */ +inline int roaring_trailing_zeroes(unsigned long long input_num) { + unsigned long index; +#ifdef _WIN64 // highly recommended!!! + _BitScanForward64(&index, input_num); +#else // if we must support 32-bit Windows + if ((uint32_t)input_num != 0) { + _BitScanForward(&index, (uint32_t)input_num); + } else { + _BitScanForward(&index, (uint32_t)(input_num >> 32)); + index += 32; + } +#endif // _WIN64 + return index; +} + +/* wrappers for Visual Studio built-ins that look like gcc built-ins + * __builtin_clzll */ +/** result might be undefined when input_num is zero */ +inline int roaring_leading_zeroes(unsigned long long input_num) { + unsigned long index; +#ifdef _WIN64 // highly recommended!!! + _BitScanReverse64(&index, input_num); +#else // if we must support 32-bit Windows + if (input_num > 0xFFFFFFFF) { + _BitScanReverse(&index, (uint32_t)(input_num >> 32)); + index += 32; + } else { + _BitScanReverse(&index, (uint32_t)(input_num)); + } +#endif // _WIN64 + return 63 - index; +} + +/* Use #define so this is effective even under /Ob0 (no inline) */ +#define roaring_unreachable __assume(0) +#endif // __clang__ + +#endif // CROARING_REGULAR_VISUAL_STUDIO + +#ifndef CROARING_INTRINSICS +#define CROARING_INTRINSICS 1 +#define roaring_unreachable __builtin_unreachable() +/** result might be undefined when input_num is zero */ +inline int roaring_trailing_zeroes(unsigned long long input_num) { + return __builtin_ctzll(input_num); +} +/** result might be undefined when input_num is zero */ +inline int roaring_leading_zeroes(unsigned long long input_num) { + return __builtin_clzll(input_num); +} +#endif + +#if CROARING_REGULAR_VISUAL_STUDIO +#define ALIGNED(x) __declspec(align(x)) +#elif defined(__GNUC__) || defined(__clang__) +#define ALIGNED(x) __attribute__((aligned(x))) +#else +#warning "Warning. Unrecognized compiler." +#define ALIGNED(x) +#endif + +#if defined(__GNUC__) || defined(__clang__) +#define CROARING_WARN_UNUSED __attribute__((warn_unused_result)) +#else +#define CROARING_WARN_UNUSED +#endif + +#define IS_BIG_ENDIAN (*(uint16_t *)"\0\xff" < 0x100) + +#ifdef CROARING_USENEON +// we can always compute the popcount fast. +#elif (defined(_M_ARM) || defined(_M_ARM64)) && \ + ((defined(_WIN64) || defined(_WIN32)) && \ + defined(CROARING_REGULAR_VISUAL_STUDIO) && \ + CROARING_REGULAR_VISUAL_STUDIO) +// we will need this function: +static inline int roaring_hamming_backup(uint64_t x) { + uint64_t c1 = UINT64_C(0x5555555555555555); + uint64_t c2 = UINT64_C(0x3333333333333333); + uint64_t c4 = UINT64_C(0x0F0F0F0F0F0F0F0F); + x -= (x >> 1) & c1; + x = ((x >> 2) & c2) + (x & c2); + x = (x + (x >> 4)) & c4; + x *= UINT64_C(0x0101010101010101); + return x >> 56; +} +#endif + +static inline int roaring_hamming(uint64_t x) { +#if defined(_WIN64) && defined(CROARING_REGULAR_VISUAL_STUDIO) && \ + CROARING_REGULAR_VISUAL_STUDIO +#ifdef CROARING_USENEON + return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); +#elif defined(_M_ARM64) + return roaring_hamming_backup(x); + // (int) _CountOneBits64(x); is unavailable +#else // _M_ARM64 + return (int)__popcnt64(x); +#endif // _M_ARM64 +#elif defined(_WIN32) && defined(CROARING_REGULAR_VISUAL_STUDIO) && \ + CROARING_REGULAR_VISUAL_STUDIO +#ifdef _M_ARM + return roaring_hamming_backup(x); + // _CountOneBits is unavailable +#else // _M_ARM + return (int)__popcnt((unsigned int)x) + + (int)__popcnt((unsigned int)(x >> 32)); +#endif // _M_ARM +#else + return __builtin_popcountll(x); +#endif +} + +#ifndef UINT64_C +#define UINT64_C(c) (c##ULL) +#endif // UINT64_C + +#ifndef UINT32_C +#define UINT32_C(c) (c##UL) +#endif // UINT32_C + +#ifdef __cplusplus +} // extern "C" { +#endif // __cplusplus + +// this is almost standard? +#undef STRINGIFY_IMPLEMENTATION_ +#undef STRINGIFY +#define STRINGIFY_IMPLEMENTATION_(a) #a +#define STRINGIFY(a) STRINGIFY_IMPLEMENTATION_(a) + +// Our fast kernels require 64-bit systems. +// +// On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions. +// Furthermore, the number of SIMD registers is reduced. +// +// On 32-bit ARM, we would have smaller registers. +// +// The library should still have the fallback kernel. It is +// slower, but it should run everywhere. + +// +// Enable valid runtime implementations, and select +// CROARING_BUILTIN_IMPLEMENTATION +// + +// We are going to use runtime dispatch. +#if CROARING_IS_X64 +#ifdef __clang__ +// clang does not have GCC push pop +// warning: clang attribute push can't be used within a namespace in clang up +// til 8.0 so CROARING_TARGET_REGION and CROARING_UNTARGET_REGION must be +// *outside* of a namespace. +#define CROARING_TARGET_REGION(T) \ + _Pragma(STRINGIFY(clang attribute push(__attribute__((target(T))), \ + apply_to = function))) +#define CROARING_UNTARGET_REGION _Pragma("clang attribute pop") +#elif defined(__GNUC__) +// GCC is easier +#define CROARING_TARGET_REGION(T) \ + _Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T))) +#define CROARING_UNTARGET_REGION _Pragma("GCC pop_options") +#endif // clang then gcc + +#endif // CROARING_IS_X64 + +// Default target region macros don't do anything. +#ifndef CROARING_TARGET_REGION +#define CROARING_TARGET_REGION(T) +#define CROARING_UNTARGET_REGION +#endif + +#define CROARING_TARGET_AVX2 \ + CROARING_TARGET_REGION("avx2,bmi,pclmul,lzcnt,popcnt") +#define CROARING_TARGET_AVX512 \ + CROARING_TARGET_REGION( \ + "avx2,bmi,bmi2,pclmul,lzcnt,popcnt,avx512f,avx512dq,avx512bw," \ + "avx512vbmi2,avx512bitalg,avx512vpopcntdq") +#define CROARING_UNTARGET_AVX2 CROARING_UNTARGET_REGION +#define CROARING_UNTARGET_AVX512 CROARING_UNTARGET_REGION + +#ifdef __AVX2__ +// No need for runtime dispatching. +// It is unnecessary and harmful to old clang to tag regions. +#undef CROARING_TARGET_AVX2 +#define CROARING_TARGET_AVX2 +#undef CROARING_UNTARGET_AVX2 +#define CROARING_UNTARGET_AVX2 +#endif + +#if defined(__AVX512F__) && defined(__AVX512DQ__) && defined(__AVX512BW__) && \ + defined(__AVX512VBMI2__) && defined(__AVX512BITALG__) && \ + defined(__AVX512VPOPCNTDQ__) +// No need for runtime dispatching. +// It is unnecessary and harmful to old clang to tag regions. +#undef CROARING_TARGET_AVX512 +#define CROARING_TARGET_AVX512 +#undef CROARING_UNTARGET_AVX512 +#define CROARING_UNTARGET_AVX512 +#endif + +// Allow unaligned memory access +#if defined(__GNUC__) || defined(__clang__) +#define ALLOW_UNALIGNED __attribute__((no_sanitize("alignment"))) +#else +#define ALLOW_UNALIGNED +#endif + +#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) +#define CROARING_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#elif defined(_WIN32) +#define CROARING_IS_BIG_ENDIAN 0 +#else +#if defined(__APPLE__) || \ + defined(__FreeBSD__) // defined __BYTE_ORDER__ && defined + // __ORDER_BIG_ENDIAN__ +#include <machine/endian.h> +#elif defined(sun) || \ + defined(__sun) // defined(__APPLE__) || defined(__FreeBSD__) +#error #include <sys/byteorder.h> +#else // defined(__APPLE__) || defined(__FreeBSD__) + +#ifdef __has_include +#if __has_include(<endian.h>) +#include <endian.h> +#endif //__has_include(<endian.h>) +#endif //__has_include + +#endif // defined(__APPLE__) || defined(__FreeBSD__) + +#ifndef !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) +#define CROARING_IS_BIG_ENDIAN 0 +#endif + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define CROARING_IS_BIG_ENDIAN 0 +#else // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define CROARING_IS_BIG_ENDIAN 1 +#endif // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#endif + +// Host <-> big endian conversion. +#if CROARING_IS_BIG_ENDIAN +#define croaring_htobe64(x) (x) + +#elif defined(_WIN32) || defined(_WIN64) // CROARING_IS_BIG_ENDIAN +#include <stdlib.h> +#define croaring_htobe64(x) _byteswap_uint64(x) + +#elif defined(__APPLE__) // CROARING_IS_BIG_ENDIAN +#include <libkern/OSByteOrder.h> +#define croaring_htobe64(x) OSSwapInt64(x) + +#elif defined(__has_include) && \ + __has_include(<byteswap.h>) // CROARING_IS_BIG_ENDIAN +#include <byteswap.h> +#define croaring_htobe64(x) __bswap_64(x) + +#else // CROARING_IS_BIG_ENDIAN +// Gets compiled to bswap or equivalent on most compilers. +#define croaring_htobe64(x) \ + (((x & 0x00000000000000FFULL) << 56) | \ + ((x & 0x000000000000FF00ULL) << 40) | \ + ((x & 0x0000000000FF0000ULL) << 24) | \ + ((x & 0x00000000FF000000ULL) << 8) | ((x & 0x000000FF00000000ULL) >> 8) | \ + ((x & 0x0000FF0000000000ULL) >> 24) | \ + ((x & 0x00FF000000000000ULL) >> 40) | \ + ((x & 0xFF00000000000000ULL) >> 56)) +#endif // CROARING_IS_BIG_ENDIAN +#define croaring_be64toh(x) croaring_htobe64(x) +// End of host <-> big endian conversion. + +// Defines for the possible CROARING atomic implementations +#define CROARING_ATOMIC_IMPL_NONE 1 +#define CROARING_ATOMIC_IMPL_CPP 2 +#define CROARING_ATOMIC_IMPL_C 3 +#define CROARING_ATOMIC_IMPL_C_WINDOWS 4 + +// If the use has forced a specific implementation, use that, otherwise, +// figure out the best implementation we can use. +#if !defined(CROARING_ATOMIC_IMPL) +#if defined(__cplusplus) && __cplusplus >= 201103L +#ifdef __has_include +#if __has_include(<atomic>) +#define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_CPP +#endif //__has_include(<atomic>) +#else + // We lack __has_include to check: +#define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_CPP +#endif //__has_include +#elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) +#define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_C +#elif CROARING_REGULAR_VISUAL_STUDIO + // https://www.technetworkhub.com/c11-atomics-in-visual-studio-2022-version-17/ +#define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_C_WINDOWS +#endif +#endif // !defined(CROARING_ATOMIC_IMPL) + +#if CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_C +#include <stdatomic.h> +typedef _Atomic(uint32_t) croaring_refcount_t; + +static inline void croaring_refcount_inc(croaring_refcount_t *val) { + // Increasing the reference counter can always be done with + // memory_order_relaxed: New references to an object can only be formed from + // an existing reference, and passing an existing reference from one thread + // to another must already provide any required synchronization. + atomic_fetch_add_explicit(val, 1, memory_order_relaxed); +} + +static inline bool croaring_refcount_dec(croaring_refcount_t *val) { + // It is important to enforce any possible access to the object in one + // thread (through an existing reference) to happen before deleting the + // object in a different thread. This is achieved by a "release" operation + // after dropping a reference (any access to the object through this + // reference must obviously happened before), and an "acquire" operation + // before deleting the object. + bool is_zero = atomic_fetch_sub_explicit(val, 1, memory_order_release) == 1; + if (is_zero) { + atomic_thread_fence(memory_order_acquire); + } + return is_zero; +} + +static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { + return atomic_load_explicit(val, memory_order_relaxed); +} +#elif CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_CPP +#include <atomic> +typedef std::atomic<uint32_t> croaring_refcount_t; + +static inline void croaring_refcount_inc(croaring_refcount_t *val) { + val->fetch_add(1, std::memory_order_relaxed); +} + +static inline bool croaring_refcount_dec(croaring_refcount_t *val) { + // See above comments on the c11 atomic implementation for memory ordering + bool is_zero = val->fetch_sub(1, std::memory_order_release) == 1; + if (is_zero) { + std::atomic_thread_fence(std::memory_order_acquire); + } + return is_zero; +} + +static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { + return val->load(std::memory_order_relaxed); +} +#elif CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_C_WINDOWS +#include <intrin.h> +#pragma intrinsic(_InterlockedIncrement) +#pragma intrinsic(_InterlockedDecrement) + +// _InterlockedIncrement and _InterlockedDecrement take a (signed) long, and +// overflow is defined to wrap, so we can pretend it is a uint32_t for our case +typedef volatile long croaring_refcount_t; + +static inline void croaring_refcount_inc(croaring_refcount_t *val) { + _InterlockedIncrement(val); +} + +static inline bool croaring_refcount_dec(croaring_refcount_t *val) { + return _InterlockedDecrement(val) == 0; +} + +static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { + // Per + // https://learn.microsoft.com/en-us/windows/win32/sync/interlocked-variable-access + // > Simple reads and writes to properly-aligned 32-bit variables are atomic + // > operations. In other words, you will not end up with only one portion + // > of the variable updated; all bits are updated in an atomic fashion. + return *val; +} +#elif CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_NONE +#include <assert.h> +typedef uint32_t croaring_refcount_t; + +static inline void croaring_refcount_inc(croaring_refcount_t *val) { + *val += 1; +} + +static inline bool croaring_refcount_dec(croaring_refcount_t *val) { + assert(*val > 0); + *val -= 1; + return val == 0; +} + +static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { + return *val; +} +#else +#error "Unknown atomic implementation" +#endif + +#if defined(__GNUC__) || defined(__clang__) +#define CROARING_DEPRECATED __attribute__((deprecated)) +#else +#define CROARING_DEPRECATED +#endif // defined(__GNUC__) || defined(__clang__) + +// We need portability.h to be included first, +// but we also always want isadetection.h to be +// included (right after). +// See https://github.com/RoaringBitmap/CRoaring/issues/394 +// There is no scenario where we want portability.h to +// be included, but not isadetection.h: the latter is a +// strict requirement. +#include <roaring/isadetection.h> // include it last! +#endif /* INCLUDE_PORTABILITY_H_ */ diff --git a/contrib/libs/croaring/include/roaring/roaring.h b/contrib/libs/croaring/include/roaring/roaring.h new file mode 100644 index 00000000000..295bc3cb33a --- /dev/null +++ b/contrib/libs/croaring/include/roaring/roaring.h @@ -0,0 +1,1132 @@ +/* + * An implementation of Roaring Bitmaps in C. + */ + +#ifndef ROARING_H +#define ROARING_H + +#include <stdbool.h> +#include <stddef.h> // for `size_t` +#include <stdint.h> + +#include <roaring/roaring_types.h> + +// Include other headers after roaring_types.h +#include <roaring/bitset/bitset.h> +#include <roaring/memory.h> +#include <roaring/portability.h> +#include <roaring/roaring_version.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace api { +#endif + +typedef struct roaring_bitmap_s { + roaring_array_t high_low_container; +} roaring_bitmap_t; + +/** + * Dynamically allocates a new bitmap (initially empty). + * Returns NULL if the allocation fails. + * Capacity is a performance hint for how many "containers" the data will need. + * Client is responsible for calling `roaring_bitmap_free()`. + */ +roaring_bitmap_t *roaring_bitmap_create_with_capacity(uint32_t cap); + +/** + * Dynamically allocates a new bitmap (initially empty). + * Returns NULL if the allocation fails. + * Client is responsible for calling `roaring_bitmap_free()`. + */ +inline roaring_bitmap_t *roaring_bitmap_create(void) { + return roaring_bitmap_create_with_capacity(0); +} + +/** + * Initialize a roaring bitmap structure in memory controlled by client. + * Capacity is a performance hint for how many "containers" the data will need. + * Can return false if auxiliary allocations fail when capacity greater than 0. + */ +bool roaring_bitmap_init_with_capacity(roaring_bitmap_t *r, uint32_t cap); + +/** + * Initialize a roaring bitmap structure in memory controlled by client. + * The bitmap will be in a "clear" state, with no auxiliary allocations. + * Since this performs no allocations, the function will not fail. + */ +inline void roaring_bitmap_init_cleared(roaring_bitmap_t *r) { + roaring_bitmap_init_with_capacity(r, 0); +} + +/** + * Add all the values between min (included) and max (excluded) that are at a + * distance k*step from min. + */ +roaring_bitmap_t *roaring_bitmap_from_range(uint64_t min, uint64_t max, + uint32_t step); + +/** + * Creates a new bitmap from a pointer of uint32_t integers + */ +roaring_bitmap_t *roaring_bitmap_of_ptr(size_t n_args, const uint32_t *vals); + +/* + * Whether you want to use copy-on-write. + * Saves memory and avoids copies, but needs more care in a threaded context. + * Most users should ignore this flag. + * + * Note: If you do turn this flag to 'true', enabling COW, then ensure that you + * do so for all of your bitmaps, since interactions between bitmaps with and + * without COW is unsafe. + */ +inline bool roaring_bitmap_get_copy_on_write(const roaring_bitmap_t *r) { + return r->high_low_container.flags & ROARING_FLAG_COW; +} +inline void roaring_bitmap_set_copy_on_write(roaring_bitmap_t *r, bool cow) { + if (cow) { + r->high_low_container.flags |= ROARING_FLAG_COW; + } else { + r->high_low_container.flags &= ~ROARING_FLAG_COW; + } +} + +roaring_bitmap_t *roaring_bitmap_add_offset(const roaring_bitmap_t *bm, + int64_t offset); +/** + * Describe the inner structure of the bitmap. + */ +void roaring_bitmap_printf_describe(const roaring_bitmap_t *r); + +/** + * Creates a new bitmap from a list of uint32_t integers + * + * This function is deprecated, use `roaring_bitmap_from` instead, which + * doesn't require the number of elements to be passed in. + * + * @see roaring_bitmap_from + */ +CROARING_DEPRECATED roaring_bitmap_t *roaring_bitmap_of(size_t n, ...); + +#ifdef __cplusplus +/** + * Creates a new bitmap which contains all values passed in as arguments. + * + * To create a bitmap from a variable number of arguments, use the + * `roaring_bitmap_of_ptr` function instead. + */ +// Use an immediately invoked closure, capturing by reference +// (in case __VA_ARGS__ refers to context outside the closure) +// Include a 0 at the beginning of the array to make the array length > 0 +// (zero sized arrays are not valid in standard c/c++) +#define roaring_bitmap_from(...) \ + [&]() { \ + const uint32_t roaring_bitmap_from_array[] = {0, __VA_ARGS__}; \ + return roaring_bitmap_of_ptr((sizeof(roaring_bitmap_from_array) / \ + sizeof(roaring_bitmap_from_array[0])) - \ + 1, \ + &roaring_bitmap_from_array[1]); \ + }() +#else +/** + * Creates a new bitmap which contains all values passed in as arguments. + * + * To create a bitmap from a variable number of arguments, use the + * `roaring_bitmap_of_ptr` function instead. + */ +// While __VA_ARGS__ occurs twice in expansion, one of the times is in a sizeof +// expression, which is an unevaluated context, so it's even safe in the case +// where expressions passed have side effects (roaring64_bitmap_from(my_func(), +// ++i)) +// Include a 0 at the beginning of the array to make the array length > 0 +// (zero sized arrays are not valid in standard c/c++) +#define roaring_bitmap_from(...) \ + roaring_bitmap_of_ptr( \ + (sizeof((const uint32_t[]){0, __VA_ARGS__}) / sizeof(uint32_t)) - 1, \ + &((const uint32_t[]){0, __VA_ARGS__})[1]) +#endif + +/** + * Copies a bitmap (this does memory allocation). + * The caller is responsible for memory management. + */ +roaring_bitmap_t *roaring_bitmap_copy(const roaring_bitmap_t *r); + +/** + * Copies a bitmap from src to dest. It is assumed that the pointer dest + * is to an already allocated bitmap. The content of the dest bitmap is + * freed/deleted. + * + * It might be preferable and simpler to call roaring_bitmap_copy except + * that roaring_bitmap_overwrite can save on memory allocations. + * + * Returns true if successful, or false if there was an error. On failure, + * the dest bitmap is left in a valid, empty state (even if it was not empty + * before). + */ +bool roaring_bitmap_overwrite(roaring_bitmap_t *dest, + const roaring_bitmap_t *src); + +/** + * Print the content of the bitmap. + */ +void roaring_bitmap_printf(const roaring_bitmap_t *r); + +/** + * Computes the intersection between two bitmaps and returns new bitmap. The + * caller is responsible for memory management. + * + * Performance hint: if you are computing the intersection between several + * bitmaps, two-by-two, it is best to start with the smallest bitmap. + * You may also rely on roaring_bitmap_and_inplace to avoid creating + * many temporary bitmaps. + */ +roaring_bitmap_t *roaring_bitmap_and(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Computes the size of the intersection between two bitmaps. + */ +uint64_t roaring_bitmap_and_cardinality(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Check whether two bitmaps intersect. + */ +bool roaring_bitmap_intersect(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Check whether a bitmap and an open range intersect. + */ +bool roaring_bitmap_intersect_with_range(const roaring_bitmap_t *bm, uint64_t x, + uint64_t y); + +/** + * Computes the Jaccard index between two bitmaps. (Also known as the Tanimoto + * distance, or the Jaccard similarity coefficient) + * + * The Jaccard index is undefined if both bitmaps are empty. + */ +double roaring_bitmap_jaccard_index(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Computes the size of the union between two bitmaps. + */ +uint64_t roaring_bitmap_or_cardinality(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Computes the size of the difference (andnot) between two bitmaps. + */ +uint64_t roaring_bitmap_andnot_cardinality(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Computes the size of the symmetric difference (xor) between two bitmaps. + */ +uint64_t roaring_bitmap_xor_cardinality(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Inplace version of `roaring_bitmap_and()`, modifies r1 + * r1 == r2 is allowed. + * + * Performance hint: if you are computing the intersection between several + * bitmaps, two-by-two, it is best to start with the smallest bitmap. + */ +void roaring_bitmap_and_inplace(roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Computes the union between two bitmaps and returns new bitmap. The caller is + * responsible for memory management. + */ +roaring_bitmap_t *roaring_bitmap_or(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Inplace version of `roaring_bitmap_or(), modifies r1. + * TODO: decide whether r1 == r2 ok + */ +void roaring_bitmap_or_inplace(roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Compute the union of 'number' bitmaps. + * Caller is responsible for freeing the result. + * See also `roaring_bitmap_or_many_heap()` + */ +roaring_bitmap_t *roaring_bitmap_or_many(size_t number, + const roaring_bitmap_t **rs); + +/** + * Compute the union of 'number' bitmaps using a heap. This can sometimes be + * faster than `roaring_bitmap_or_many() which uses a naive algorithm. + * Caller is responsible for freeing the result. + */ +roaring_bitmap_t *roaring_bitmap_or_many_heap(uint32_t number, + const roaring_bitmap_t **rs); + +/** + * Computes the symmetric difference (xor) between two bitmaps + * and returns new bitmap. The caller is responsible for memory management. + */ +roaring_bitmap_t *roaring_bitmap_xor(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Inplace version of roaring_bitmap_xor, modifies r1, r1 != r2. + */ +void roaring_bitmap_xor_inplace(roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Compute the xor of 'number' bitmaps. + * Caller is responsible for freeing the result. + */ +roaring_bitmap_t *roaring_bitmap_xor_many(size_t number, + const roaring_bitmap_t **rs); + +/** + * Computes the difference (andnot) between two bitmaps and returns new bitmap. + * Caller is responsible for freeing the result. + */ +roaring_bitmap_t *roaring_bitmap_andnot(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Inplace version of roaring_bitmap_andnot, modifies r1, r1 != r2. + */ +void roaring_bitmap_andnot_inplace(roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * TODO: consider implementing: + * + * "Compute the xor of 'number' bitmaps using a heap. This can sometimes be + * faster than roaring_bitmap_xor_many which uses a naive algorithm. Caller is + * responsible for freeing the result."" + * + * roaring_bitmap_t *roaring_bitmap_xor_many_heap(uint32_t number, + * const roaring_bitmap_t **rs); + */ + +/** + * Frees the memory. + */ +void roaring_bitmap_free(const roaring_bitmap_t *r); + +/** + * A bit of context usable with `roaring_bitmap_*_bulk()` functions + * + * Should be initialized with `{0}` (or `memset()` to all zeros). + * Callers should treat it as an opaque type. + * + * A context may only be used with a single bitmap + * (unless re-initialized to zero), and any modification to a bitmap + * (other than modifications performed with `_bulk()` functions with the context + * passed) will invalidate any contexts associated with that bitmap. + */ +typedef struct roaring_bulk_context_s { + ROARING_CONTAINER_T *container; + int idx; + uint16_t key; + uint8_t typecode; +} roaring_bulk_context_t; + +/** + * Add an item, using context from a previous insert for speed optimization. + * + * `context` will be used to store information between calls to make bulk + * operations faster. `*context` should be zero-initialized before the first + * call to this function. + * + * Modifying the bitmap in any way (other than `-bulk` suffixed functions) + * will invalidate the stored context, calling this function with a non-zero + * context after doing any modification invokes undefined behavior. + * + * In order to exploit this optimization, the caller should call this function + * with values with the same "key" (high 16 bits of the value) consecutively. + */ +void roaring_bitmap_add_bulk(roaring_bitmap_t *r, + roaring_bulk_context_t *context, uint32_t val); + +/** + * Add value n_args from pointer vals, faster than repeatedly calling + * `roaring_bitmap_add()` + * + * In order to exploit this optimization, the caller should attempt to keep + * values with the same "key" (high 16 bits of the value) as consecutive + * elements in `vals` + */ +void roaring_bitmap_add_many(roaring_bitmap_t *r, size_t n_args, + const uint32_t *vals); + +/** + * Add value x + */ +void roaring_bitmap_add(roaring_bitmap_t *r, uint32_t x); + +/** + * Add value x + * Returns true if a new value was added, false if the value already existed. + */ +bool roaring_bitmap_add_checked(roaring_bitmap_t *r, uint32_t x); + +/** + * Add all values in range [min, max] + */ +void roaring_bitmap_add_range_closed(roaring_bitmap_t *r, uint32_t min, + uint32_t max); + +/** + * Add all values in range [min, max) + */ +inline void roaring_bitmap_add_range(roaring_bitmap_t *r, uint64_t min, + uint64_t max) { + if (max <= min) return; + roaring_bitmap_add_range_closed(r, (uint32_t)min, (uint32_t)(max - 1)); +} + +/** + * Remove value x + */ +void roaring_bitmap_remove(roaring_bitmap_t *r, uint32_t x); + +/** + * Remove all values in range [min, max] + */ +void roaring_bitmap_remove_range_closed(roaring_bitmap_t *r, uint32_t min, + uint32_t max); + +/** + * Remove all values in range [min, max) + */ +inline void roaring_bitmap_remove_range(roaring_bitmap_t *r, uint64_t min, + uint64_t max) { + if (max <= min) return; + roaring_bitmap_remove_range_closed(r, (uint32_t)min, (uint32_t)(max - 1)); +} + +/** + * Remove multiple values + */ +void roaring_bitmap_remove_many(roaring_bitmap_t *r, size_t n_args, + const uint32_t *vals); + +/** + * Remove value x + * Returns true if a new value was removed, false if the value was not existing. + */ +bool roaring_bitmap_remove_checked(roaring_bitmap_t *r, uint32_t x); + +/** + * Check if value is present + */ +bool roaring_bitmap_contains(const roaring_bitmap_t *r, uint32_t val); + +/** + * Check whether a range of values from range_start (included) + * to range_end (excluded) is present + */ +bool roaring_bitmap_contains_range(const roaring_bitmap_t *r, + uint64_t range_start, uint64_t range_end); + +/** + * Check if an items is present, using context from a previous insert or search + * for speed optimization. + * + * `context` will be used to store information between calls to make bulk + * operations faster. `*context` should be zero-initialized before the first + * call to this function. + * + * Modifying the bitmap in any way (other than `-bulk` suffixed functions) + * will invalidate the stored context, calling this function with a non-zero + * context after doing any modification invokes undefined behavior. + * + * In order to exploit this optimization, the caller should call this function + * with values with the same "key" (high 16 bits of the value) consecutively. + */ +bool roaring_bitmap_contains_bulk(const roaring_bitmap_t *r, + roaring_bulk_context_t *context, + uint32_t val); + +/** + * Get the cardinality of the bitmap (number of elements). + */ +uint64_t roaring_bitmap_get_cardinality(const roaring_bitmap_t *r); + +/** + * Returns the number of elements in the range [range_start, range_end). + */ +uint64_t roaring_bitmap_range_cardinality(const roaring_bitmap_t *r, + uint64_t range_start, + uint64_t range_end); + +/** + * Returns true if the bitmap is empty (cardinality is zero). + */ +bool roaring_bitmap_is_empty(const roaring_bitmap_t *r); + +/** + * Empties the bitmap. It will have no auxiliary allocations (so if the bitmap + * was initialized in client memory via roaring_bitmap_init(), then a call to + * roaring_bitmap_clear() would be enough to "free" it) + */ +void roaring_bitmap_clear(roaring_bitmap_t *r); + +/** + * Convert the bitmap to a sorted array, output in `ans`. + * + * Caller is responsible to ensure that there is enough memory allocated, e.g. + * + * ans = malloc(roaring_bitmap_get_cardinality(bitmap) * sizeof(uint32_t)); + */ +void roaring_bitmap_to_uint32_array(const roaring_bitmap_t *r, uint32_t *ans); + +/** + * Store the bitmap to a bitset. This can be useful for people + * who need the performance and simplicity of a standard bitset. + * We assume that the input bitset is originally empty (does not + * have any set bit). + * + * bitset_t * out = bitset_create(); + * // if the bitset has content in it, call "bitset_clear(out)" + * bool success = roaring_bitmap_to_bitset(mybitmap, out); + * // on failure, success will be false. + * // You can then query the bitset: + * bool is_present = bitset_get(out, 10011 ); + * // you must free the memory: + * bitset_free(out); + * + */ +bool roaring_bitmap_to_bitset(const roaring_bitmap_t *r, bitset_t *bitset); + +/** + * Convert the bitmap to a sorted array from `offset` by `limit`, output in + * `ans`. + * + * Caller is responsible to ensure that there is enough memory allocated, e.g. + * + * ans = malloc(roaring_bitmap_get_cardinality(limit) * sizeof(uint32_t)); + * + * Return false in case of failure (e.g., insufficient memory) + */ +bool roaring_bitmap_range_uint32_array(const roaring_bitmap_t *r, size_t offset, + size_t limit, uint32_t *ans); + +/** + * Remove run-length encoding even when it is more space efficient. + * Return whether a change was applied. + */ +bool roaring_bitmap_remove_run_compression(roaring_bitmap_t *r); + +/** + * Convert array and bitmap containers to run containers when it is more + * efficient; also convert from run containers when more space efficient. + * + * Returns true if the result has at least one run container. + * Additional savings might be possible by calling `shrinkToFit()`. + */ +bool roaring_bitmap_run_optimize(roaring_bitmap_t *r); + +/** + * If needed, reallocate memory to shrink the memory usage. + * Returns the number of bytes saved. + */ +size_t roaring_bitmap_shrink_to_fit(roaring_bitmap_t *r); + +/** + * Write the bitmap to an output pointer, this output buffer should refer to + * at least `roaring_bitmap_size_in_bytes(r)` allocated bytes. + * + * See `roaring_bitmap_portable_serialize()` if you want a format that's + * compatible with Java and Go implementations. This format can sometimes be + * more space efficient than the portable form, e.g. when the data is sparse. + * + * Returns how many bytes written, should be `roaring_bitmap_size_in_bytes(r)`. + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. + */ +size_t roaring_bitmap_serialize(const roaring_bitmap_t *r, char *buf); + +/** + * Use with `roaring_bitmap_serialize()`. + * + * (See `roaring_bitmap_portable_deserialize()` if you want a format that's + * compatible with Java and Go implementations). + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. + */ +roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf); + +/** + * Use with `roaring_bitmap_serialize()`. + * + * (See `roaring_bitmap_portable_deserialize_safe()` if you want a format that's + * compatible with Java and Go implementations). + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. + * + * The difference with `roaring_bitmap_deserialize()` is that this function + * checks that the input buffer is a valid bitmap. If the buffer is too small, + * NULL is returned. + */ +roaring_bitmap_t *roaring_bitmap_deserialize_safe(const void *buf, + size_t maxbytes); + +/** + * How many bytes are required to serialize this bitmap (NOT compatible + * with Java and Go versions) + */ +size_t roaring_bitmap_size_in_bytes(const roaring_bitmap_t *r); + +/** + * Read bitmap from a serialized buffer. + * In case of failure, NULL is returned. + * + * This function is unsafe in the sense that if there is no valid serialized + * bitmap at the pointer, then many bytes could be read, possibly causing a + * buffer overflow. See also roaring_bitmap_portable_deserialize_safe(). + * + * This is meant to be compatible with the Java and Go versions: + * https://github.com/RoaringBitmap/RoaringFormatSpec + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. + */ +roaring_bitmap_t *roaring_bitmap_portable_deserialize(const char *buf); + +/** + * Read bitmap from a serialized buffer safely (reading up to maxbytes). + * In case of failure, NULL is returned. + * + * This is meant to be compatible with the Java and Go versions: + * https://github.com/RoaringBitmap/RoaringFormatSpec + * + * The function itself is safe in the sense that it will not cause buffer + * overflows. However, for correct operations, it is assumed that the bitmap + * read was once serialized from a valid bitmap (i.e., it follows the format + * specification). If you provided an incorrect input (garbage), then the bitmap + * read may not be in a valid state and following operations may not lead to + * sensible results. In particular, the serialized array containers need to be + * in sorted order, and the run containers should be in sorted non-overlapping + * order. This is is guaranteed to happen when serializing an existing bitmap, + * but not for random inputs. + * + * You may use roaring_bitmap_internal_validate to check the validity of the + * bitmap prior to using it. You may also use other strategies to check for + * corrupted inputs (e.g., checksums). + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. + */ +roaring_bitmap_t *roaring_bitmap_portable_deserialize_safe(const char *buf, + size_t maxbytes); + +/** + * Read bitmap from a serialized buffer. + * In case of failure, NULL is returned. + * + * Bitmap returned by this function can be used in all readonly contexts. + * Bitmap must be freed as usual, by calling roaring_bitmap_free(). + * Underlying buffer must not be freed or modified while it backs any bitmaps. + * + * The function is unsafe in the following ways: + * 1) It may execute unaligned memory accesses. + * 2) A buffer overflow may occur if buf does not point to a valid serialized + * bitmap. + * + * This is meant to be compatible with the Java and Go versions: + * https://github.com/RoaringBitmap/RoaringFormatSpec + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. + */ +roaring_bitmap_t *roaring_bitmap_portable_deserialize_frozen(const char *buf); + +/** + * Check how many bytes would be read (up to maxbytes) at this pointer if there + * is a bitmap, returns zero if there is no valid bitmap. + * + * This is meant to be compatible with the Java and Go versions: + * https://github.com/RoaringBitmap/RoaringFormatSpec + */ +size_t roaring_bitmap_portable_deserialize_size(const char *buf, + size_t maxbytes); + +/** + * How many bytes are required to serialize this bitmap. + * + * This is meant to be compatible with the Java and Go versions: + * https://github.com/RoaringBitmap/RoaringFormatSpec + */ +size_t roaring_bitmap_portable_size_in_bytes(const roaring_bitmap_t *r); + +/** + * Write a bitmap to a char buffer. The output buffer should refer to at least + * `roaring_bitmap_portable_size_in_bytes(r)` bytes of allocated memory. + * + * Returns how many bytes were written which should match + * `roaring_bitmap_portable_size_in_bytes(r)`. + * + * This is meant to be compatible with the Java and Go versions: + * https://github.com/RoaringBitmap/RoaringFormatSpec + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. + */ +size_t roaring_bitmap_portable_serialize(const roaring_bitmap_t *r, char *buf); + +/* + * "Frozen" serialization format imitates memory layout of roaring_bitmap_t. + * Deserialized bitmap is a constant view of the underlying buffer. + * This significantly reduces amount of allocations and copying required during + * deserialization. + * It can be used with memory mapped files. + * Example can be found in benchmarks/frozen_benchmark.c + * + * [#####] const roaring_bitmap_t * + * | | | + * +----+ | +-+ + * | | | + * [#####################################] underlying buffer + * + * Note that because frozen serialization format imitates C memory layout + * of roaring_bitmap_t, it is not fixed. It is different on big/little endian + * platforms and can be changed in future. + */ + +/** + * Returns number of bytes required to serialize bitmap using frozen format. + */ +size_t roaring_bitmap_frozen_size_in_bytes(const roaring_bitmap_t *r); + +/** + * Serializes bitmap using frozen format. + * Buffer size must be at least roaring_bitmap_frozen_size_in_bytes(). + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. + */ +void roaring_bitmap_frozen_serialize(const roaring_bitmap_t *r, char *buf); + +/** + * Creates constant bitmap that is a view of a given buffer. + * Buffer data should have been written by `roaring_bitmap_frozen_serialize()` + * Its beginning must also be aligned by 32 bytes. + * Length must be equal exactly to `roaring_bitmap_frozen_size_in_bytes()`. + * In case of failure, NULL is returned. + * + * Bitmap returned by this function can be used in all readonly contexts. + * Bitmap must be freed as usual, by calling roaring_bitmap_free(). + * Underlying buffer must not be freed or modified while it backs any bitmaps. + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. + */ +const roaring_bitmap_t *roaring_bitmap_frozen_view(const char *buf, + size_t length); + +/** + * Iterate over the bitmap elements. The function iterator is called once for + * all the values with ptr (can be NULL) as the second parameter of each call. + * + * `roaring_iterator` is simply a pointer to a function that returns bool + * (true means that the iteration should continue while false means that it + * should stop), and takes (uint32_t,void*) as inputs. + * + * Returns true if the roaring_iterator returned true throughout (so that all + * data points were necessarily visited). + * + * Iteration is ordered: from the smallest to the largest elements. + */ +bool roaring_iterate(const roaring_bitmap_t *r, roaring_iterator iterator, + void *ptr); + +bool roaring_iterate64(const roaring_bitmap_t *r, roaring_iterator64 iterator, + uint64_t high_bits, void *ptr); + +/** + * Return true if the two bitmaps contain the same elements. + */ +bool roaring_bitmap_equals(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Return true if all the elements of r1 are also in r2. + */ +bool roaring_bitmap_is_subset(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Return true if all the elements of r1 are also in r2, and r2 is strictly + * greater than r1. + */ +bool roaring_bitmap_is_strict_subset(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * (For expert users who seek high performance.) + * + * Computes the union between two bitmaps and returns new bitmap. The caller is + * responsible for memory management. + * + * The lazy version defers some computations such as the maintenance of the + * cardinality counts. Thus you must call `roaring_bitmap_repair_after_lazy()` + * after executing "lazy" computations. + * + * It is safe to repeatedly call roaring_bitmap_lazy_or_inplace on the result. + * + * `bitsetconversion` is a flag which determines whether container-container + * operations force a bitset conversion. + */ +roaring_bitmap_t *roaring_bitmap_lazy_or(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2, + const bool bitsetconversion); + +/** + * (For expert users who seek high performance.) + * + * Inplace version of roaring_bitmap_lazy_or, modifies r1. + * + * `bitsetconversion` is a flag which determines whether container-container + * operations force a bitset conversion. + */ +void roaring_bitmap_lazy_or_inplace(roaring_bitmap_t *r1, + const roaring_bitmap_t *r2, + const bool bitsetconversion); + +/** + * (For expert users who seek high performance.) + * + * Execute maintenance on a bitmap created from `roaring_bitmap_lazy_or()` + * or modified with `roaring_bitmap_lazy_or_inplace()`. + */ +void roaring_bitmap_repair_after_lazy(roaring_bitmap_t *r1); + +/** + * Computes the symmetric difference between two bitmaps and returns new bitmap. + * The caller is responsible for memory management. + * + * The lazy version defers some computations such as the maintenance of the + * cardinality counts. Thus you must call `roaring_bitmap_repair_after_lazy()` + * after executing "lazy" computations. + * + * It is safe to repeatedly call `roaring_bitmap_lazy_xor_inplace()` on + * the result. + */ +roaring_bitmap_t *roaring_bitmap_lazy_xor(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * (For expert users who seek high performance.) + * + * Inplace version of roaring_bitmap_lazy_xor, modifies r1. r1 != r2 + */ +void roaring_bitmap_lazy_xor_inplace(roaring_bitmap_t *r1, + const roaring_bitmap_t *r2); + +/** + * Compute the negation of the bitmap in the interval [range_start, range_end). + * The number of negated values is range_end - range_start. + * Areas outside the range are passed through unchanged. + */ +roaring_bitmap_t *roaring_bitmap_flip(const roaring_bitmap_t *r1, + uint64_t range_start, uint64_t range_end); + +/** + * compute (in place) the negation of the roaring bitmap within a specified + * interval: [range_start, range_end). The number of negated values is + * range_end - range_start. + * Areas outside the range are passed through unchanged. + */ +void roaring_bitmap_flip_inplace(roaring_bitmap_t *r1, uint64_t range_start, + uint64_t range_end); + +/** + * Selects the element at index 'rank' where the smallest element is at index 0. + * If the size of the roaring bitmap is strictly greater than rank, then this + * function returns true and sets element to the element of given rank. + * Otherwise, it returns false. + */ +bool roaring_bitmap_select(const roaring_bitmap_t *r, uint32_t rank, + uint32_t *element); + +/** + * roaring_bitmap_rank returns the number of integers that are smaller or equal + * to x. Thus if x is the first element, this function will return 1. If + * x is smaller than the smallest element, this function will return 0. + * + * The indexing convention differs between roaring_bitmap_select and + * roaring_bitmap_rank: roaring_bitmap_select refers to the smallest value + * as having index 0, whereas roaring_bitmap_rank returns 1 when ranking + * the smallest value. + */ +uint64_t roaring_bitmap_rank(const roaring_bitmap_t *r, uint32_t x); + +/** + * roaring_bitmap_rank_many is an `Bulk` version of `roaring_bitmap_rank` + * it puts rank value of each element in `[begin .. end)` to `ans[]` + * + * the values in `[begin .. end)` must be sorted in Ascending order; + * Caller is responsible to ensure that there is enough memory allocated, e.g. + * + * ans = malloc((end-begin) * sizeof(uint64_t)); + */ +void roaring_bitmap_rank_many(const roaring_bitmap_t *r, const uint32_t *begin, + const uint32_t *end, uint64_t *ans); + +/** + * Returns the index of x in the given roaring bitmap. + * If the roaring bitmap doesn't contain x , this function will return -1. + * The difference with rank function is that this function will return -1 when x + * is not the element of roaring bitmap, but the rank function will return a + * non-negative number. + */ +int64_t roaring_bitmap_get_index(const roaring_bitmap_t *r, uint32_t x); + +/** + * Returns the smallest value in the set, or UINT32_MAX if the set is empty. + */ +uint32_t roaring_bitmap_minimum(const roaring_bitmap_t *r); + +/** + * Returns the greatest value in the set, or 0 if the set is empty. + */ +uint32_t roaring_bitmap_maximum(const roaring_bitmap_t *r); + +/** + * (For advanced users.) + * + * Collect statistics about the bitmap, see roaring_types.h for + * a description of roaring_statistics_t + */ +void roaring_bitmap_statistics(const roaring_bitmap_t *r, + roaring_statistics_t *stat); + +/** + * Perform internal consistency checks. Returns true if the bitmap is + * consistent. It may be useful to call this after deserializing bitmaps from + * untrusted sources. If roaring_bitmap_internal_validate returns true, then the + * bitmap should be consistent and can be trusted not to cause crashes or memory + * corruption. + * + * Note that some operations intentionally leave bitmaps in an inconsistent + * state temporarily, for example, `roaring_bitmap_lazy_*` functions, until + * `roaring_bitmap_repair_after_lazy` is called. + * + * If reason is non-null, it will be set to a string describing the first + * inconsistency found if any. + */ +bool roaring_bitmap_internal_validate(const roaring_bitmap_t *r, + const char **reason); + +/********************* +* What follows is code use to iterate through values in a roaring bitmap + +roaring_bitmap_t *r =... +roaring_uint32_iterator_t i; +roaring_iterator_create(r, &i); +while(i.has_value) { + printf("value = %d\n", i.current_value); + roaring_uint32_iterator_advance(&i); +} + +Obviously, if you modify the underlying bitmap, the iterator +becomes invalid. So don't. +*/ + +/** + * A struct used to keep iterator state. Users should only access + * `current_value` and `has_value`, the rest of the type should be treated as + * opaque. + */ +typedef struct roaring_uint32_iterator_s { + const roaring_bitmap_t *parent; // Owner + const ROARING_CONTAINER_T *container; // Current container + uint8_t typecode; // Typecode of current container + int32_t container_index; // Current container index + uint32_t highbits; // High 16 bits of the current value + roaring_container_iterator_t container_it; + + uint32_t current_value; + bool has_value; +} roaring_uint32_iterator_t; + +/** + * Initialize an iterator object that can be used to iterate through the values. + * If there is a value, then this iterator points to the first value and + * `it->has_value` is true. The value is in `it->current_value`. + */ +void roaring_iterator_init(const roaring_bitmap_t *r, + roaring_uint32_iterator_t *newit); + +/** DEPRECATED, use `roaring_iterator_init`. */ +CROARING_DEPRECATED static inline void roaring_init_iterator( + const roaring_bitmap_t *r, roaring_uint32_iterator_t *newit) { + roaring_iterator_init(r, newit); +} + +/** + * Initialize an iterator object that can be used to iterate through the values. + * If there is a value, then this iterator points to the last value and + * `it->has_value` is true. The value is in `it->current_value`. + */ +void roaring_iterator_init_last(const roaring_bitmap_t *r, + roaring_uint32_iterator_t *newit); + +/** DEPRECATED, use `roaring_iterator_init_last`. */ +CROARING_DEPRECATED static inline void roaring_init_iterator_last( + const roaring_bitmap_t *r, roaring_uint32_iterator_t *newit) { + roaring_iterator_init_last(r, newit); +} + +/** + * Create an iterator object that can be used to iterate through the values. + * Caller is responsible for calling `roaring_free_iterator()`. + * + * The iterator is initialized (this function calls `roaring_iterator_init()`) + * If there is a value, then this iterator points to the first value and + * `it->has_value` is true. The value is in `it->current_value`. + */ +roaring_uint32_iterator_t *roaring_iterator_create(const roaring_bitmap_t *r); + +/** DEPRECATED, use `roaring_iterator_create`. */ +CROARING_DEPRECATED static inline roaring_uint32_iterator_t * +roaring_create_iterator(const roaring_bitmap_t *r) { + return roaring_iterator_create(r); +} + +/** + * Advance the iterator. If there is a new value, then `it->has_value` is true. + * The new value is in `it->current_value`. Values are traversed in increasing + * orders. For convenience, returns `it->has_value`. + * + * Once `it->has_value` is false, `roaring_uint32_iterator_advance` should not + * be called on the iterator again. Calling `roaring_uint32_iterator_previous` + * is allowed. + */ +bool roaring_uint32_iterator_advance(roaring_uint32_iterator_t *it); + +/** DEPRECATED, use `roaring_uint32_iterator_advance`. */ +CROARING_DEPRECATED static inline bool roaring_advance_uint32_iterator( + roaring_uint32_iterator_t *it) { + return roaring_uint32_iterator_advance(it); +} + +/** + * Decrement the iterator. If there's a new value, then `it->has_value` is true. + * The new value is in `it->current_value`. Values are traversed in decreasing + * order. For convenience, returns `it->has_value`. + * + * Once `it->has_value` is false, `roaring_uint32_iterator_previous` should not + * be called on the iterator again. Calling `roaring_uint32_iterator_advance` is + * allowed. + */ +bool roaring_uint32_iterator_previous(roaring_uint32_iterator_t *it); + +/** DEPRECATED, use `roaring_uint32_iterator_previous`. */ +CROARING_DEPRECATED static inline bool roaring_previous_uint32_iterator( + roaring_uint32_iterator_t *it) { + return roaring_uint32_iterator_previous(it); +} + +/** + * Move the iterator to the first value >= `val`. If there is a such a value, + * then `it->has_value` is true. The new value is in `it->current_value`. + * For convenience, returns `it->has_value`. + */ +bool roaring_uint32_iterator_move_equalorlarger(roaring_uint32_iterator_t *it, + uint32_t val); + +/** DEPRECATED, use `roaring_uint32_iterator_move_equalorlarger`. */ +CROARING_DEPRECATED static inline bool +roaring_move_uint32_iterator_equalorlarger(roaring_uint32_iterator_t *it, + uint32_t val) { + return roaring_uint32_iterator_move_equalorlarger(it, val); +} + +/** + * Creates a copy of an iterator. + * Caller must free it. + */ +roaring_uint32_iterator_t *roaring_uint32_iterator_copy( + const roaring_uint32_iterator_t *it); + +/** DEPRECATED, use `roaring_uint32_iterator_copy`. */ +CROARING_DEPRECATED static inline roaring_uint32_iterator_t * +roaring_copy_uint32_iterator(const roaring_uint32_iterator_t *it) { + return roaring_uint32_iterator_copy(it); +} + +/** + * Free memory following `roaring_iterator_create()` + */ +void roaring_uint32_iterator_free(roaring_uint32_iterator_t *it); + +/** DEPRECATED, use `roaring_uint32_iterator_free`. */ +CROARING_DEPRECATED static inline void roaring_free_uint32_iterator( + roaring_uint32_iterator_t *it) { + roaring_uint32_iterator_free(it); +} + +/* + * Reads next ${count} values from iterator into user-supplied ${buf}. + * Returns the number of read elements. + * This number can be smaller than ${count}, which means that iterator is + * drained. + * + * This function satisfies semantics of iteration and can be used together with + * other iterator functions. + * - first value is copied from ${it}->current_value + * - after function returns, iterator is positioned at the next element + */ +uint32_t roaring_uint32_iterator_read(roaring_uint32_iterator_t *it, + uint32_t *buf, uint32_t count); + +/** DEPRECATED, use `roaring_uint32_iterator_read`. */ +CROARING_DEPRECATED static inline uint32_t roaring_read_uint32_iterator( + roaring_uint32_iterator_t *it, uint32_t *buf, uint32_t count) { + return roaring_uint32_iterator_read(it, buf, count); +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace api { +#endif + +#endif /* ROARING_H */ + +#ifdef __cplusplus +/** + * Best practices for C++ headers is to avoid polluting global scope. + * But for C compatibility when just `roaring.h` is included building as + * C++, default to global access for the C public API. + * + * BUT when `roaring.hh` is included instead, it sets this flag. That way + * explicit namespacing must be used to get the C functions. + * + * This is outside the include guard so that if you include BOTH headers, + * the order won't matter; you still get the global definitions. + */ +#if !defined(ROARING_API_NOT_IN_GLOBAL_NAMESPACE) +using namespace ::roaring::api; +#endif +#endif diff --git a/contrib/libs/croaring/include/roaring/roaring64.h b/contrib/libs/croaring/include/roaring/roaring64.h new file mode 100644 index 00000000000..cd73ba604a0 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/roaring64.h @@ -0,0 +1,657 @@ +#ifndef ROARING64_H +#define ROARING64_H + +#include <stdbool.h> +#include <stddef.h> +#include <stdint.h> + +#include <roaring/memory.h> +#include <roaring/portability.h> +#include <roaring/roaring_types.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace api { +#endif + +typedef struct roaring64_bitmap_s roaring64_bitmap_t; +typedef struct roaring64_leaf_s roaring64_leaf_t; +typedef struct roaring64_iterator_s roaring64_iterator_t; + +/** + * A bit of context usable with `roaring64_bitmap_*_bulk()` functions. + * + * Should be initialized with `{0}` (or `memset()` to all zeros). + * Callers should treat it as an opaque type. + * + * A context may only be used with a single bitmap (unless re-initialized to + * zero), and any modification to a bitmap (other than modifications performed + * with `_bulk()` functions with the context passed) will invalidate any + * contexts associated with that bitmap. + */ +typedef struct roaring64_bulk_context_s { + uint8_t high_bytes[6]; + roaring64_leaf_t *leaf; +} roaring64_bulk_context_t; + +/** + * Dynamically allocates a new bitmap (initially empty). + * Client is responsible for calling `roaring64_bitmap_free()`. + */ +roaring64_bitmap_t *roaring64_bitmap_create(void); +void roaring64_bitmap_free(roaring64_bitmap_t *r); + +/** + * Returns a copy of a bitmap. + */ +roaring64_bitmap_t *roaring64_bitmap_copy(const roaring64_bitmap_t *r); + +/** + * Creates a new bitmap of a pointer to N 64-bit integers. + */ +roaring64_bitmap_t *roaring64_bitmap_of_ptr(size_t n_args, + const uint64_t *vals); + +#ifdef __cplusplus +/** + * Creates a new bitmap which contains all values passed in as arguments. + * + * To create a bitmap from a variable number of arguments, use the + * `roaring64_bitmap_of_ptr` function instead. + */ +// Use an immediately invoked closure, capturing by reference +// (in case __VA_ARGS__ refers to context outside the closure) +// Include a 0 at the beginning of the array to make the array length > 0 +// (zero sized arrays are not valid in standard c/c++) +#define roaring64_bitmap_from(...) \ + [&]() { \ + const uint64_t roaring64_bitmap_from_array[] = {0, __VA_ARGS__}; \ + return roaring64_bitmap_of_ptr( \ + (sizeof(roaring64_bitmap_from_array) / \ + sizeof(roaring64_bitmap_from_array[0])) - \ + 1, \ + &roaring64_bitmap_from_array[1]); \ + }() +#else +/** + * Creates a new bitmap which contains all values passed in as arguments. + * + * To create a bitmap from a variable number of arguments, use the + * `roaring64_bitmap_of_ptr` function instead. + */ +// While __VA_ARGS__ occurs twice in expansion, one of the times is in a sizeof +// expression, which is an unevaluated context, so it's even safe in the case +// where expressions passed have side effects (roaring64_bitmap_from(my_func(), +// ++i)) +// Include a 0 at the beginning of the array to make the array length > 0 +// (zero sized arrays are not valid in standard c/c++) +#define roaring64_bitmap_from(...) \ + roaring64_bitmap_of_ptr( \ + (sizeof((const uint64_t[]){0, __VA_ARGS__}) / sizeof(uint64_t)) - 1, \ + &((const uint64_t[]){0, __VA_ARGS__})[1]) +#endif + +/** + * Create a new bitmap containing all the values in [min, max) that are at a + * distance k*step from min. + */ +roaring64_bitmap_t *roaring64_bitmap_from_range(uint64_t min, uint64_t max, + uint64_t step); + +/** + * Adds the provided value to the bitmap. + */ +void roaring64_bitmap_add(roaring64_bitmap_t *r, uint64_t val); + +/** + * Adds the provided value to the bitmap. + * Returns true if a new value was added, false if the value already existed. + */ +bool roaring64_bitmap_add_checked(roaring64_bitmap_t *r, uint64_t val); + +/** + * Add an item, using context from a previous insert for faster insertion. + * + * `context` will be used to store information between calls to make bulk + * operations faster. `*context` should be zero-initialized before the first + * call to this function. + * + * Modifying the bitmap in any way (other than `-bulk` suffixed functions) + * will invalidate the stored context, calling this function with a non-zero + * context after doing any modification invokes undefined behavior. + * + * In order to exploit this optimization, the caller should call this function + * with values with the same high 48 bits of the value consecutively. + */ +void roaring64_bitmap_add_bulk(roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, uint64_t val); + +/** + * Add `n_args` values from `vals`, faster than repeatedly calling + * `roaring64_bitmap_add()` + * + * In order to exploit this optimization, the caller should attempt to keep + * values with the same high 48 bits of the value as consecutive elements in + * `vals`. + */ +void roaring64_bitmap_add_many(roaring64_bitmap_t *r, size_t n_args, + const uint64_t *vals); + +/** + * Add all values in range [min, max). + */ +void roaring64_bitmap_add_range(roaring64_bitmap_t *r, uint64_t min, + uint64_t max); + +/** + * Add all values in range [min, max]. + */ +void roaring64_bitmap_add_range_closed(roaring64_bitmap_t *r, uint64_t min, + uint64_t max); + +/** + * Removes a value from the bitmap if present. + */ +void roaring64_bitmap_remove(roaring64_bitmap_t *r, uint64_t val); + +/** + * Removes a value from the bitmap if present, returns true if the value was + * removed and false if the value was not present. + */ +bool roaring64_bitmap_remove_checked(roaring64_bitmap_t *r, uint64_t val); + +/** + * Remove an item, using context from a previous insert for faster removal. + * + * `context` will be used to store information between calls to make bulk + * operations faster. `*context` should be zero-initialized before the first + * call to this function. + * + * Modifying the bitmap in any way (other than `-bulk` suffixed functions) + * will invalidate the stored context, calling this function with a non-zero + * context after doing any modification invokes undefined behavior. + * + * In order to exploit this optimization, the caller should call this function + * with values with the same high 48 bits of the value consecutively. + */ +void roaring64_bitmap_remove_bulk(roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, + uint64_t val); + +/** + * Remove `n_args` values from `vals`, faster than repeatedly calling + * `roaring64_bitmap_remove()` + * + * In order to exploit this optimization, the caller should attempt to keep + * values with the same high 48 bits of the value as consecutive elements in + * `vals`. + */ +void roaring64_bitmap_remove_many(roaring64_bitmap_t *r, size_t n_args, + const uint64_t *vals); + +/** + * Remove all values in range [min, max). + */ +void roaring64_bitmap_remove_range(roaring64_bitmap_t *r, uint64_t min, + uint64_t max); + +/** + * Remove all values in range [min, max]. + */ +void roaring64_bitmap_remove_range_closed(roaring64_bitmap_t *r, uint64_t min, + uint64_t max); + +/** + * Returns true if the provided value is present. + */ +bool roaring64_bitmap_contains(const roaring64_bitmap_t *r, uint64_t val); + +/** + * Returns true if all values in the range [min, max) are present. + */ +bool roaring64_bitmap_contains_range(const roaring64_bitmap_t *r, uint64_t min, + uint64_t max); + +/** + * Check if an item is present using context from a previous insert or search + * for faster search. + * + * `context` will be used to store information between calls to make bulk + * operations faster. `*context` should be zero-initialized before the first + * call to this function. + * + * Modifying the bitmap in any way (other than `-bulk` suffixed functions) + * will invalidate the stored context, calling this function with a non-zero + * context after doing any modification invokes undefined behavior. + * + * In order to exploit this optimization, the caller should call this function + * with values with the same high 48 bits of the value consecutively. + */ +bool roaring64_bitmap_contains_bulk(const roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, + uint64_t val); + +/** + * Selects the element at index 'rank' where the smallest element is at index 0. + * If the size of the bitmap is strictly greater than rank, then this function + * returns true and sets element to the element of given rank. Otherwise, it + * returns false. + */ +bool roaring64_bitmap_select(const roaring64_bitmap_t *r, uint64_t rank, + uint64_t *element); + +/** + * Returns the number of integers that are smaller or equal to x. Thus if x is + * the first element, this function will return 1. If x is smaller than the + * smallest element, this function will return 0. + * + * The indexing convention differs between roaring64_bitmap_select and + * roaring64_bitmap_rank: roaring_bitmap64_select refers to the smallest value + * as having index 0, whereas roaring64_bitmap_rank returns 1 when ranking + * the smallest value. + */ +uint64_t roaring64_bitmap_rank(const roaring64_bitmap_t *r, uint64_t val); + +/** + * Returns true if the given value is in the bitmap, and sets `out_index` to the + * (0-based) index of the value in the bitmap. Returns false if the value is not + * in the bitmap. + */ +bool roaring64_bitmap_get_index(const roaring64_bitmap_t *r, uint64_t val, + uint64_t *out_index); + +/** + * Returns the number of values in the bitmap. + */ +uint64_t roaring64_bitmap_get_cardinality(const roaring64_bitmap_t *r); + +/** + * Returns the number of elements in the range [min, max). + */ +uint64_t roaring64_bitmap_range_cardinality(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max); + +/** + * Returns true if the bitmap is empty (cardinality is zero). + */ +bool roaring64_bitmap_is_empty(const roaring64_bitmap_t *r); + +/** + * Returns the smallest value in the set, or UINT64_MAX if the set is empty. + */ +uint64_t roaring64_bitmap_minimum(const roaring64_bitmap_t *r); + +/** + * Returns the largest value in the set, or 0 if empty. + */ +uint64_t roaring64_bitmap_maximum(const roaring64_bitmap_t *r); + +/** + * Returns true if the result has at least one run container. + */ +bool roaring64_bitmap_run_optimize(roaring64_bitmap_t *r); + +/** + * Perform internal consistency checks. + * + * Returns true if the bitmap is consistent. It may be useful to call this + * after deserializing bitmaps from untrusted sources. If + * roaring64_bitmap_internal_validate returns true, then the bitmap is + * consistent and can be trusted not to cause crashes or memory corruption. + * + * If reason is non-null, it will be set to a string describing the first + * inconsistency found if any. + */ +bool roaring64_bitmap_internal_validate(const roaring64_bitmap_t *r, + const char **reason); + +/** + * Return true if the two bitmaps contain the same elements. + */ +bool roaring64_bitmap_equals(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Return true if all the elements of r1 are also in r2. + */ +bool roaring64_bitmap_is_subset(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Return true if all the elements of r1 are also in r2, and r2 is strictly + * greater than r1. + */ +bool roaring64_bitmap_is_strict_subset(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the intersection between two bitmaps and returns new bitmap. The + * caller is responsible for free-ing the result. + * + * Performance hint: if you are computing the intersection between several + * bitmaps, two-by-two, it is best to start with the smallest bitmaps. You may + * also rely on roaring64_bitmap_and_inplace to avoid creating many temporary + * bitmaps. + */ +roaring64_bitmap_t *roaring64_bitmap_and(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the size of the intersection between two bitmaps. + */ +uint64_t roaring64_bitmap_and_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * In-place version of `roaring64_bitmap_and()`, modifies `r1`. `r1` and `r2` + * are allowed to be equal. + * + * Performance hint: if you are computing the intersection between several + * bitmaps, two-by-two, it is best to start with the smallest bitmaps. + */ +void roaring64_bitmap_and_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Check whether two bitmaps intersect. + */ +bool roaring64_bitmap_intersect(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Check whether a bitmap intersects the range [min, max). + */ +bool roaring64_bitmap_intersect_with_range(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max); + +/** + * Computes the Jaccard index between two bitmaps. (Also known as the Tanimoto + * distance, or the Jaccard similarity coefficient) + * + * The Jaccard index is undefined if both bitmaps are empty. + */ +double roaring64_bitmap_jaccard_index(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the union between two bitmaps and returns new bitmap. The caller is + * responsible for free-ing the result. + */ +roaring64_bitmap_t *roaring64_bitmap_or(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the size of the union between two bitmaps. + */ +uint64_t roaring64_bitmap_or_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * In-place version of `roaring64_bitmap_or(), modifies `r1`. + */ +void roaring64_bitmap_or_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the symmetric difference (xor) between two bitmaps and returns a new + * bitmap. The caller is responsible for free-ing the result. + */ +roaring64_bitmap_t *roaring64_bitmap_xor(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the size of the symmetric difference (xor) between two bitmaps. + */ +uint64_t roaring64_bitmap_xor_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * In-place version of `roaring64_bitmap_xor()`, modifies `r1`. `r1` and `r2` + * are not allowed to be equal (that would result in an empty bitmap). + */ +void roaring64_bitmap_xor_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the difference (andnot) between two bitmaps and returns a new + * bitmap. The caller is responsible for free-ing the result. + */ +roaring64_bitmap_t *roaring64_bitmap_andnot(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Computes the size of the difference (andnot) between two bitmaps. + */ +uint64_t roaring64_bitmap_andnot_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * In-place version of `roaring64_bitmap_andnot()`, modifies `r1`. `r1` and `r2` + * are not allowed to be equal (that would result in an empty bitmap). + */ +void roaring64_bitmap_andnot_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2); + +/** + * Compute the negation of the bitmap in the interval [min, max). + * The number of negated values is `max - min`. Areas outside the range are + * passed through unchanged. + */ +roaring64_bitmap_t *roaring64_bitmap_flip(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max); + +/** + * Compute the negation of the bitmap in the interval [min, max]. + * The number of negated values is `max - min + 1`. Areas outside the range are + * passed through unchanged. + */ +roaring64_bitmap_t *roaring64_bitmap_flip_closed(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max); + +/** + * In-place version of `roaring64_bitmap_flip`. Compute the negation of the + * bitmap in the interval [min, max). The number of negated values is `max - + * min`. Areas outside the range are passed through unchanged. + */ +void roaring64_bitmap_flip_inplace(roaring64_bitmap_t *r, uint64_t min, + uint64_t max); +/** + * In-place version of `roaring64_bitmap_flip_closed`. Compute the negation of + * the bitmap in the interval [min, max]. The number of negated values is `max - + * min + 1`. Areas outside the range are passed through unchanged. + */ +void roaring64_bitmap_flip_closed_inplace(roaring64_bitmap_t *r, uint64_t min, + uint64_t max); +/** + * How many bytes are required to serialize this bitmap. + * + * This is meant to be compatible with other languages: + * https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + */ +size_t roaring64_bitmap_portable_size_in_bytes(const roaring64_bitmap_t *r); + +/** + * Write a bitmap to a buffer. The output buffer should refer to at least + * `roaring64_bitmap_portable_size_in_bytes(r)` bytes of allocated memory. + * + * Returns how many bytes were written, which should match + * `roaring64_bitmap_portable_size_in_bytes(r)`. + * + * This is meant to be compatible with other languages: + * https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. + */ +size_t roaring64_bitmap_portable_serialize(const roaring64_bitmap_t *r, + char *buf); +/** + * Check how many bytes would be read (up to maxbytes) at this pointer if there + * is a valid bitmap, returns zero if there is no valid bitmap. + * + * This is meant to be compatible with other languages + * https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + */ +size_t roaring64_bitmap_portable_deserialize_size(const char *buf, + size_t maxbytes); + +/** + * Read a bitmap from a serialized buffer safely (reading up to maxbytes). + * In case of failure, NULL is returned. + * + * This is meant to be compatible with other languages + * https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + * + * The function itself is safe in the sense that it will not cause buffer + * overflows. However, for correct operations, it is assumed that the bitmap + * read was once serialized from a valid bitmap (i.e., it follows the format + * specification). If you provided an incorrect input (garbage), then the bitmap + * read may not be in a valid state and following operations may not lead to + * sensible results. In particular, the serialized array containers need to be + * in sorted order, and the run containers should be in sorted non-overlapping + * order. This is is guaranteed to happen when serializing an existing bitmap, + * but not for random inputs. + * + * This function is endian-sensitive. If you have a big-endian system (e.g., a + * mainframe IBM s390x), the data format is going to be big-endian and not + * compatible with little-endian systems. + */ +roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe(const char *buf, + size_t maxbytes); + +/** + * Iterate over the bitmap elements. The function `iterator` is called once for + * all the values with `ptr` (can be NULL) as the second parameter of each call. + * + * `roaring_iterator64` is simply a pointer to a function that returns a bool + * and takes `(uint64_t, void*)` as inputs. True means that the iteration should + * continue, while false means that it should stop. + * + * Returns true if the `roaring64_iterator` returned true throughout (so that + * all data points were necessarily visited). + * + * Iteration is ordered from the smallest to the largest elements. + */ +bool roaring64_bitmap_iterate(const roaring64_bitmap_t *r, + roaring_iterator64 iterator, void *ptr); + +/** + * Convert the bitmap to a sorted array `out`. + * + * Caller is responsible to ensure that there is enough memory allocated, e.g. + * ``` + * out = malloc(roaring64_bitmap_get_cardinality(bitmap) * sizeof(uint64_t)); + * ``` + */ +void roaring64_bitmap_to_uint64_array(const roaring64_bitmap_t *r, + uint64_t *out); + +/** + * Create an iterator object that can be used to iterate through the values. + * Caller is responsible for calling `roaring64_iterator_free()`. + * + * The iterator is initialized. If there is a value, then this iterator points + * to the first value and `roaring64_iterator_has_value()` returns true. The + * value can be retrieved with `roaring64_iterator_value()`. + */ +roaring64_iterator_t *roaring64_iterator_create(const roaring64_bitmap_t *r); + +/** + * Create an iterator object that can be used to iterate through the values. + * Caller is responsible for calling `roaring64_iterator_free()`. + * + * The iterator is initialized. If there is a value, then this iterator points + * to the last value and `roaring64_iterator_has_value()` returns true. The + * value can be retrieved with `roaring64_iterator_value()`. + */ +roaring64_iterator_t *roaring64_iterator_create_last( + const roaring64_bitmap_t *r); + +/** + * Re-initializes an existing iterator. Functionally the same as + * `roaring64_iterator_create` without a allocation. + */ +void roaring64_iterator_reinit(const roaring64_bitmap_t *r, + roaring64_iterator_t *it); + +/** + * Re-initializes an existing iterator. Functionally the same as + * `roaring64_iterator_create_last` without a allocation. + */ +void roaring64_iterator_reinit_last(const roaring64_bitmap_t *r, + roaring64_iterator_t *it); + +/** + * Creates a copy of the iterator. Caller is responsible for calling + * `roaring64_iterator_free()` on the resulting iterator. + */ +roaring64_iterator_t *roaring64_iterator_copy(const roaring64_iterator_t *it); + +/** + * Free the iterator. + */ +void roaring64_iterator_free(roaring64_iterator_t *it); + +/** + * Returns true if the iterator currently points to a value. If so, calling + * `roaring64_iterator_value()` returns the value. + */ +bool roaring64_iterator_has_value(const roaring64_iterator_t *it); + +/** + * Returns the value the iterator currently points to. Should only be called if + * `roaring64_iterator_has_value()` returns true. + */ +uint64_t roaring64_iterator_value(const roaring64_iterator_t *it); + +/** + * Advance the iterator. If there is a new value, then + * `roaring64_iterator_has_value()` returns true. Values are traversed in + * increasing order. For convenience, returns the result of + * `roaring64_iterator_has_value()`. + * + * Once this returns false, `roaring64_iterator_advance` should not be called on + * the iterator again. Calling `roaring64_iterator_previous` is allowed. + */ +bool roaring64_iterator_advance(roaring64_iterator_t *it); + +/** + * Decrement the iterator. If there is a new value, then + * `roaring64_iterator_has_value()` returns true. Values are traversed in + * decreasing order. For convenience, returns the result of + * `roaring64_iterator_has_value()`. + * + * Once this returns false, `roaring64_iterator_previous` should not be called + * on the iterator again. Calling `roaring64_iterator_advance` is allowed. + */ +bool roaring64_iterator_previous(roaring64_iterator_t *it); + +/** + * Move the iterator to the first value greater than or equal to `val`, if it + * exists at or after the current position of the iterator. If there is a new + * value, then `roaring64_iterator_has_value()` returns true. Values are + * traversed in increasing order. For convenience, returns the result of + * `roaring64_iterator_has_value()`. + */ +bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, + uint64_t val); + +/** + * Reads up to `count` values from the iterator into the given `buf`. Returns + * the number of elements read. The number of elements read can be smaller than + * `count`, which means that there are no more elements in the bitmap. + * + * This function can be used together with other iterator functions. + */ +uint64_t roaring64_iterator_read(roaring64_iterator_t *it, uint64_t *buf, + uint64_t count); + +#ifdef __cplusplus +} // extern "C" +} // namespace roaring +} // namespace api +#endif + +#endif /* ROARING64_H */ diff --git a/contrib/libs/croaring/include/roaring/roaring_array.h b/contrib/libs/croaring/include/roaring/roaring_array.h new file mode 100644 index 00000000000..594a10b6089 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/roaring_array.h @@ -0,0 +1,305 @@ +#ifndef INCLUDE_ROARING_ARRAY_H +#define INCLUDE_ROARING_ARRAY_H + +#include <assert.h> +#include <stdbool.h> +#include <stdint.h> + +#include <roaring/array_util.h> +#include <roaring/containers/containers.h> // get_writable_copy_if_shared() + +#ifdef __cplusplus +extern "C" { +namespace roaring { + +// Note: in pure C++ code, you should avoid putting `using` in header files +using api::roaring_array_t; + +namespace internal { +#endif + +enum { + SERIAL_COOKIE_NO_RUNCONTAINER = 12346, + SERIAL_COOKIE = 12347, + FROZEN_COOKIE = 13766, + NO_OFFSET_THRESHOLD = 4 +}; + +/** + * Create a new roaring array + */ +roaring_array_t *ra_create(void); + +/** + * Initialize an existing roaring array with the specified capacity (in number + * of containers) + */ +bool ra_init_with_capacity(roaring_array_t *new_ra, uint32_t cap); + +/** + * Initialize with zero capacity + */ +void ra_init(roaring_array_t *t); + +/** + * Copies this roaring array, we assume that dest is not initialized + */ +bool ra_copy(const roaring_array_t *source, roaring_array_t *dest, + bool copy_on_write); + +/* + * Shrinks the capacity, returns the number of bytes saved. + */ +int ra_shrink_to_fit(roaring_array_t *ra); + +/** + * Copies this roaring array, we assume that dest is initialized + */ +bool ra_overwrite(const roaring_array_t *source, roaring_array_t *dest, + bool copy_on_write); + +/** + * Frees the memory used by a roaring array + */ +void ra_clear(roaring_array_t *r); + +/** + * Frees the memory used by a roaring array, but does not free the containers + */ +void ra_clear_without_containers(roaring_array_t *r); + +/** + * Frees just the containers + */ +void ra_clear_containers(roaring_array_t *ra); + +/** + * Get the index corresponding to a 16-bit key + */ +inline int32_t ra_get_index(const roaring_array_t *ra, uint16_t x) { + if ((ra->size == 0) || ra->keys[ra->size - 1] == x) return ra->size - 1; + return binarySearch(ra->keys, (int32_t)ra->size, x); +} + +/** + * Retrieves the container at index i, filling in the typecode + */ +inline container_t *ra_get_container_at_index(const roaring_array_t *ra, + uint16_t i, uint8_t *typecode) { + *typecode = ra->typecodes[i]; + return ra->containers[i]; +} + +/** + * Retrieves the key at index i + */ +inline uint16_t ra_get_key_at_index(const roaring_array_t *ra, uint16_t i) { + return ra->keys[i]; +} + +/** + * Add a new key-value pair at index i + */ +void ra_insert_new_key_value_at(roaring_array_t *ra, int32_t i, uint16_t key, + container_t *c, uint8_t typecode); + +/** + * Append a new key-value pair + */ +void ra_append(roaring_array_t *ra, uint16_t key, container_t *c, + uint8_t typecode); + +/** + * Append a new key-value pair to ra, cloning (in COW sense) a value from sa + * at index index + */ +void ra_append_copy(roaring_array_t *ra, const roaring_array_t *sa, + uint16_t index, bool copy_on_write); + +/** + * Append new key-value pairs to ra, cloning (in COW sense) values from sa + * at indexes + * [start_index, end_index) + */ +void ra_append_copy_range(roaring_array_t *ra, const roaring_array_t *sa, + int32_t start_index, int32_t end_index, + bool copy_on_write); + +/** appends from sa to ra, ending with the greatest key that is + * is less or equal stopping_key + */ +void ra_append_copies_until(roaring_array_t *ra, const roaring_array_t *sa, + uint16_t stopping_key, bool copy_on_write); + +/** appends from sa to ra, starting with the smallest key that is + * is strictly greater than before_start + */ + +void ra_append_copies_after(roaring_array_t *ra, const roaring_array_t *sa, + uint16_t before_start, bool copy_on_write); + +/** + * Move the key-value pairs to ra from sa at indexes + * [start_index, end_index), old array should not be freed + * (use ra_clear_without_containers) + **/ +void ra_append_move_range(roaring_array_t *ra, roaring_array_t *sa, + int32_t start_index, int32_t end_index); +/** + * Append new key-value pairs to ra, from sa at indexes + * [start_index, end_index) + */ +void ra_append_range(roaring_array_t *ra, roaring_array_t *sa, + int32_t start_index, int32_t end_index, + bool copy_on_write); + +/** + * Set the container at the corresponding index using the specified + * typecode. + */ +inline void ra_set_container_at_index(const roaring_array_t *ra, int32_t i, + container_t *c, uint8_t typecode) { + assert(i < ra->size); + ra->containers[i] = c; + ra->typecodes[i] = typecode; +} + +container_t *ra_get_container(roaring_array_t *ra, uint16_t x, + uint8_t *typecode); + +/** + * If needed, increase the capacity of the array so that it can fit k values + * (at + * least); + */ +bool extend_array(roaring_array_t *ra, int32_t k); + +inline int32_t ra_get_size(const roaring_array_t *ra) { return ra->size; } + +static inline int32_t ra_advance_until(const roaring_array_t *ra, uint16_t x, + int32_t pos) { + return advanceUntil(ra->keys, pos, ra->size, x); +} + +int32_t ra_advance_until_freeing(roaring_array_t *ra, uint16_t x, int32_t pos); + +void ra_downsize(roaring_array_t *ra, int32_t new_length); + +inline void ra_replace_key_and_container_at_index(roaring_array_t *ra, + int32_t i, uint16_t key, + container_t *c, + uint8_t typecode) { + assert(i < ra->size); + + ra->keys[i] = key; + ra->containers[i] = c; + ra->typecodes[i] = typecode; +} + +// write set bits to an array +void ra_to_uint32_array(const roaring_array_t *ra, uint32_t *ans); + +bool ra_range_uint32_array(const roaring_array_t *ra, size_t offset, + size_t limit, uint32_t *ans); + +/** + * write a bitmap to a buffer. This is meant to be compatible with + * the + * Java and Go versions. Return the size in bytes of the serialized + * output (which should be ra_portable_size_in_bytes(ra)). + */ +size_t ra_portable_serialize(const roaring_array_t *ra, char *buf); + +/** + * read a bitmap from a serialized version. This is meant to be compatible + * with the Java and Go versions. + * maxbytes indicates how many bytes available from buf. + * When the function returns true, roaring_array_t is populated with the data + * and *readbytes indicates how many bytes were read. In all cases, if the + * function returns true, then maxbytes >= *readbytes. + */ +bool ra_portable_deserialize(roaring_array_t *ra, const char *buf, + const size_t maxbytes, size_t *readbytes); + +/** + * Quickly checks whether there is a serialized bitmap at the pointer, + * not exceeding size "maxbytes" in bytes. This function does not allocate + * memory dynamically. + * + * This function returns 0 if and only if no valid bitmap is found. + * Otherwise, it returns how many bytes are occupied by the bitmap data. + */ +size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes); + +/** + * How many bytes are required to serialize this bitmap (meant to be + * compatible + * with Java and Go versions) + */ +size_t ra_portable_size_in_bytes(const roaring_array_t *ra); + +/** + * return true if it contains at least one run container. + */ +bool ra_has_run_container(const roaring_array_t *ra); + +/** + * Size of the header when serializing (meant to be compatible + * with Java and Go versions) + */ +uint32_t ra_portable_header_size(const roaring_array_t *ra); + +/** + * If the container at the index i is share, unshare it (creating a local + * copy if needed). + */ +static inline void ra_unshare_container_at_index(roaring_array_t *ra, + uint16_t i) { + assert(i < ra->size); + ra->containers[i] = + get_writable_copy_if_shared(ra->containers[i], &ra->typecodes[i]); +} + +/** + * remove at index i, sliding over all entries after i + */ +void ra_remove_at_index(roaring_array_t *ra, int32_t i); + +/** + * clears all containers, sets the size at 0 and shrinks the memory usage. + */ +void ra_reset(roaring_array_t *ra); + +/** + * remove at index i, sliding over all entries after i. Free removed container. + */ +void ra_remove_at_index_and_free(roaring_array_t *ra, int32_t i); + +/** + * remove a chunk of indices, sliding over entries after it + */ +// void ra_remove_index_range(roaring_array_t *ra, int32_t begin, int32_t end); + +// used in inplace andNot only, to slide left the containers from +// the mutated RoaringBitmap that are after the largest container of +// the argument RoaringBitmap. It is followed by a call to resize. +// +void ra_copy_range(roaring_array_t *ra, uint32_t begin, uint32_t end, + uint32_t new_begin); + +/** + * Shifts rightmost $count containers to the left (distance < 0) or + * to the right (distance > 0). + * Allocates memory if necessary. + * This function doesn't free or create new containers. + * Caller is responsible for that. + */ +void ra_shift_tail(roaring_array_t *ra, int32_t count, int32_t distance); + +#ifdef __cplusplus +} // namespace internal +} +} // extern "C" { namespace roaring { +#endif + +#endif diff --git a/contrib/libs/croaring/include/roaring/roaring_types.h b/contrib/libs/croaring/include/roaring/roaring_types.h new file mode 100644 index 00000000000..d215c248b7b --- /dev/null +++ b/contrib/libs/croaring/include/roaring/roaring_types.h @@ -0,0 +1,115 @@ +/* + Typedefs used by various components +*/ + +#ifndef ROARING_TYPES_H +#define ROARING_TYPES_H + +#include <stdbool.h> +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace api { +#endif + +/** + * When building .c files as C++, there's added compile-time checking if the + * container types are derived from a `container_t` base class. So long as + * such a base class is empty, the struct will behave compatibly with C structs + * despite the derivation. This is due to the Empty Base Class Optimization: + * + * https://en.cppreference.com/w/cpp/language/ebo + * + * But since C isn't namespaced, taking `container_t` globally might collide + * with other projects. So roaring.h uses ROARING_CONTAINER_T, while internal + * code #undefs that after declaring `typedef ROARING_CONTAINER_T container_t;` + */ +#if defined(__cplusplus) +extern "C++" { +struct container_s {}; +} +#define ROARING_CONTAINER_T ::roaring::api::container_s +#else +#define ROARING_CONTAINER_T void // no compile-time checking +#endif + +#define ROARING_FLAG_COW UINT8_C(0x1) +#define ROARING_FLAG_FROZEN UINT8_C(0x2) + +/** + * Roaring arrays are array-based key-value pairs having containers as values + * and 16-bit integer keys. A roaring bitmap might be implemented as such. + */ + +// parallel arrays. Element sizes quite different. +// Alternative is array +// of structs. Which would have better +// cache performance through binary searches? + +typedef struct roaring_array_s { + int32_t size; + int32_t allocation_size; + ROARING_CONTAINER_T **containers; // Use container_t in non-API files! + uint16_t *keys; + uint8_t *typecodes; + uint8_t flags; +} roaring_array_t; + +typedef bool (*roaring_iterator)(uint32_t value, void *param); +typedef bool (*roaring_iterator64)(uint64_t value, void *param); + +/** + * (For advanced users.) + * The roaring_statistics_t can be used to collect detailed statistics about + * the composition of a roaring bitmap. + */ +typedef struct roaring_statistics_s { + uint32_t n_containers; /* number of containers */ + + uint32_t n_array_containers; /* number of array containers */ + uint32_t n_run_containers; /* number of run containers */ + uint32_t n_bitset_containers; /* number of bitmap containers */ + + uint32_t + n_values_array_containers; /* number of values in array containers */ + uint32_t n_values_run_containers; /* number of values in run containers */ + uint32_t + n_values_bitset_containers; /* number of values in bitmap containers */ + + uint32_t n_bytes_array_containers; /* number of allocated bytes in array + containers */ + uint32_t n_bytes_run_containers; /* number of allocated bytes in run + containers */ + uint32_t n_bytes_bitset_containers; /* number of allocated bytes in bitmap + containers */ + + uint32_t + max_value; /* the maximal value, undefined if cardinality is zero */ + uint32_t + min_value; /* the minimal value, undefined if cardinality is zero */ + uint64_t sum_value; /* the sum of all values (could be used to compute + average) */ + + uint64_t cardinality; /* total number of values stored in the bitmap */ + + // and n_values_arrays, n_values_rle, n_values_bitmap +} roaring_statistics_t; + +/** + * Roaring-internal type used to iterate within a roaring container. + */ +typedef struct roaring_container_iterator_s { + // For bitset and array containers this is the index of the bit / entry. + // For run containers this points at the run. + int32_t index; +} roaring_container_iterator_t; + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace api { +#endif + +#endif /* ROARING_TYPES_H */ diff --git a/contrib/libs/croaring/include/roaring/roaring_version.h b/contrib/libs/croaring/include/roaring/roaring_version.h new file mode 100644 index 00000000000..b2efa33a07c --- /dev/null +++ b/contrib/libs/croaring/include/roaring/roaring_version.h @@ -0,0 +1,10 @@ +// /include/roaring/roaring_version.h automatically generated by release.py, do not change by hand +#ifndef ROARING_INCLUDE_ROARING_VERSION +#define ROARING_INCLUDE_ROARING_VERSION +#define ROARING_VERSION "3.0.0" +enum { + ROARING_VERSION_MAJOR = 3, + ROARING_VERSION_MINOR = 0, + ROARING_VERSION_REVISION = 0 +}; +#endif // ROARING_INCLUDE_ROARING_VERSION diff --git a/contrib/libs/croaring/include/roaring/utilasm.h b/contrib/libs/croaring/include/roaring/utilasm.h new file mode 100644 index 00000000000..860a706e335 --- /dev/null +++ b/contrib/libs/croaring/include/roaring/utilasm.h @@ -0,0 +1,80 @@ +/* + * utilasm.h + * + */ + +#ifndef INCLUDE_UTILASM_H_ +#define INCLUDE_UTILASM_H_ + +#include <roaring/portability.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +#endif + +#if defined(CROARING_INLINE_ASM) +#define CROARING_ASMBITMANIPOPTIMIZATION // optimization flag + +#define ASM_SHIFT_RIGHT(srcReg, bitsReg, destReg) \ + __asm volatile("shrx %1, %2, %0" \ + : "=r"(destReg) \ + : /* write */ \ + "r"(bitsReg), /* read only */ \ + "r"(srcReg) /* read only */ \ + ) + +#define ASM_INPLACESHIFT_RIGHT(srcReg, bitsReg) \ + __asm volatile("shrx %1, %0, %0" \ + : "+r"(srcReg) \ + : /* read/write */ \ + "r"(bitsReg) /* read only */ \ + ) + +#define ASM_SHIFT_LEFT(srcReg, bitsReg, destReg) \ + __asm volatile("shlx %1, %2, %0" \ + : "=r"(destReg) \ + : /* write */ \ + "r"(bitsReg), /* read only */ \ + "r"(srcReg) /* read only */ \ + ) +// set bit at position testBit within testByte to 1 and +// copy cmovDst to cmovSrc if that bit was previously clear +#define ASM_SET_BIT_INC_WAS_CLEAR(testByte, testBit, count) \ + __asm volatile( \ + "bts %2, %0\n" \ + "sbb $-1, %1\n" \ + : "+r"(testByte), /* read/write */ \ + "+r"(count) \ + : /* read/write */ \ + "r"(testBit) /* read only */ \ + ) + +#define ASM_CLEAR_BIT_DEC_WAS_SET(testByte, testBit, count) \ + __asm volatile( \ + "btr %2, %0\n" \ + "sbb $0, %1\n" \ + : "+r"(testByte), /* read/write */ \ + "+r"(count) \ + : /* read/write */ \ + "r"(testBit) /* read only */ \ + ) + +#define ASM_BT64(testByte, testBit, count) \ + __asm volatile( \ + "bt %2,%1\n" \ + "sbb %0,%0" /*could use setb */ \ + : "=r"(count) \ + : /* write */ \ + "r"(testByte), /* read only */ \ + "r"(testBit) /* read only */ \ + ) + +#endif + +#ifdef __cplusplus +} +} // extern "C" { namespace roaring { +#endif + +#endif /* INCLUDE_UTILASM_H_ */ diff --git a/contrib/libs/croaring/src/array_util.c b/contrib/libs/croaring/src/array_util.c new file mode 100644 index 00000000000..a4ff98d2dfe --- /dev/null +++ b/contrib/libs/croaring/src/array_util.c @@ -0,0 +1,2172 @@ +#include <assert.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <roaring/array_util.h> +#include <roaring/portability.h> +#include <roaring/utilasm.h> + +#if CROARING_IS_X64 +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif + +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif +#ifdef __cplusplus +using namespace ::roaring::internal; +extern "C" { +namespace roaring { +namespace internal { +#endif + +extern inline int32_t binarySearch(const uint16_t *array, int32_t lenarray, + uint16_t ikey); + +#if CROARING_IS_X64 +// used by intersect_vector16 +ALIGNED(0x1000) +static const uint8_t shuffle_mask16[] = { + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 4, 5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 4, 5, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 6, 7, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 6, 7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 6, 7, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 6, 7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 4, 5, 6, 7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, 6, 7, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, + 6, 7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 4, 5, 6, 7, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 8, 9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 8, 9, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 8, 9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 8, 9, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 8, 9, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 4, 5, 8, 9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, 8, 9, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 4, 5, 8, 9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 6, 7, 8, 9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 6, 7, 8, 9, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 6, 7, + 8, 9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 6, 7, 8, 9, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 6, 7, 8, 9, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, + 6, 7, 8, 9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 4, 5, 6, 7, 8, 9, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 10, 11, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 4, 5, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, 10, 11, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, + 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 4, 5, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 6, 7, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 6, 7, + 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 6, 7, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 6, 7, 10, 11, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 6, 7, + 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 4, 5, 6, 7, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, 6, 7, 10, 11, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 4, 5, 6, 7, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 8, 9, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 8, 9, 10, 11, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 8, 9, + 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 8, 9, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 8, 9, 10, 11, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, + 8, 9, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 4, 5, 8, 9, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 6, 7, 8, 9, + 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 6, 7, 8, 9, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 6, 7, 8, 9, 10, 11, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 6, 7, 8, 9, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 4, 5, 6, 7, 8, 9, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, + 6, 7, 8, 9, 10, 11, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 0xFF, 0xFF, 0xFF, 0xFF, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 12, 13, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 12, 13, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 12, 13, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 4, 5, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, 12, 13, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 4, 5, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 6, 7, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 6, 7, 12, 13, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 6, 7, + 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 6, 7, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 6, 7, 12, 13, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, + 6, 7, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 4, 5, 6, 7, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 4, 5, 6, 7, + 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 8, 9, 12, 13, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 8, 9, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 8, 9, 12, 13, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 8, 9, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 4, 5, 8, 9, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, 8, 9, 12, 13, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, + 8, 9, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 4, 5, 8, 9, 12, 13, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 6, 7, 8, 9, 12, 13, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 6, 7, + 8, 9, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 6, 7, 8, 9, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 6, 7, 8, 9, + 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 6, 7, + 8, 9, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 4, 5, 6, 7, 8, 9, 12, 13, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, 6, 7, 8, 9, + 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 4, 5, 6, 7, 8, 9, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, + 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 10, 11, 12, 13, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 10, 11, + 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 10, 11, 12, 13, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, + 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 4, 5, 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 4, 5, 10, 11, + 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 6, 7, 10, 11, + 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 6, 7, 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 6, 7, 10, 11, 12, 13, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 6, 7, 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 4, 5, 6, 7, 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, 6, 7, 10, 11, + 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, + 6, 7, 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, + 0xFF, 0xFF, 0xFF, 0xFF, 8, 9, 10, 11, 12, 13, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 8, 9, + 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 8, 9, 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 8, 9, 10, 11, + 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 8, 9, + 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 4, 5, 8, 9, 10, 11, 12, 13, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, 8, 9, 10, 11, + 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 4, 5, 8, 9, 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, + 6, 7, 8, 9, 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 6, 7, 8, 9, 10, 11, + 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 6, 7, + 8, 9, 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 6, 7, 8, 9, 10, 11, 12, 13, + 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, + 6, 7, 8, 9, 10, 11, 12, 13, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 0xFF, 0xFF, 14, 15, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 4, 5, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, 14, 15, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 4, 5, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 6, 7, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 6, 7, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 6, 7, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 6, 7, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 6, 7, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 4, 5, 6, 7, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, 6, 7, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 4, 5, 6, 7, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 8, 9, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 8, 9, 14, 15, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 8, 9, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 8, 9, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 8, 9, 14, 15, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, + 8, 9, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 4, 5, 8, 9, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 4, 5, 8, 9, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 6, 7, 8, 9, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 6, 7, 8, 9, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 6, 7, 8, 9, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 6, 7, 8, 9, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 4, 5, 6, 7, 8, 9, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, 6, 7, 8, 9, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, + 6, 7, 8, 9, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 10, 11, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 10, 11, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 10, 11, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 4, 5, 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, 10, 11, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 4, 5, 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 6, 7, 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 6, 7, 10, 11, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 6, 7, + 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 6, 7, 10, 11, 14, 15, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 6, 7, 10, 11, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, + 6, 7, 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 4, 5, 6, 7, 10, 11, 14, 15, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 4, 5, 6, 7, + 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 8, 9, 10, 11, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 8, 9, 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 8, 9, 10, 11, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 8, 9, 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 4, 5, 8, 9, 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, 8, 9, 10, 11, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, + 8, 9, 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 6, 7, 8, 9, 10, 11, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 6, 7, + 8, 9, 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 6, 7, 8, 9, 10, 11, 14, 15, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 6, 7, 8, 9, + 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 6, 7, + 8, 9, 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 4, 5, 6, 7, 8, 9, 10, 11, 14, 15, 0xFF, 0xFF, + 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 12, 13, 14, 15, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 12, 13, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 12, 13, 14, 15, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, + 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 4, 5, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 4, 5, 12, 13, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 6, 7, 12, 13, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 6, 7, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 6, 7, 12, 13, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 6, 7, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 4, 5, 6, 7, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, 6, 7, 12, 13, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, + 6, 7, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 8, 9, 12, 13, 14, 15, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 8, 9, + 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 8, 9, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 8, 9, 12, 13, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 8, 9, + 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 4, 5, 8, 9, 12, 13, 14, 15, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, 8, 9, 12, 13, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 4, 5, 8, 9, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 6, 7, 8, 9, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 6, 7, 8, 9, 12, 13, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 6, 7, + 8, 9, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 6, 7, 8, 9, 12, 13, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 6, 7, 8, 9, 12, 13, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, + 6, 7, 8, 9, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 4, 5, 6, 7, 8, 9, 12, 13, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 12, 13, 14, 15, 0xFF, 0xFF, 10, 11, 12, 13, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 10, 11, 12, 13, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 4, 5, 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, 10, 11, 12, 13, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, + 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 4, 5, 10, 11, 12, 13, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 6, 7, 10, 11, 12, 13, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 6, 7, + 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 6, 7, 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 6, 7, 10, 11, + 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 6, 7, + 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 4, 5, 6, 7, 10, 11, + 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 4, 5, 6, 7, 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, + 8, 9, 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 8, 9, 10, 11, 12, 13, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 8, 9, + 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 4, 5, 8, 9, 10, 11, 12, 13, + 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, + 8, 9, 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, + 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, 4, 5, 8, 9, + 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0, 1, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 2, 3, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 2, 3, + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 0xFF, 0xFF, 0xFF, 0xFF, 0, 1, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, 2, 3, 4, 5, + 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0xFF, 0xFF, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15}; + +/** + * From Schlegel et al., Fast Sorted-Set Intersection using SIMD Instructions + * Optimized by D. Lemire on May 3rd 2013 + */ +CROARING_TARGET_AVX2 +int32_t intersect_vector16(const uint16_t *__restrict__ A, size_t s_a, + const uint16_t *__restrict__ B, size_t s_b, + uint16_t *C) { + size_t count = 0; + size_t i_a = 0, i_b = 0; + const int vectorlength = sizeof(__m128i) / sizeof(uint16_t); + const size_t st_a = (s_a / vectorlength) * vectorlength; + const size_t st_b = (s_b / vectorlength) * vectorlength; + __m128i v_a, v_b; + if ((i_a < st_a) && (i_b < st_b)) { + v_a = _mm_lddqu_si128((__m128i *)&A[i_a]); + v_b = _mm_lddqu_si128((__m128i *)&B[i_b]); + while ((A[i_a] == 0) || (B[i_b] == 0)) { + const __m128i res_v = _mm_cmpestrm( + v_b, vectorlength, v_a, vectorlength, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + const int r = _mm_extract_epi32(res_v, 0); + __m128i sm16 = _mm_loadu_si128((const __m128i *)shuffle_mask16 + r); + __m128i p = _mm_shuffle_epi8(v_a, sm16); + _mm_storeu_si128((__m128i *)&C[count], p); // can overflow + count += _mm_popcnt_u32(r); + const uint16_t a_max = A[i_a + vectorlength - 1]; + const uint16_t b_max = B[i_b + vectorlength - 1]; + if (a_max <= b_max) { + i_a += vectorlength; + if (i_a == st_a) break; + v_a = _mm_lddqu_si128((__m128i *)&A[i_a]); + } + if (b_max <= a_max) { + i_b += vectorlength; + if (i_b == st_b) break; + v_b = _mm_lddqu_si128((__m128i *)&B[i_b]); + } + } + if ((i_a < st_a) && (i_b < st_b)) + while (true) { + const __m128i res_v = _mm_cmpistrm( + v_b, v_a, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + const int r = _mm_extract_epi32(res_v, 0); + __m128i sm16 = + _mm_loadu_si128((const __m128i *)shuffle_mask16 + r); + __m128i p = _mm_shuffle_epi8(v_a, sm16); + _mm_storeu_si128((__m128i *)&C[count], p); // can overflow + count += _mm_popcnt_u32(r); + const uint16_t a_max = A[i_a + vectorlength - 1]; + const uint16_t b_max = B[i_b + vectorlength - 1]; + if (a_max <= b_max) { + i_a += vectorlength; + if (i_a == st_a) break; + v_a = _mm_lddqu_si128((__m128i *)&A[i_a]); + } + if (b_max <= a_max) { + i_b += vectorlength; + if (i_b == st_b) break; + v_b = _mm_lddqu_si128((__m128i *)&B[i_b]); + } + } + } + // intersect the tail using scalar intersection + while (i_a < s_a && i_b < s_b) { + uint16_t a = A[i_a]; + uint16_t b = B[i_b]; + if (a < b) { + i_a++; + } else if (b < a) { + i_b++; + } else { + C[count] = a; //==b; + count++; + i_a++; + i_b++; + } + } + return (int32_t)count; +} + +ALLOW_UNALIGNED +int array_container_to_uint32_array_vector16(void *vout, const uint16_t *array, + size_t cardinality, + uint32_t base) { + int outpos = 0; + uint32_t *out = (uint32_t *)vout; + size_t i = 0; + for (; i + sizeof(__m128i) / sizeof(uint16_t) <= cardinality; + i += sizeof(__m128i) / sizeof(uint16_t)) { + __m128i vinput = _mm_loadu_si128((const __m128i *)(array + i)); + __m256i voutput = _mm256_add_epi32(_mm256_cvtepu16_epi32(vinput), + _mm256_set1_epi32(base)); + _mm256_storeu_si256((__m256i *)(out + outpos), voutput); + outpos += sizeof(__m256i) / sizeof(uint32_t); + } + for (; i < cardinality; ++i) { + const uint32_t val = base + array[i]; + memcpy(out + outpos, &val, + sizeof(uint32_t)); // should be compiled as a MOV on x64 + outpos++; + } + return outpos; +} + +int32_t intersect_vector16_inplace(uint16_t *__restrict__ A, size_t s_a, + const uint16_t *__restrict__ B, size_t s_b) { + size_t count = 0; + size_t i_a = 0, i_b = 0; + const int vectorlength = sizeof(__m128i) / sizeof(uint16_t); + const size_t st_a = (s_a / vectorlength) * vectorlength; + const size_t st_b = (s_b / vectorlength) * vectorlength; + __m128i v_a, v_b; + if ((i_a < st_a) && (i_b < st_b)) { + v_a = _mm_lddqu_si128((__m128i *)&A[i_a]); + v_b = _mm_lddqu_si128((__m128i *)&B[i_b]); + __m128i tmp[2] = {_mm_setzero_si128()}; + size_t tmp_count = 0; + while ((A[i_a] == 0) || (B[i_b] == 0)) { + const __m128i res_v = _mm_cmpestrm( + v_b, vectorlength, v_a, vectorlength, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + const int r = _mm_extract_epi32(res_v, 0); + __m128i sm16 = _mm_loadu_si128((const __m128i *)shuffle_mask16 + r); + __m128i p = _mm_shuffle_epi8(v_a, sm16); + _mm_storeu_si128((__m128i *)&((uint16_t *)tmp)[tmp_count], p); + tmp_count += _mm_popcnt_u32(r); + const uint16_t a_max = A[i_a + vectorlength - 1]; + const uint16_t b_max = B[i_b + vectorlength - 1]; + if (a_max <= b_max) { + _mm_storeu_si128((__m128i *)&A[count], tmp[0]); + _mm_storeu_si128(tmp, _mm_setzero_si128()); + count += tmp_count; + tmp_count = 0; + i_a += vectorlength; + if (i_a == st_a) break; + v_a = _mm_lddqu_si128((__m128i *)&A[i_a]); + } + if (b_max <= a_max) { + i_b += vectorlength; + if (i_b == st_b) break; + v_b = _mm_lddqu_si128((__m128i *)&B[i_b]); + } + } + if ((i_a < st_a) && (i_b < st_b)) { + while (true) { + const __m128i res_v = _mm_cmpistrm( + v_b, v_a, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + const int r = _mm_extract_epi32(res_v, 0); + __m128i sm16 = + _mm_loadu_si128((const __m128i *)shuffle_mask16 + r); + __m128i p = _mm_shuffle_epi8(v_a, sm16); + _mm_storeu_si128((__m128i *)&((uint16_t *)tmp)[tmp_count], p); + tmp_count += _mm_popcnt_u32(r); + const uint16_t a_max = A[i_a + vectorlength - 1]; + const uint16_t b_max = B[i_b + vectorlength - 1]; + if (a_max <= b_max) { + _mm_storeu_si128((__m128i *)&A[count], tmp[0]); + _mm_storeu_si128(tmp, _mm_setzero_si128()); + count += tmp_count; + tmp_count = 0; + i_a += vectorlength; + if (i_a == st_a) break; + v_a = _mm_lddqu_si128((__m128i *)&A[i_a]); + } + if (b_max <= a_max) { + i_b += vectorlength; + if (i_b == st_b) break; + v_b = _mm_lddqu_si128((__m128i *)&B[i_b]); + } + } + } + // tmp_count <= 8, so this does not affect efficiency so much + for (size_t i = 0; i < tmp_count; i++) { + A[count] = ((uint16_t *)tmp)[i]; + count++; + } + i_a += tmp_count; // We can at least jump pass $tmp_count elements in A + } + // intersect the tail using scalar intersection + while (i_a < s_a && i_b < s_b) { + uint16_t a = A[i_a]; + uint16_t b = B[i_b]; + if (a < b) { + i_a++; + } else if (b < a) { + i_b++; + } else { + A[count] = a; //==b; + count++; + i_a++; + i_b++; + } + } + return (int32_t)count; +} +CROARING_UNTARGET_AVX2 + +CROARING_TARGET_AVX2 +int32_t intersect_vector16_cardinality(const uint16_t *__restrict__ A, + size_t s_a, + const uint16_t *__restrict__ B, + size_t s_b) { + size_t count = 0; + size_t i_a = 0, i_b = 0; + const int vectorlength = sizeof(__m128i) / sizeof(uint16_t); + const size_t st_a = (s_a / vectorlength) * vectorlength; + const size_t st_b = (s_b / vectorlength) * vectorlength; + __m128i v_a, v_b; + if ((i_a < st_a) && (i_b < st_b)) { + v_a = _mm_lddqu_si128((__m128i *)&A[i_a]); + v_b = _mm_lddqu_si128((__m128i *)&B[i_b]); + while ((A[i_a] == 0) || (B[i_b] == 0)) { + const __m128i res_v = _mm_cmpestrm( + v_b, vectorlength, v_a, vectorlength, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + const int r = _mm_extract_epi32(res_v, 0); + count += _mm_popcnt_u32(r); + const uint16_t a_max = A[i_a + vectorlength - 1]; + const uint16_t b_max = B[i_b + vectorlength - 1]; + if (a_max <= b_max) { + i_a += vectorlength; + if (i_a == st_a) break; + v_a = _mm_lddqu_si128((__m128i *)&A[i_a]); + } + if (b_max <= a_max) { + i_b += vectorlength; + if (i_b == st_b) break; + v_b = _mm_lddqu_si128((__m128i *)&B[i_b]); + } + } + if ((i_a < st_a) && (i_b < st_b)) + while (true) { + const __m128i res_v = _mm_cmpistrm( + v_b, v_a, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + const int r = _mm_extract_epi32(res_v, 0); + count += _mm_popcnt_u32(r); + const uint16_t a_max = A[i_a + vectorlength - 1]; + const uint16_t b_max = B[i_b + vectorlength - 1]; + if (a_max <= b_max) { + i_a += vectorlength; + if (i_a == st_a) break; + v_a = _mm_lddqu_si128((__m128i *)&A[i_a]); + } + if (b_max <= a_max) { + i_b += vectorlength; + if (i_b == st_b) break; + v_b = _mm_lddqu_si128((__m128i *)&B[i_b]); + } + } + } + // intersect the tail using scalar intersection + while (i_a < s_a && i_b < s_b) { + uint16_t a = A[i_a]; + uint16_t b = B[i_b]; + if (a < b) { + i_a++; + } else if (b < a) { + i_b++; + } else { + count++; + i_a++; + i_b++; + } + } + return (int32_t)count; +} +CROARING_UNTARGET_AVX2 + +CROARING_TARGET_AVX2 +///////// +// Warning: +// This function may not be safe if A == C or B == C. +///////// +int32_t difference_vector16(const uint16_t *__restrict__ A, size_t s_a, + const uint16_t *__restrict__ B, size_t s_b, + uint16_t *C) { + // we handle the degenerate case + if (s_a == 0) return 0; + if (s_b == 0) { + if (A != C) memcpy(C, A, sizeof(uint16_t) * s_a); + return (int32_t)s_a; + } + // handle the leading zeroes, it is messy but it allows us to use the fast + // _mm_cmpistrm instrinsic safely + int32_t count = 0; + if ((A[0] == 0) || (B[0] == 0)) { + if ((A[0] == 0) && (B[0] == 0)) { + A++; + s_a--; + B++; + s_b--; + } else if (A[0] == 0) { + C[count++] = 0; + A++; + s_a--; + } else { + B++; + s_b--; + } + } + // at this point, we have two non-empty arrays, made of non-zero + // increasing values. + size_t i_a = 0, i_b = 0; + const size_t vectorlength = sizeof(__m128i) / sizeof(uint16_t); + const size_t st_a = (s_a / vectorlength) * vectorlength; + const size_t st_b = (s_b / vectorlength) * vectorlength; + if ((i_a < st_a) && (i_b < st_b)) { // this is the vectorized code path + __m128i v_a, v_b; //, v_bmax; + // we load a vector from A and a vector from B + v_a = _mm_lddqu_si128((__m128i *)&A[i_a]); + v_b = _mm_lddqu_si128((__m128i *)&B[i_b]); + // we have a runningmask which indicates which values from A have been + // spotted in B, these don't get written out. + __m128i runningmask_a_found_in_b = _mm_setzero_si128(); + /**** + * start of the main vectorized loop + *****/ + while (true) { + // afoundinb will contain a mask indicate for each entry in A + // whether it is seen + // in B + const __m128i a_found_in_b = _mm_cmpistrm( + v_b, v_a, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + runningmask_a_found_in_b = + _mm_or_si128(runningmask_a_found_in_b, a_found_in_b); + // we always compare the last values of A and B + const uint16_t a_max = A[i_a + vectorlength - 1]; + const uint16_t b_max = B[i_b + vectorlength - 1]; + if (a_max <= b_max) { + // Ok. In this code path, we are ready to write our v_a + // because there is no need to read more from B, they will + // all be large values. + const int bitmask_belongs_to_difference = + _mm_extract_epi32(runningmask_a_found_in_b, 0) ^ 0xFF; + /*** next few lines are probably expensive *****/ + __m128i sm16 = _mm_loadu_si128((const __m128i *)shuffle_mask16 + + bitmask_belongs_to_difference); + __m128i p = _mm_shuffle_epi8(v_a, sm16); + _mm_storeu_si128((__m128i *)&C[count], p); // can overflow + count += _mm_popcnt_u32(bitmask_belongs_to_difference); + // we advance a + i_a += vectorlength; + if (i_a == st_a) // no more + break; + runningmask_a_found_in_b = _mm_setzero_si128(); + v_a = _mm_lddqu_si128((__m128i *)&A[i_a]); + } + if (b_max <= a_max) { + // in this code path, the current v_b has become useless + i_b += vectorlength; + if (i_b == st_b) break; + v_b = _mm_lddqu_si128((__m128i *)&B[i_b]); + } + } + // at this point, either we have i_a == st_a, which is the end of the + // vectorized processing, + // or we have i_b == st_b, and we are not done processing the vector... + // so we need to finish it off. + if (i_a < st_a) { // we have unfinished business... + uint16_t buffer[8]; // buffer to do a masked load + memset(buffer, 0, 8 * sizeof(uint16_t)); + memcpy(buffer, B + i_b, (s_b - i_b) * sizeof(uint16_t)); + v_b = _mm_lddqu_si128((__m128i *)buffer); + const __m128i a_found_in_b = _mm_cmpistrm( + v_b, v_a, + _SIDD_UWORD_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_BIT_MASK); + runningmask_a_found_in_b = + _mm_or_si128(runningmask_a_found_in_b, a_found_in_b); + const int bitmask_belongs_to_difference = + _mm_extract_epi32(runningmask_a_found_in_b, 0) ^ 0xFF; + __m128i sm16 = _mm_loadu_si128((const __m128i *)shuffle_mask16 + + bitmask_belongs_to_difference); + __m128i p = _mm_shuffle_epi8(v_a, sm16); + _mm_storeu_si128((__m128i *)&C[count], p); // can overflow + count += _mm_popcnt_u32(bitmask_belongs_to_difference); + i_a += vectorlength; + } + // at this point we should have i_a == st_a and i_b == st_b + } + // do the tail using scalar code + while (i_a < s_a && i_b < s_b) { + uint16_t a = A[i_a]; + uint16_t b = B[i_b]; + if (b < a) { + i_b++; + } else if (a < b) { + C[count] = a; + count++; + i_a++; + } else { //== + i_a++; + i_b++; + } + } + if (i_a < s_a) { + if (C == A) { + assert((size_t)count <= i_a); + if ((size_t)count < i_a) { + memmove(C + count, A + i_a, sizeof(uint16_t) * (s_a - i_a)); + } + } else { + for (size_t i = 0; i < (s_a - i_a); i++) { + C[count + i] = A[i + i_a]; + } + } + count += (int32_t)(s_a - i_a); + } + return count; +} +CROARING_UNTARGET_AVX2 +#endif // CROARING_IS_X64 + +/** + * Branchless binary search going after 4 values at once. + * Assumes that array is sorted. + * You have that array[*index1] >= target1, array[*index12] >= target2, ... + * except when *index1 = n, in which case you know that all values in array are + * smaller than target1, and so forth. + * It has logarithmic complexity. + */ +static void binarySearch4(const uint16_t *array, int32_t n, uint16_t target1, + uint16_t target2, uint16_t target3, uint16_t target4, + int32_t *index1, int32_t *index2, int32_t *index3, + int32_t *index4) { + const uint16_t *base1 = array; + const uint16_t *base2 = array; + const uint16_t *base3 = array; + const uint16_t *base4 = array; + if (n == 0) return; + while (n > 1) { + int32_t half = n >> 1; + base1 = (base1[half] < target1) ? &base1[half] : base1; + base2 = (base2[half] < target2) ? &base2[half] : base2; + base3 = (base3[half] < target3) ? &base3[half] : base3; + base4 = (base4[half] < target4) ? &base4[half] : base4; + n -= half; + } + *index1 = (int32_t)((*base1 < target1) + base1 - array); + *index2 = (int32_t)((*base2 < target2) + base2 - array); + *index3 = (int32_t)((*base3 < target3) + base3 - array); + *index4 = (int32_t)((*base4 < target4) + base4 - array); +} + +/** + * Branchless binary search going after 2 values at once. + * Assumes that array is sorted. + * You have that array[*index1] >= target1, array[*index12] >= target2. + * except when *index1 = n, in which case you know that all values in array are + * smaller than target1, and so forth. + * It has logarithmic complexity. + */ +static void binarySearch2(const uint16_t *array, int32_t n, uint16_t target1, + uint16_t target2, int32_t *index1, int32_t *index2) { + const uint16_t *base1 = array; + const uint16_t *base2 = array; + if (n == 0) return; + while (n > 1) { + int32_t half = n >> 1; + base1 = (base1[half] < target1) ? &base1[half] : base1; + base2 = (base2[half] < target2) ? &base2[half] : base2; + n -= half; + } + *index1 = (int32_t)((*base1 < target1) + base1 - array); + *index2 = (int32_t)((*base2 < target2) + base2 - array); +} + +/* Computes the intersection between one small and one large set of uint16_t. + * Stores the result into buffer and return the number of elements. + * Processes the small set in blocks of 4 values calling binarySearch4 + * and binarySearch2. This approach can be slightly superior to a conventional + * galloping search in some instances. + */ +int32_t intersect_skewed_uint16(const uint16_t *small, size_t size_s, + const uint16_t *large, size_t size_l, + uint16_t *buffer) { + size_t pos = 0, idx_l = 0, idx_s = 0; + + if (0 == size_s) { + return 0; + } + int32_t index1 = 0, index2 = 0, index3 = 0, index4 = 0; + while ((idx_s + 4 <= size_s) && (idx_l < size_l)) { + uint16_t target1 = small[idx_s]; + uint16_t target2 = small[idx_s + 1]; + uint16_t target3 = small[idx_s + 2]; + uint16_t target4 = small[idx_s + 3]; + binarySearch4(large + idx_l, (int32_t)(size_l - idx_l), target1, + target2, target3, target4, &index1, &index2, &index3, + &index4); + if ((index1 + idx_l < size_l) && (large[idx_l + index1] == target1)) { + buffer[pos++] = target1; + } + if ((index2 + idx_l < size_l) && (large[idx_l + index2] == target2)) { + buffer[pos++] = target2; + } + if ((index3 + idx_l < size_l) && (large[idx_l + index3] == target3)) { + buffer[pos++] = target3; + } + if ((index4 + idx_l < size_l) && (large[idx_l + index4] == target4)) { + buffer[pos++] = target4; + } + idx_s += 4; + idx_l += index4; + } + if ((idx_s + 2 <= size_s) && (idx_l < size_l)) { + uint16_t target1 = small[idx_s]; + uint16_t target2 = small[idx_s + 1]; + binarySearch2(large + idx_l, (int32_t)(size_l - idx_l), target1, + target2, &index1, &index2); + if ((index1 + idx_l < size_l) && (large[idx_l + index1] == target1)) { + buffer[pos++] = target1; + } + if ((index2 + idx_l < size_l) && (large[idx_l + index2] == target2)) { + buffer[pos++] = target2; + } + idx_s += 2; + idx_l += index2; + } + if ((idx_s < size_s) && (idx_l < size_l)) { + uint16_t val_s = small[idx_s]; + int32_t index = + binarySearch(large + idx_l, (int32_t)(size_l - idx_l), val_s); + if (index >= 0) buffer[pos++] = val_s; + } + return (int32_t)pos; +} + +// TODO: this could be accelerated, possibly, by using binarySearch4 as above. +int32_t intersect_skewed_uint16_cardinality(const uint16_t *small, + size_t size_s, + const uint16_t *large, + size_t size_l) { + size_t pos = 0, idx_l = 0, idx_s = 0; + + if (0 == size_s) { + return 0; + } + + uint16_t val_l = large[idx_l], val_s = small[idx_s]; + + while (true) { + if (val_l < val_s) { + idx_l = advanceUntil(large, (int32_t)idx_l, (int32_t)size_l, val_s); + if (idx_l == size_l) break; + val_l = large[idx_l]; + } else if (val_s < val_l) { + idx_s++; + if (idx_s == size_s) break; + val_s = small[idx_s]; + } else { + pos++; + idx_s++; + if (idx_s == size_s) break; + val_s = small[idx_s]; + idx_l = advanceUntil(large, (int32_t)idx_l, (int32_t)size_l, val_s); + if (idx_l == size_l) break; + val_l = large[idx_l]; + } + } + + return (int32_t)pos; +} + +bool intersect_skewed_uint16_nonempty(const uint16_t *small, size_t size_s, + const uint16_t *large, size_t size_l) { + size_t idx_l = 0, idx_s = 0; + + if (0 == size_s) { + return false; + } + + uint16_t val_l = large[idx_l], val_s = small[idx_s]; + + while (true) { + if (val_l < val_s) { + idx_l = advanceUntil(large, (int32_t)idx_l, (int32_t)size_l, val_s); + if (idx_l == size_l) break; + val_l = large[idx_l]; + } else if (val_s < val_l) { + idx_s++; + if (idx_s == size_s) break; + val_s = small[idx_s]; + } else { + return true; + } + } + + return false; +} + +/** + * Generic intersection function. + */ +int32_t intersect_uint16(const uint16_t *A, const size_t lenA, + const uint16_t *B, const size_t lenB, uint16_t *out) { + const uint16_t *initout = out; + if (lenA == 0 || lenB == 0) return 0; + const uint16_t *endA = A + lenA; + const uint16_t *endB = B + lenB; + + while (1) { + while (*A < *B) { + SKIP_FIRST_COMPARE: + if (++A == endA) return (int32_t)(out - initout); + } + while (*A > *B) { + if (++B == endB) return (int32_t)(out - initout); + } + if (*A == *B) { + *out++ = *A; + if (++A == endA || ++B == endB) return (int32_t)(out - initout); + } else { + goto SKIP_FIRST_COMPARE; + } + } + // return (int32_t)(out - initout); // NOTREACHED +} + +int32_t intersect_uint16_cardinality(const uint16_t *A, const size_t lenA, + const uint16_t *B, const size_t lenB) { + int32_t answer = 0; + if (lenA == 0 || lenB == 0) return 0; + const uint16_t *endA = A + lenA; + const uint16_t *endB = B + lenB; + + while (1) { + while (*A < *B) { + SKIP_FIRST_COMPARE: + if (++A == endA) return answer; + } + while (*A > *B) { + if (++B == endB) return answer; + } + if (*A == *B) { + ++answer; + if (++A == endA || ++B == endB) return answer; + } else { + goto SKIP_FIRST_COMPARE; + } + } + // return answer; // NOTREACHED +} + +bool intersect_uint16_nonempty(const uint16_t *A, const size_t lenA, + const uint16_t *B, const size_t lenB) { + if (lenA == 0 || lenB == 0) return 0; + const uint16_t *endA = A + lenA; + const uint16_t *endB = B + lenB; + + while (1) { + while (*A < *B) { + SKIP_FIRST_COMPARE: + if (++A == endA) return false; + } + while (*A > *B) { + if (++B == endB) return false; + } + if (*A == *B) { + return true; + } else { + goto SKIP_FIRST_COMPARE; + } + } + return false; // NOTREACHED +} + +/** + * Generic intersection function. + */ +size_t intersection_uint32(const uint32_t *A, const size_t lenA, + const uint32_t *B, const size_t lenB, + uint32_t *out) { + const uint32_t *initout = out; + if (lenA == 0 || lenB == 0) return 0; + const uint32_t *endA = A + lenA; + const uint32_t *endB = B + lenB; + + while (1) { + while (*A < *B) { + SKIP_FIRST_COMPARE: + if (++A == endA) return (out - initout); + } + while (*A > *B) { + if (++B == endB) return (out - initout); + } + if (*A == *B) { + *out++ = *A; + if (++A == endA || ++B == endB) return (out - initout); + } else { + goto SKIP_FIRST_COMPARE; + } + } + // return (out - initout); // NOTREACHED +} + +size_t intersection_uint32_card(const uint32_t *A, const size_t lenA, + const uint32_t *B, const size_t lenB) { + if (lenA == 0 || lenB == 0) return 0; + size_t card = 0; + const uint32_t *endA = A + lenA; + const uint32_t *endB = B + lenB; + + while (1) { + while (*A < *B) { + SKIP_FIRST_COMPARE: + if (++A == endA) return card; + } + while (*A > *B) { + if (++B == endB) return card; + } + if (*A == *B) { + card++; + if (++A == endA || ++B == endB) return card; + } else { + goto SKIP_FIRST_COMPARE; + } + } + // return card; // NOTREACHED +} + +// can one vectorize the computation of the union? (Update: Yes! See +// union_vector16). + +size_t union_uint16(const uint16_t *set_1, size_t size_1, const uint16_t *set_2, + size_t size_2, uint16_t *buffer) { + size_t pos = 0, idx_1 = 0, idx_2 = 0; + + if (0 == size_2) { + memmove(buffer, set_1, size_1 * sizeof(uint16_t)); + return size_1; + } + if (0 == size_1) { + memmove(buffer, set_2, size_2 * sizeof(uint16_t)); + return size_2; + } + + uint16_t val_1 = set_1[idx_1], val_2 = set_2[idx_2]; + + while (true) { + if (val_1 < val_2) { + buffer[pos++] = val_1; + ++idx_1; + if (idx_1 >= size_1) break; + val_1 = set_1[idx_1]; + } else if (val_2 < val_1) { + buffer[pos++] = val_2; + ++idx_2; + if (idx_2 >= size_2) break; + val_2 = set_2[idx_2]; + } else { + buffer[pos++] = val_1; + ++idx_1; + ++idx_2; + if (idx_1 >= size_1 || idx_2 >= size_2) break; + val_1 = set_1[idx_1]; + val_2 = set_2[idx_2]; + } + } + + if (idx_1 < size_1) { + const size_t n_elems = size_1 - idx_1; + memmove(buffer + pos, set_1 + idx_1, n_elems * sizeof(uint16_t)); + pos += n_elems; + } else if (idx_2 < size_2) { + const size_t n_elems = size_2 - idx_2; + memmove(buffer + pos, set_2 + idx_2, n_elems * sizeof(uint16_t)); + pos += n_elems; + } + + return pos; +} + +int difference_uint16(const uint16_t *a1, int length1, const uint16_t *a2, + int length2, uint16_t *a_out) { + int out_card = 0; + int k1 = 0, k2 = 0; + if (length1 == 0) return 0; + if (length2 == 0) { + if (a1 != a_out) memcpy(a_out, a1, sizeof(uint16_t) * length1); + return length1; + } + uint16_t s1 = a1[k1]; + uint16_t s2 = a2[k2]; + while (true) { + if (s1 < s2) { + a_out[out_card++] = s1; + ++k1; + if (k1 >= length1) { + break; + } + s1 = a1[k1]; + } else if (s1 == s2) { + ++k1; + ++k2; + if (k1 >= length1) { + break; + } + if (k2 >= length2) { + memmove(a_out + out_card, a1 + k1, + sizeof(uint16_t) * (length1 - k1)); + return out_card + length1 - k1; + } + s1 = a1[k1]; + s2 = a2[k2]; + } else { // if (val1>val2) + ++k2; + if (k2 >= length2) { + memmove(a_out + out_card, a1 + k1, + sizeof(uint16_t) * (length1 - k1)); + return out_card + length1 - k1; + } + s2 = a2[k2]; + } + } + return out_card; +} + +int32_t xor_uint16(const uint16_t *array_1, int32_t card_1, + const uint16_t *array_2, int32_t card_2, uint16_t *out) { + int32_t pos1 = 0, pos2 = 0, pos_out = 0; + while (pos1 < card_1 && pos2 < card_2) { + const uint16_t v1 = array_1[pos1]; + const uint16_t v2 = array_2[pos2]; + if (v1 == v2) { + ++pos1; + ++pos2; + continue; + } + if (v1 < v2) { + out[pos_out++] = v1; + ++pos1; + } else { + out[pos_out++] = v2; + ++pos2; + } + } + if (pos1 < card_1) { + const size_t n_elems = card_1 - pos1; + memcpy(out + pos_out, array_1 + pos1, n_elems * sizeof(uint16_t)); + pos_out += (int32_t)n_elems; + } else if (pos2 < card_2) { + const size_t n_elems = card_2 - pos2; + memcpy(out + pos_out, array_2 + pos2, n_elems * sizeof(uint16_t)); + pos_out += (int32_t)n_elems; + } + return pos_out; +} + +#if CROARING_IS_X64 + +/*** + * start of the SIMD 16-bit union code + * + */ +CROARING_TARGET_AVX2 + +// Assuming that vInput1 and vInput2 are sorted, produces a sorted output going +// from vecMin all the way to vecMax +// developed originally for merge sort using SIMD instructions. +// Standard merge. See, e.g., Inoue and Taura, SIMD- and Cache-Friendly +// Algorithm for Sorting an Array of Structures +static inline void sse_merge(const __m128i *vInput1, + const __m128i *vInput2, // input 1 & 2 + __m128i *vecMin, __m128i *vecMax) { // output + __m128i vecTmp; + vecTmp = _mm_min_epu16(*vInput1, *vInput2); + *vecMax = _mm_max_epu16(*vInput1, *vInput2); + vecTmp = _mm_alignr_epi8(vecTmp, vecTmp, 2); + *vecMin = _mm_min_epu16(vecTmp, *vecMax); + *vecMax = _mm_max_epu16(vecTmp, *vecMax); + vecTmp = _mm_alignr_epi8(*vecMin, *vecMin, 2); + *vecMin = _mm_min_epu16(vecTmp, *vecMax); + *vecMax = _mm_max_epu16(vecTmp, *vecMax); + vecTmp = _mm_alignr_epi8(*vecMin, *vecMin, 2); + *vecMin = _mm_min_epu16(vecTmp, *vecMax); + *vecMax = _mm_max_epu16(vecTmp, *vecMax); + vecTmp = _mm_alignr_epi8(*vecMin, *vecMin, 2); + *vecMin = _mm_min_epu16(vecTmp, *vecMax); + *vecMax = _mm_max_epu16(vecTmp, *vecMax); + vecTmp = _mm_alignr_epi8(*vecMin, *vecMin, 2); + *vecMin = _mm_min_epu16(vecTmp, *vecMax); + *vecMax = _mm_max_epu16(vecTmp, *vecMax); + vecTmp = _mm_alignr_epi8(*vecMin, *vecMin, 2); + *vecMin = _mm_min_epu16(vecTmp, *vecMax); + *vecMax = _mm_max_epu16(vecTmp, *vecMax); + vecTmp = _mm_alignr_epi8(*vecMin, *vecMin, 2); + *vecMin = _mm_min_epu16(vecTmp, *vecMax); + *vecMax = _mm_max_epu16(vecTmp, *vecMax); + *vecMin = _mm_alignr_epi8(*vecMin, *vecMin, 2); +} +CROARING_UNTARGET_AVX2 +// used by store_unique, generated by simdunion.py +static uint8_t uniqshuf[] = { + 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, + 0xc, 0xd, 0xe, 0xf, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, + 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, + 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, + 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x6, 0x7, 0x8, 0x9, + 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0x2, 0x3, 0x6, 0x7, + 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x4, 0x5, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, + 0x2, 0x3, 0x4, 0x5, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, 0x8, 0x9, 0xa, 0xb, + 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x8, 0x9, + 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x8, 0x9, + 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, + 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, + 0x6, 0x7, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x4, 0x5, 0x6, 0x7, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x6, 0x7, 0xa, 0xb, 0xc, 0xd, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x6, 0x7, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x6, 0x7, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x6, 0x7, 0xa, 0xb, 0xc, 0xd, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0x7, 0xa, 0xb, + 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, 0xa, 0xb, 0xc, 0xd, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, + 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x4, 0x5, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0xa, 0xb, 0xc, 0xd, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0xa, 0xb, + 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, + 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xc, 0xd, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, + 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x6, 0x7, + 0x8, 0x9, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x6, 0x7, 0x8, 0x9, 0xc, 0xd, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x6, 0x7, 0x8, 0x9, 0xc, 0xd, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x6, 0x7, + 0x8, 0x9, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x6, 0x7, 0x8, 0x9, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x8, 0x9, + 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, + 0x8, 0x9, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x4, 0x5, 0x8, 0x9, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x8, 0x9, 0xc, 0xd, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x8, 0x9, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x8, 0x9, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x8, 0x9, 0xc, 0xd, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x8, 0x9, 0xc, 0xd, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0xc, 0xd, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0xc, 0xd, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, + 0x6, 0x7, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x4, 0x5, 0x6, 0x7, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x6, 0x7, 0xc, 0xd, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x6, 0x7, + 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x6, 0x7, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0x7, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x4, 0x5, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x4, 0x5, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, 0xc, 0xd, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0xc, 0xd, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0xc, 0xd, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xc, 0xd, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, + 0x8, 0x9, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, + 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0x7, 0x8, 0x9, + 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x8, 0x9, 0xa, 0xb, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, 0x8, 0x9, 0xa, 0xb, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, + 0x8, 0x9, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x4, 0x5, 0x8, 0x9, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x8, 0x9, 0xa, 0xb, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x8, 0x9, + 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x8, 0x9, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x8, 0x9, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x4, 0x5, 0x6, 0x7, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, 0x6, 0x7, 0xa, 0xb, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x6, 0x7, + 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x6, 0x7, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x6, 0x7, 0xa, 0xb, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x6, 0x7, + 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x6, 0x7, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0xa, 0xb, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, + 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x4, 0x5, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0xa, 0xb, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xa, 0xb, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, + 0x6, 0x7, 0x8, 0x9, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x6, 0x7, 0x8, 0x9, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x6, 0x7, + 0x8, 0x9, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x6, 0x7, 0x8, 0x9, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0x7, 0x8, 0x9, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x4, 0x5, 0x8, 0x9, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x4, 0x5, 0x8, 0x9, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, 0x8, 0x9, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x8, 0x9, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x8, 0x9, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x8, 0x9, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x8, 0x9, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x8, 0x9, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, + 0x6, 0x7, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x4, 0x5, 0x6, 0x7, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x6, 0x7, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x6, 0x7, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x6, 0x7, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x6, 0x7, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0x7, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, + 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x4, 0x5, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0xe, 0xf, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0xe, 0xf, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xe, 0xf, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, + 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, + 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x6, 0x7, + 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, + 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x6, 0x7, + 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x8, 0x9, + 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, + 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x4, 0x5, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x8, 0x9, 0xa, 0xb, + 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0xa, 0xb, 0xc, 0xd, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0xa, 0xb, + 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, + 0x6, 0x7, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x4, 0x5, 0x6, 0x7, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x6, 0x7, 0xa, 0xb, + 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x6, 0x7, + 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x6, 0x7, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0x7, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x4, 0x5, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x4, 0x5, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, 0xa, 0xb, 0xc, 0xd, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0xa, 0xb, + 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0xa, 0xb, + 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xa, 0xb, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, + 0x8, 0x9, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, + 0x6, 0x7, 0x8, 0x9, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xc, 0xd, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xc, 0xd, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x6, 0x7, 0x8, 0x9, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x6, 0x7, 0x8, 0x9, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x6, 0x7, 0x8, 0x9, 0xc, 0xd, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0x7, 0x8, 0x9, + 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x8, 0x9, 0xc, 0xd, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, 0x8, 0x9, 0xc, 0xd, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, + 0x8, 0x9, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x4, 0x5, 0x8, 0x9, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x8, 0x9, 0xc, 0xd, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x8, 0x9, + 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x8, 0x9, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x8, 0x9, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x4, 0x5, 0x6, 0x7, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, 0x6, 0x7, 0xc, 0xd, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x6, 0x7, + 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x6, 0x7, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x6, 0x7, 0xc, 0xd, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x6, 0x7, + 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x6, 0x7, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0xc, 0xd, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, + 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x4, 0x5, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0xc, 0xd, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xc, 0xd, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, + 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, + 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x6, 0x7, 0x8, 0x9, + 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x6, 0x7, + 0x8, 0x9, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x4, 0x5, 0x8, 0x9, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x4, 0x5, 0x8, 0x9, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, 0x8, 0x9, 0xa, 0xb, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x8, 0x9, + 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x8, 0x9, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x8, 0x9, 0xa, 0xb, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x8, 0x9, + 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x8, 0x9, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, + 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, + 0x6, 0x7, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x4, 0x5, 0x6, 0x7, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x6, 0x7, 0xa, 0xb, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x6, 0x7, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x6, 0x7, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x6, 0x7, 0xa, 0xb, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0x7, 0xa, 0xb, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, 0xa, 0xb, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, + 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x4, 0x5, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0xa, 0xb, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0xa, 0xb, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xa, 0xb, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x6, 0x7, + 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x6, 0x7, 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x6, 0x7, 0x8, 0x9, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x6, 0x7, + 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x6, 0x7, 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x8, 0x9, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, + 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x4, 0x5, 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x8, 0x9, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x8, 0x9, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, + 0x6, 0x7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x4, 0x5, 0x6, 0x7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, 0x6, 0x7, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0x6, 0x7, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x6, 0x7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x6, 0x7, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x2, 0x3, + 0x4, 0x5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x2, 0x3, 0x4, 0x5, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0x4, 0x5, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x4, 0x5, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x0, 0x1, 0x2, 0x3, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0x2, 0x3, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0, 0x1, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF}; +CROARING_TARGET_AVX2 +// write vector new, while omitting repeated values assuming that previously +// written vector was "old" +static inline int store_unique(__m128i old, __m128i newval, uint16_t *output) { + __m128i vecTmp = _mm_alignr_epi8(newval, old, 16 - 2); + // lots of high latency instructions follow (optimize?) + int M = _mm_movemask_epi8( + _mm_packs_epi16(_mm_cmpeq_epi16(vecTmp, newval), _mm_setzero_si128())); + int numberofnewvalues = 8 - _mm_popcnt_u32(M); + __m128i key = _mm_lddqu_si128((const __m128i *)uniqshuf + M); + __m128i val = _mm_shuffle_epi8(newval, key); + _mm_storeu_si128((__m128i *)output, val); + return numberofnewvalues; +} +CROARING_UNTARGET_AVX2 + +// working in-place, this function overwrites the repeated values +// could be avoided? +static inline uint32_t unique(uint16_t *out, uint32_t len) { + uint32_t pos = 1; + for (uint32_t i = 1; i < len; ++i) { + if (out[i] != out[i - 1]) { + out[pos++] = out[i]; + } + } + return pos; +} + +// use with qsort, could be avoided +static int uint16_compare(const void *a, const void *b) { + return (*(uint16_t *)a - *(uint16_t *)b); +} + +CROARING_TARGET_AVX2 +// a one-pass SSE union algorithm +// This function may not be safe if array1 == output or array2 == output. +uint32_t union_vector16(const uint16_t *__restrict__ array1, uint32_t length1, + const uint16_t *__restrict__ array2, uint32_t length2, + uint16_t *__restrict__ output) { + if ((length1 < 8) || (length2 < 8)) { + return (uint32_t)union_uint16(array1, length1, array2, length2, output); + } + __m128i vA, vB, V, vecMin, vecMax; + __m128i laststore; + uint16_t *initoutput = output; + uint32_t len1 = length1 / 8; + uint32_t len2 = length2 / 8; + uint32_t pos1 = 0; + uint32_t pos2 = 0; + // we start the machine + vA = _mm_lddqu_si128((const __m128i *)array1 + pos1); + pos1++; + vB = _mm_lddqu_si128((const __m128i *)array2 + pos2); + pos2++; + sse_merge(&vA, &vB, &vecMin, &vecMax); + laststore = _mm_set1_epi16(-1); + output += store_unique(laststore, vecMin, output); + laststore = vecMin; + if ((pos1 < len1) && (pos2 < len2)) { + uint16_t curA, curB; + curA = array1[8 * pos1]; + curB = array2[8 * pos2]; + while (true) { + if (curA <= curB) { + V = _mm_lddqu_si128((const __m128i *)array1 + pos1); + pos1++; + if (pos1 < len1) { + curA = array1[8 * pos1]; + } else { + break; + } + } else { + V = _mm_lddqu_si128((const __m128i *)array2 + pos2); + pos2++; + if (pos2 < len2) { + curB = array2[8 * pos2]; + } else { + break; + } + } + sse_merge(&V, &vecMax, &vecMin, &vecMax); + output += store_unique(laststore, vecMin, output); + laststore = vecMin; + } + sse_merge(&V, &vecMax, &vecMin, &vecMax); + output += store_unique(laststore, vecMin, output); + laststore = vecMin; + } + // we finish the rest off using a scalar algorithm + // could be improved? + // + // copy the small end on a tmp buffer + uint32_t len = (uint32_t)(output - initoutput); + uint16_t buffer[16]; + uint32_t leftoversize = store_unique(laststore, vecMax, buffer); + if (pos1 == len1) { + memcpy(buffer + leftoversize, array1 + 8 * pos1, + (length1 - 8 * len1) * sizeof(uint16_t)); + leftoversize += length1 - 8 * len1; + qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + + leftoversize = unique(buffer, leftoversize); + len += (uint32_t)union_uint16(buffer, leftoversize, array2 + 8 * pos2, + length2 - 8 * pos2, output); + } else { + memcpy(buffer + leftoversize, array2 + 8 * pos2, + (length2 - 8 * len2) * sizeof(uint16_t)); + leftoversize += length2 - 8 * len2; + qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + leftoversize = unique(buffer, leftoversize); + len += (uint32_t)union_uint16(buffer, leftoversize, array1 + 8 * pos1, + length1 - 8 * pos1, output); + } + return len; +} +CROARING_UNTARGET_AVX2 + +/** + * End of the SIMD 16-bit union code + * + */ + +/** + * Start of SIMD 16-bit XOR code + */ + +CROARING_TARGET_AVX2 +// write vector new, while omitting repeated values assuming that previously +// written vector was "old" +static inline int store_unique_xor(__m128i old, __m128i newval, + uint16_t *output) { + __m128i vecTmp1 = _mm_alignr_epi8(newval, old, 16 - 4); + __m128i vecTmp2 = _mm_alignr_epi8(newval, old, 16 - 2); + __m128i equalleft = _mm_cmpeq_epi16(vecTmp2, vecTmp1); + __m128i equalright = _mm_cmpeq_epi16(vecTmp2, newval); + __m128i equalleftoright = _mm_or_si128(equalleft, equalright); + int M = _mm_movemask_epi8( + _mm_packs_epi16(equalleftoright, _mm_setzero_si128())); + int numberofnewvalues = 8 - _mm_popcnt_u32(M); + __m128i key = _mm_lddqu_si128((const __m128i *)uniqshuf + M); + __m128i val = _mm_shuffle_epi8(vecTmp2, key); + _mm_storeu_si128((__m128i *)output, val); + return numberofnewvalues; +} +CROARING_UNTARGET_AVX2 + +// working in-place, this function overwrites the repeated values +// could be avoided? Warning: assumes len > 0 +static inline uint32_t unique_xor(uint16_t *out, uint32_t len) { + uint32_t pos = 1; + for (uint32_t i = 1; i < len; ++i) { + if (out[i] != out[i - 1]) { + out[pos++] = out[i]; + } else + pos--; // if it is identical to previous, delete it + } + return pos; +} +CROARING_TARGET_AVX2 +// a one-pass SSE xor algorithm +uint32_t xor_vector16(const uint16_t *__restrict__ array1, uint32_t length1, + const uint16_t *__restrict__ array2, uint32_t length2, + uint16_t *__restrict__ output) { + if ((length1 < 8) || (length2 < 8)) { + return xor_uint16(array1, length1, array2, length2, output); + } + __m128i vA, vB, V, vecMin, vecMax; + __m128i laststore; + uint16_t *initoutput = output; + uint32_t len1 = length1 / 8; + uint32_t len2 = length2 / 8; + uint32_t pos1 = 0; + uint32_t pos2 = 0; + // we start the machine + vA = _mm_lddqu_si128((const __m128i *)array1 + pos1); + pos1++; + vB = _mm_lddqu_si128((const __m128i *)array2 + pos2); + pos2++; + sse_merge(&vA, &vB, &vecMin, &vecMax); + laststore = _mm_set1_epi16(-1); + uint16_t buffer[17]; + output += store_unique_xor(laststore, vecMin, output); + + laststore = vecMin; + if ((pos1 < len1) && (pos2 < len2)) { + uint16_t curA, curB; + curA = array1[8 * pos1]; + curB = array2[8 * pos2]; + while (true) { + if (curA <= curB) { + V = _mm_lddqu_si128((const __m128i *)array1 + pos1); + pos1++; + if (pos1 < len1) { + curA = array1[8 * pos1]; + } else { + break; + } + } else { + V = _mm_lddqu_si128((const __m128i *)array2 + pos2); + pos2++; + if (pos2 < len2) { + curB = array2[8 * pos2]; + } else { + break; + } + } + sse_merge(&V, &vecMax, &vecMin, &vecMax); + // conditionally stores the last value of laststore as well as all + // but the + // last value of vecMin + output += store_unique_xor(laststore, vecMin, output); + laststore = vecMin; + } + sse_merge(&V, &vecMax, &vecMin, &vecMax); + // conditionally stores the last value of laststore as well as all but + // the + // last value of vecMin + output += store_unique_xor(laststore, vecMin, output); + laststore = vecMin; + } + uint32_t len = (uint32_t)(output - initoutput); + + // we finish the rest off using a scalar algorithm + // could be improved? + // conditionally stores the last value of laststore as well as all but the + // last value of vecMax, + // we store to "buffer" + int leftoversize = store_unique_xor(laststore, vecMax, buffer); + uint16_t vec7 = (uint16_t)_mm_extract_epi16(vecMax, 7); + uint16_t vec6 = (uint16_t)_mm_extract_epi16(vecMax, 6); + if (vec7 != vec6) buffer[leftoversize++] = vec7; + if (pos1 == len1) { + memcpy(buffer + leftoversize, array1 + 8 * pos1, + (length1 - 8 * len1) * sizeof(uint16_t)); + leftoversize += length1 - 8 * len1; + if (leftoversize == 0) { // trivial case + memcpy(output, array2 + 8 * pos2, + (length2 - 8 * pos2) * sizeof(uint16_t)); + len += (length2 - 8 * pos2); + } else { + qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + leftoversize = unique_xor(buffer, leftoversize); + len += xor_uint16(buffer, leftoversize, array2 + 8 * pos2, + length2 - 8 * pos2, output); + } + } else { + memcpy(buffer + leftoversize, array2 + 8 * pos2, + (length2 - 8 * len2) * sizeof(uint16_t)); + leftoversize += length2 - 8 * len2; + if (leftoversize == 0) { // trivial case + memcpy(output, array1 + 8 * pos1, + (length1 - 8 * pos1) * sizeof(uint16_t)); + len += (length1 - 8 * pos1); + } else { + qsort(buffer, leftoversize, sizeof(uint16_t), uint16_compare); + leftoversize = unique_xor(buffer, leftoversize); + len += xor_uint16(buffer, leftoversize, array1 + 8 * pos1, + length1 - 8 * pos1, output); + } + } + return len; +} +CROARING_UNTARGET_AVX2 +/** + * End of SIMD 16-bit XOR code + */ + +#endif // CROARING_IS_X64 + +size_t union_uint32(const uint32_t *set_1, size_t size_1, const uint32_t *set_2, + size_t size_2, uint32_t *buffer) { + size_t pos = 0, idx_1 = 0, idx_2 = 0; + + if (0 == size_2) { + memmove(buffer, set_1, size_1 * sizeof(uint32_t)); + return size_1; + } + if (0 == size_1) { + memmove(buffer, set_2, size_2 * sizeof(uint32_t)); + return size_2; + } + + uint32_t val_1 = set_1[idx_1], val_2 = set_2[idx_2]; + + while (true) { + if (val_1 < val_2) { + buffer[pos++] = val_1; + ++idx_1; + if (idx_1 >= size_1) break; + val_1 = set_1[idx_1]; + } else if (val_2 < val_1) { + buffer[pos++] = val_2; + ++idx_2; + if (idx_2 >= size_2) break; + val_2 = set_2[idx_2]; + } else { + buffer[pos++] = val_1; + ++idx_1; + ++idx_2; + if (idx_1 >= size_1 || idx_2 >= size_2) break; + val_1 = set_1[idx_1]; + val_2 = set_2[idx_2]; + } + } + + if (idx_1 < size_1) { + const size_t n_elems = size_1 - idx_1; + memmove(buffer + pos, set_1 + idx_1, n_elems * sizeof(uint32_t)); + pos += n_elems; + } else if (idx_2 < size_2) { + const size_t n_elems = size_2 - idx_2; + memmove(buffer + pos, set_2 + idx_2, n_elems * sizeof(uint32_t)); + pos += n_elems; + } + + return pos; +} + +size_t union_uint32_card(const uint32_t *set_1, size_t size_1, + const uint32_t *set_2, size_t size_2) { + size_t pos = 0, idx_1 = 0, idx_2 = 0; + + if (0 == size_2) { + return size_1; + } + if (0 == size_1) { + return size_2; + } + + uint32_t val_1 = set_1[idx_1], val_2 = set_2[idx_2]; + + while (true) { + if (val_1 < val_2) { + ++idx_1; + ++pos; + if (idx_1 >= size_1) break; + val_1 = set_1[idx_1]; + } else if (val_2 < val_1) { + ++idx_2; + ++pos; + if (idx_2 >= size_2) break; + val_2 = set_2[idx_2]; + } else { + ++idx_1; + ++idx_2; + ++pos; + if (idx_1 >= size_1 || idx_2 >= size_2) break; + val_1 = set_1[idx_1]; + val_2 = set_2[idx_2]; + } + } + + if (idx_1 < size_1) { + const size_t n_elems = size_1 - idx_1; + pos += n_elems; + } else if (idx_2 < size_2) { + const size_t n_elems = size_2 - idx_2; + pos += n_elems; + } + return pos; +} + +size_t fast_union_uint16(const uint16_t *set_1, size_t size_1, + const uint16_t *set_2, size_t size_2, + uint16_t *buffer) { +#if CROARING_IS_X64 + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + // compute union with smallest array first + if (size_1 < size_2) { + return union_vector16(set_1, (uint32_t)size_1, set_2, + (uint32_t)size_2, buffer); + } else { + return union_vector16(set_2, (uint32_t)size_2, set_1, + (uint32_t)size_1, buffer); + } + } else { + // compute union with smallest array first + if (size_1 < size_2) { + return union_uint16(set_1, size_1, set_2, size_2, buffer); + } else { + return union_uint16(set_2, size_2, set_1, size_1, buffer); + } + } +#else + // compute union with smallest array first + if (size_1 < size_2) { + return union_uint16(set_1, size_1, set_2, size_2, buffer); + } else { + return union_uint16(set_2, size_2, set_1, size_1, buffer); + } +#endif +} +#if CROARING_IS_X64 +#if CROARING_COMPILER_SUPPORTS_AVX512 +CROARING_TARGET_AVX512 +static inline bool _avx512_memequals(const void *s1, const void *s2, size_t n) { + const uint8_t *ptr1 = (const uint8_t *)s1; + const uint8_t *ptr2 = (const uint8_t *)s2; + const uint8_t *end1 = ptr1 + n; + const uint8_t *end8 = ptr1 + ((n >> 3) << 3); + const uint8_t *end32 = ptr1 + ((n >> 5) << 5); + const uint8_t *end64 = ptr1 + ((n >> 6) << 6); + + while (ptr1 < end64) { + __m512i r1 = _mm512_loadu_si512((const __m512i *)ptr1); + __m512i r2 = _mm512_loadu_si512((const __m512i *)ptr2); + + uint64_t mask = _mm512_cmpeq_epi8_mask(r1, r2); + + if (mask != UINT64_MAX) { + return false; + } + + ptr1 += 64; + ptr2 += 64; + } + + while (ptr1 < end32) { + __m256i r1 = _mm256_loadu_si256((const __m256i *)ptr1); + __m256i r2 = _mm256_loadu_si256((const __m256i *)ptr2); + int mask = _mm256_movemask_epi8(_mm256_cmpeq_epi8(r1, r2)); + if ((uint32_t)mask != UINT32_MAX) { + return false; + } + ptr1 += 32; + ptr2 += 32; + } + + while (ptr1 < end8) { + uint64_t v1, v2; + memcpy(&v1, ptr1, sizeof(uint64_t)); + memcpy(&v2, ptr2, sizeof(uint64_t)); + if (v1 != v2) { + return false; + } + ptr1 += 8; + ptr2 += 8; + } + + while (ptr1 < end1) { + if (*ptr1 != *ptr2) { + return false; + } + ptr1++; + ptr2++; + } + + return true; +} +CROARING_UNTARGET_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + +CROARING_TARGET_AVX2 +static inline bool _avx2_memequals(const void *s1, const void *s2, size_t n) { + const uint8_t *ptr1 = (const uint8_t *)s1; + const uint8_t *ptr2 = (const uint8_t *)s2; + const uint8_t *end1 = ptr1 + n; + const uint8_t *end8 = ptr1 + n / 8 * 8; + const uint8_t *end32 = ptr1 + n / 32 * 32; + + while (ptr1 < end32) { + __m256i r1 = _mm256_loadu_si256((const __m256i *)ptr1); + __m256i r2 = _mm256_loadu_si256((const __m256i *)ptr2); + int mask = _mm256_movemask_epi8(_mm256_cmpeq_epi8(r1, r2)); + if ((uint32_t)mask != UINT32_MAX) { + return false; + } + ptr1 += 32; + ptr2 += 32; + } + + while (ptr1 < end8) { + uint64_t v1, v2; + memcpy(&v1, ptr1, sizeof(uint64_t)); + memcpy(&v2, ptr2, sizeof(uint64_t)); + if (v1 != v2) { + return false; + } + ptr1 += 8; + ptr2 += 8; + } + + while (ptr1 < end1) { + if (*ptr1 != *ptr2) { + return false; + } + ptr1++; + ptr2++; + } + + return true; +} +CROARING_UNTARGET_AVX2 +#endif + +bool memequals(const void *s1, const void *s2, size_t n) { + if (n == 0) { + return true; + } +#if CROARING_IS_X64 + int support = croaring_hardware_support(); +#if CROARING_COMPILER_SUPPORTS_AVX512 + if (support & ROARING_SUPPORTS_AVX512) { + return _avx512_memequals(s1, s2, n); + } else +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + if (support & ROARING_SUPPORTS_AVX2) { + return _avx2_memequals(s1, s2, n); + } else { + return memcmp(s1, s2, n) == 0; + } +#else + return memcmp(s1, s2, n) == 0; +#endif +} + +#if CROARING_IS_X64 +#if CROARING_COMPILER_SUPPORTS_AVX512 +CROARING_TARGET_AVX512 +ALLOW_UNALIGNED +int avx512_array_container_to_uint32_array(void *vout, const uint16_t *array, + size_t cardinality, uint32_t base) { + int outpos = 0; + uint32_t *out = (uint32_t *)vout; + size_t i = 0; + for (; i + sizeof(__m256i) / sizeof(uint16_t) <= cardinality; + i += sizeof(__m256i) / sizeof(uint16_t)) { + __m256i vinput = _mm256_loadu_si256((const __m256i *)(array + i)); + __m512i voutput = _mm512_add_epi32(_mm512_cvtepu16_epi32(vinput), + _mm512_set1_epi32(base)); + _mm512_storeu_si512((__m512i *)(out + outpos), voutput); + outpos += sizeof(__m512i) / sizeof(uint32_t); + } + for (; i < cardinality; ++i) { + const uint32_t val = base + array[i]; + memcpy(out + outpos, &val, + sizeof(uint32_t)); // should be compiled as a MOV on x64 + outpos++; + } + return outpos; +} +CROARING_UNTARGET_AVX512 +#endif // #if CROARING_COMPILER_SUPPORTS_AVX512 +#endif // #if CROARING_IS_X64 + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif
\ No newline at end of file diff --git a/contrib/libs/croaring/src/art/art.c b/contrib/libs/croaring/src/art/art.c new file mode 100644 index 00000000000..35d45fdf5b9 --- /dev/null +++ b/contrib/libs/croaring/src/art/art.c @@ -0,0 +1,1890 @@ +#include <assert.h> +#include <stdio.h> +#include <string.h> + +#include <roaring/art/art.h> +#include <roaring/memory.h> +#include <roaring/portability.h> + +#define ART_NODE4_TYPE 0 +#define ART_NODE16_TYPE 1 +#define ART_NODE48_TYPE 2 +#define ART_NODE256_TYPE 3 +#define ART_NUM_TYPES 4 + +// Node48 placeholder value to indicate no child is present at this key index. +#define ART_NODE48_EMPTY_VAL 48 + +// We use the least significant bit of node pointers to indicate whether a node +// is a leaf or an inner node. This is never surfaced to the user. +// +// Using pointer tagging to indicate leaves not only saves a bit of memory by +// sparing the typecode, but also allows us to use an intrusive leaf struct. +// Using an intrusive leaf struct leaves leaf allocation up to the user. Upon +// deallocation of the ART, we know not to free the leaves without having to +// dereference the leaf pointers. +// +// All internal operations on leaves should use CAST_LEAF before using the leaf. +// The only places that use SET_LEAF are locations where a field is directly +// assigned to a leaf pointer. After using SET_LEAF, the leaf should be treated +// as a node of unknown type. +#define IS_LEAF(p) (((uintptr_t)(p) & 1)) +#define SET_LEAF(p) ((art_node_t *)((uintptr_t)(p) | 1)) +#define CAST_LEAF(p) ((art_leaf_t *)((void *)((uintptr_t)(p) & ~1))) + +#define NODE48_AVAILABLE_CHILDREN_MASK ((UINT64_C(1) << 48) - 1) + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +typedef uint8_t art_typecode_t; + +// Aliasing with a "leaf" naming so that its purpose is clearer in the context +// of the trie internals. +typedef art_val_t art_leaf_t; + +typedef struct art_internal_validate_s { + const char **reason; + art_validate_cb_t validate_cb; + + int depth; + art_key_chunk_t current_key[ART_KEY_BYTES]; +} art_internal_validate_t; + +// Set the reason message, and return false for convenience. +static inline bool art_validate_fail(const art_internal_validate_t *validate, + const char *msg) { + *validate->reason = msg; + return false; +} + +// Inner node, with prefix. +// +// We use a fixed-length array as a pointer would be larger than the array. +typedef struct art_inner_node_s { + art_typecode_t typecode; + uint8_t prefix_size; + uint8_t prefix[ART_KEY_BYTES - 1]; +} art_inner_node_t; + +// Inner node types. + +// Node4: key[i] corresponds with children[i]. Keys are sorted. +typedef struct art_node4_s { + art_inner_node_t base; + uint8_t count; + uint8_t keys[4]; + art_node_t *children[4]; +} art_node4_t; + +// Node16: key[i] corresponds with children[i]. Keys are sorted. +typedef struct art_node16_s { + art_inner_node_t base; + uint8_t count; + uint8_t keys[16]; + art_node_t *children[16]; +} art_node16_t; + +// Node48: key[i] corresponds with children[key[i]] if key[i] != +// ART_NODE48_EMPTY_VAL. Keys are naturally sorted due to direct indexing. +typedef struct art_node48_s { + art_inner_node_t base; + uint8_t count; + // Bitset where the ith bit is set if children[i] is available + // Because there are at most 48 children, only the bottom 48 bits are used. + uint64_t available_children; + uint8_t keys[256]; + art_node_t *children[48]; +} art_node48_t; + +// Node256: children[i] is directly indexed by key chunk. A child is present if +// children[i] != NULL. +typedef struct art_node256_s { + art_inner_node_t base; + uint16_t count; + art_node_t *children[256]; +} art_node256_t; + +// Helper struct to refer to a child within a node at a specific index. +typedef struct art_indexed_child_s { + art_node_t *child; + uint8_t index; + art_key_chunk_t key_chunk; +} art_indexed_child_t; + +static inline bool art_is_leaf(const art_node_t *node) { return IS_LEAF(node); } + +static void art_leaf_populate(art_leaf_t *leaf, const art_key_chunk_t key[]) { + memcpy(leaf->key, key, ART_KEY_BYTES); +} + +static inline uint8_t art_get_type(const art_inner_node_t *node) { + return node->typecode; +} + +static inline void art_init_inner_node(art_inner_node_t *node, + art_typecode_t typecode, + const art_key_chunk_t prefix[], + uint8_t prefix_size) { + node->typecode = typecode; + node->prefix_size = prefix_size; + memcpy(node->prefix, prefix, prefix_size * sizeof(art_key_chunk_t)); +} + +static void art_free_node(art_node_t *node); + +// ===================== Start of node-specific functions ====================== + +static art_node4_t *art_node4_create(const art_key_chunk_t prefix[], + uint8_t prefix_size); +static art_node16_t *art_node16_create(const art_key_chunk_t prefix[], + uint8_t prefix_size); +static art_node48_t *art_node48_create(const art_key_chunk_t prefix[], + uint8_t prefix_size); +static art_node256_t *art_node256_create(const art_key_chunk_t prefix[], + uint8_t prefix_size); + +static art_node_t *art_node4_insert(art_node4_t *node, art_node_t *child, + uint8_t key); +static art_node_t *art_node16_insert(art_node16_t *node, art_node_t *child, + uint8_t key); +static art_node_t *art_node48_insert(art_node48_t *node, art_node_t *child, + uint8_t key); +static art_node_t *art_node256_insert(art_node256_t *node, art_node_t *child, + uint8_t key); + +static art_node4_t *art_node4_create(const art_key_chunk_t prefix[], + uint8_t prefix_size) { + art_node4_t *node = (art_node4_t *)roaring_malloc(sizeof(art_node4_t)); + art_init_inner_node(&node->base, ART_NODE4_TYPE, prefix, prefix_size); + node->count = 0; + return node; +} + +static void art_free_node4(art_node4_t *node) { + for (size_t i = 0; i < node->count; ++i) { + art_free_node(node->children[i]); + } + roaring_free(node); +} + +static inline art_node_t *art_node4_find_child(const art_node4_t *node, + art_key_chunk_t key) { + for (size_t i = 0; i < node->count; ++i) { + if (node->keys[i] == key) { + return node->children[i]; + } + } + return NULL; +} + +static art_node_t *art_node4_insert(art_node4_t *node, art_node_t *child, + uint8_t key) { + if (node->count < 4) { + size_t idx = 0; + for (; idx < node->count; ++idx) { + if (node->keys[idx] > key) { + break; + } + } + size_t after = node->count - idx; + // Shift other keys to maintain sorted order. + memmove(node->keys + idx + 1, node->keys + idx, + after * sizeof(art_key_chunk_t)); + memmove(node->children + idx + 1, node->children + idx, + after * sizeof(art_node_t *)); + + node->children[idx] = child; + node->keys[idx] = key; + node->count++; + return (art_node_t *)node; + } + art_node16_t *new_node = + art_node16_create(node->base.prefix, node->base.prefix_size); + // Instead of calling insert, this could be specialized to 2x memcpy and + // setting the count. + for (size_t i = 0; i < 4; ++i) { + art_node16_insert(new_node, node->children[i], node->keys[i]); + } + roaring_free(node); + return art_node16_insert(new_node, child, key); +} + +static inline art_node_t *art_node4_erase(art_node4_t *node, + art_key_chunk_t key_chunk) { + int idx = -1; + for (size_t i = 0; i < node->count; ++i) { + if (node->keys[i] == key_chunk) { + idx = i; + } + } + if (idx == -1) { + return (art_node_t *)node; + } + if (node->count == 2) { + // Only one child remains after erasing, so compress the path by + // removing this node. + uint8_t other_idx = idx ^ 1; + art_node_t *remaining_child = node->children[other_idx]; + art_key_chunk_t remaining_child_key = node->keys[other_idx]; + if (!art_is_leaf(remaining_child)) { + // Correct the prefix of the child node. + art_inner_node_t *inner_node = (art_inner_node_t *)remaining_child; + memmove(inner_node->prefix + node->base.prefix_size + 1, + inner_node->prefix, inner_node->prefix_size); + memcpy(inner_node->prefix, node->base.prefix, + node->base.prefix_size); + inner_node->prefix[node->base.prefix_size] = remaining_child_key; + inner_node->prefix_size += node->base.prefix_size + 1; + } + roaring_free(node); + return remaining_child; + } + // Shift other keys to maintain sorted order. + size_t after_next = node->count - idx - 1; + memmove(node->keys + idx, node->keys + idx + 1, + after_next * sizeof(art_key_chunk_t)); + memmove(node->children + idx, node->children + idx + 1, + after_next * sizeof(art_node_t *)); + node->count--; + return (art_node_t *)node; +} + +static inline void art_node4_replace(art_node4_t *node, + art_key_chunk_t key_chunk, + art_node_t *new_child) { + for (size_t i = 0; i < node->count; ++i) { + if (node->keys[i] == key_chunk) { + node->children[i] = new_child; + return; + } + } +} + +static inline art_indexed_child_t art_node4_next_child(const art_node4_t *node, + int index) { + art_indexed_child_t indexed_child; + index++; + if (index >= node->count) { + indexed_child.child = NULL; + return indexed_child; + } + indexed_child.index = index; + indexed_child.child = node->children[index]; + indexed_child.key_chunk = node->keys[index]; + return indexed_child; +} + +static inline art_indexed_child_t art_node4_prev_child(const art_node4_t *node, + int index) { + if (index > node->count) { + index = node->count; + } + index--; + art_indexed_child_t indexed_child; + if (index < 0) { + indexed_child.child = NULL; + return indexed_child; + } + indexed_child.index = index; + indexed_child.child = node->children[index]; + indexed_child.key_chunk = node->keys[index]; + return indexed_child; +} + +static inline art_indexed_child_t art_node4_child_at(const art_node4_t *node, + int index) { + art_indexed_child_t indexed_child; + if (index < 0 || index >= node->count) { + indexed_child.child = NULL; + return indexed_child; + } + indexed_child.index = index; + indexed_child.child = node->children[index]; + indexed_child.key_chunk = node->keys[index]; + return indexed_child; +} + +static inline art_indexed_child_t art_node4_lower_bound( + art_node4_t *node, art_key_chunk_t key_chunk) { + art_indexed_child_t indexed_child; + for (size_t i = 0; i < node->count; ++i) { + if (node->keys[i] >= key_chunk) { + indexed_child.index = i; + indexed_child.child = node->children[i]; + indexed_child.key_chunk = node->keys[i]; + return indexed_child; + } + } + indexed_child.child = NULL; + return indexed_child; +} + +static bool art_internal_validate_at(const art_node_t *node, + art_internal_validate_t validator); + +static bool art_node4_internal_validate(const art_node4_t *node, + art_internal_validate_t validator) { + if (node->count == 0) { + return art_validate_fail(&validator, "Node4 has no children"); + } + if (node->count > 4) { + return art_validate_fail(&validator, "Node4 has too many children"); + } + if (node->count == 1) { + return art_validate_fail( + &validator, "Node4 and child node should have been combined"); + } + validator.depth++; + for (int i = 0; i < node->count; ++i) { + if (i > 0) { + if (node->keys[i - 1] >= node->keys[i]) { + return art_validate_fail( + &validator, "Node4 keys are not strictly increasing"); + } + } + for (int j = i + 1; j < node->count; ++j) { + if (node->children[i] == node->children[j]) { + return art_validate_fail(&validator, + "Node4 has duplicate children"); + } + } + validator.current_key[validator.depth - 1] = node->keys[i]; + if (!art_internal_validate_at(node->children[i], validator)) { + return false; + } + } + return true; +} + +static art_node16_t *art_node16_create(const art_key_chunk_t prefix[], + uint8_t prefix_size) { + art_node16_t *node = (art_node16_t *)roaring_malloc(sizeof(art_node16_t)); + art_init_inner_node(&node->base, ART_NODE16_TYPE, prefix, prefix_size); + node->count = 0; + return node; +} + +static void art_free_node16(art_node16_t *node) { + for (size_t i = 0; i < node->count; ++i) { + art_free_node(node->children[i]); + } + roaring_free(node); +} + +static inline art_node_t *art_node16_find_child(const art_node16_t *node, + art_key_chunk_t key) { + for (size_t i = 0; i < node->count; ++i) { + if (node->keys[i] == key) { + return node->children[i]; + } + } + return NULL; +} + +static art_node_t *art_node16_insert(art_node16_t *node, art_node_t *child, + uint8_t key) { + if (node->count < 16) { + size_t idx = 0; + for (; idx < node->count; ++idx) { + if (node->keys[idx] > key) { + break; + } + } + size_t after = node->count - idx; + // Shift other keys to maintain sorted order. + memmove(node->keys + idx + 1, node->keys + idx, + after * sizeof(art_key_chunk_t)); + memmove(node->children + idx + 1, node->children + idx, + after * sizeof(art_node_t *)); + + node->children[idx] = child; + node->keys[idx] = key; + node->count++; + return (art_node_t *)node; + } + art_node48_t *new_node = + art_node48_create(node->base.prefix, node->base.prefix_size); + for (size_t i = 0; i < 16; ++i) { + art_node48_insert(new_node, node->children[i], node->keys[i]); + } + roaring_free(node); + return art_node48_insert(new_node, child, key); +} + +static inline art_node_t *art_node16_erase(art_node16_t *node, + uint8_t key_chunk) { + for (size_t i = 0; i < node->count; ++i) { + if (node->keys[i] == key_chunk) { + // Shift other keys to maintain sorted order. + size_t after_next = node->count - i - 1; + memmove(node->keys + i, node->keys + i + 1, + after_next * sizeof(key_chunk)); + memmove(node->children + i, node->children + i + 1, + after_next * sizeof(art_node_t *)); + node->count--; + break; + } + } + if (node->count > 4) { + return (art_node_t *)node; + } + art_node4_t *new_node = + art_node4_create(node->base.prefix, node->base.prefix_size); + // Instead of calling insert, this could be specialized to 2x memcpy and + // setting the count. + for (size_t i = 0; i < 4; ++i) { + art_node4_insert(new_node, node->children[i], node->keys[i]); + } + roaring_free(node); + return (art_node_t *)new_node; +} + +static inline void art_node16_replace(art_node16_t *node, + art_key_chunk_t key_chunk, + art_node_t *new_child) { + for (uint8_t i = 0; i < node->count; ++i) { + if (node->keys[i] == key_chunk) { + node->children[i] = new_child; + return; + } + } +} + +static inline art_indexed_child_t art_node16_next_child( + const art_node16_t *node, int index) { + art_indexed_child_t indexed_child; + index++; + if (index >= node->count) { + indexed_child.child = NULL; + return indexed_child; + } + indexed_child.index = index; + indexed_child.child = node->children[index]; + indexed_child.key_chunk = node->keys[index]; + return indexed_child; +} + +static inline art_indexed_child_t art_node16_prev_child( + const art_node16_t *node, int index) { + if (index > node->count) { + index = node->count; + } + index--; + art_indexed_child_t indexed_child; + if (index < 0) { + indexed_child.child = NULL; + return indexed_child; + } + indexed_child.index = index; + indexed_child.child = node->children[index]; + indexed_child.key_chunk = node->keys[index]; + return indexed_child; +} + +static inline art_indexed_child_t art_node16_child_at(const art_node16_t *node, + int index) { + art_indexed_child_t indexed_child; + if (index < 0 || index >= node->count) { + indexed_child.child = NULL; + return indexed_child; + } + indexed_child.index = index; + indexed_child.child = node->children[index]; + indexed_child.key_chunk = node->keys[index]; + return indexed_child; +} + +static inline art_indexed_child_t art_node16_lower_bound( + art_node16_t *node, art_key_chunk_t key_chunk) { + art_indexed_child_t indexed_child; + for (size_t i = 0; i < node->count; ++i) { + if (node->keys[i] >= key_chunk) { + indexed_child.index = i; + indexed_child.child = node->children[i]; + indexed_child.key_chunk = node->keys[i]; + return indexed_child; + } + } + indexed_child.child = NULL; + return indexed_child; +} + +static bool art_node16_internal_validate(const art_node16_t *node, + art_internal_validate_t validator) { + if (node->count <= 4) { + return art_validate_fail(&validator, "Node16 has too few children"); + } + if (node->count > 16) { + return art_validate_fail(&validator, "Node16 has too many children"); + } + validator.depth++; + for (int i = 0; i < node->count; ++i) { + if (i > 0) { + if (node->keys[i - 1] >= node->keys[i]) { + return art_validate_fail( + &validator, "Node16 keys are not strictly increasing"); + } + } + for (int j = i + 1; j < node->count; ++j) { + if (node->children[i] == node->children[j]) { + return art_validate_fail(&validator, + "Node16 has duplicate children"); + } + } + validator.current_key[validator.depth - 1] = node->keys[i]; + if (!art_internal_validate_at(node->children[i], validator)) { + return false; + } + } + return true; +} + +static art_node48_t *art_node48_create(const art_key_chunk_t prefix[], + uint8_t prefix_size) { + art_node48_t *node = (art_node48_t *)roaring_malloc(sizeof(art_node48_t)); + art_init_inner_node(&node->base, ART_NODE48_TYPE, prefix, prefix_size); + node->count = 0; + node->available_children = NODE48_AVAILABLE_CHILDREN_MASK; + for (size_t i = 0; i < 256; ++i) { + node->keys[i] = ART_NODE48_EMPTY_VAL; + } + return node; +} + +static void art_free_node48(art_node48_t *node) { + uint64_t used_children = + (node->available_children) ^ NODE48_AVAILABLE_CHILDREN_MASK; + while (used_children != 0) { + // We checked above that used_children is not zero + uint8_t child_idx = roaring_trailing_zeroes(used_children); + art_free_node(node->children[child_idx]); + used_children &= ~(UINT64_C(1) << child_idx); + } + roaring_free(node); +} + +static inline art_node_t *art_node48_find_child(const art_node48_t *node, + art_key_chunk_t key) { + uint8_t val_idx = node->keys[key]; + if (val_idx != ART_NODE48_EMPTY_VAL) { + return node->children[val_idx]; + } + return NULL; +} + +static art_node_t *art_node48_insert(art_node48_t *node, art_node_t *child, + uint8_t key) { + if (node->count < 48) { + // node->available_children is only zero when the node is full (count == + // 48), we just checked count < 48 + uint8_t val_idx = roaring_trailing_zeroes(node->available_children); + node->keys[key] = val_idx; + node->children[val_idx] = child; + node->count++; + node->available_children &= ~(UINT64_C(1) << val_idx); + return (art_node_t *)node; + } + art_node256_t *new_node = + art_node256_create(node->base.prefix, node->base.prefix_size); + for (size_t i = 0; i < 256; ++i) { + uint8_t val_idx = node->keys[i]; + if (val_idx != ART_NODE48_EMPTY_VAL) { + art_node256_insert(new_node, node->children[val_idx], i); + } + } + roaring_free(node); + return art_node256_insert(new_node, child, key); +} + +static inline art_node_t *art_node48_erase(art_node48_t *node, + uint8_t key_chunk) { + uint8_t val_idx = node->keys[key_chunk]; + if (val_idx == ART_NODE48_EMPTY_VAL) { + return (art_node_t *)node; + } + node->keys[key_chunk] = ART_NODE48_EMPTY_VAL; + node->available_children |= UINT64_C(1) << val_idx; + node->count--; + if (node->count > 16) { + return (art_node_t *)node; + } + + art_node16_t *new_node = + art_node16_create(node->base.prefix, node->base.prefix_size); + for (size_t i = 0; i < 256; ++i) { + val_idx = node->keys[i]; + if (val_idx != ART_NODE48_EMPTY_VAL) { + art_node16_insert(new_node, node->children[val_idx], i); + } + } + roaring_free(node); + return (art_node_t *)new_node; +} + +static inline void art_node48_replace(art_node48_t *node, + art_key_chunk_t key_chunk, + art_node_t *new_child) { + uint8_t val_idx = node->keys[key_chunk]; + assert(val_idx != ART_NODE48_EMPTY_VAL); + node->children[val_idx] = new_child; +} + +static inline art_indexed_child_t art_node48_next_child( + const art_node48_t *node, int index) { + art_indexed_child_t indexed_child; + index++; + for (size_t i = index; i < 256; ++i) { + if (node->keys[i] != ART_NODE48_EMPTY_VAL) { + indexed_child.index = i; + indexed_child.child = node->children[node->keys[i]]; + indexed_child.key_chunk = i; + return indexed_child; + } + } + indexed_child.child = NULL; + return indexed_child; +} + +static inline art_indexed_child_t art_node48_prev_child( + const art_node48_t *node, int index) { + if (index > 256) { + index = 256; + } + index--; + art_indexed_child_t indexed_child; + for (int i = index; i >= 0; --i) { + if (node->keys[i] != ART_NODE48_EMPTY_VAL) { + indexed_child.index = i; + indexed_child.child = node->children[node->keys[i]]; + indexed_child.key_chunk = i; + return indexed_child; + } + } + indexed_child.child = NULL; + return indexed_child; +} + +static inline art_indexed_child_t art_node48_child_at(const art_node48_t *node, + int index) { + art_indexed_child_t indexed_child; + if (index < 0 || index >= 256) { + indexed_child.child = NULL; + return indexed_child; + } + indexed_child.index = index; + indexed_child.child = node->children[node->keys[index]]; + indexed_child.key_chunk = index; + return indexed_child; +} + +static inline art_indexed_child_t art_node48_lower_bound( + art_node48_t *node, art_key_chunk_t key_chunk) { + art_indexed_child_t indexed_child; + for (size_t i = key_chunk; i < 256; ++i) { + if (node->keys[i] != ART_NODE48_EMPTY_VAL) { + indexed_child.index = i; + indexed_child.child = node->children[node->keys[i]]; + indexed_child.key_chunk = i; + return indexed_child; + } + } + indexed_child.child = NULL; + return indexed_child; +} + +static bool art_node48_internal_validate(const art_node48_t *node, + art_internal_validate_t validator) { + if (node->count <= 16) { + return art_validate_fail(&validator, "Node48 has too few children"); + } + if (node->count > 48) { + return art_validate_fail(&validator, "Node48 has too many children"); + } + uint64_t used_children = 0; + for (int i = 0; i < 256; ++i) { + uint8_t child_idx = node->keys[i]; + if (child_idx != ART_NODE48_EMPTY_VAL) { + if (used_children & (UINT64_C(1) << child_idx)) { + return art_validate_fail( + &validator, "Node48 keys point to the same child index"); + } + + art_node_t *child = node->children[child_idx]; + if (child == NULL) { + return art_validate_fail(&validator, "Node48 has a NULL child"); + } + used_children |= UINT64_C(1) << child_idx; + } + } + uint64_t expected_used_children = + (node->available_children) ^ NODE48_AVAILABLE_CHILDREN_MASK; + if (used_children != expected_used_children) { + return art_validate_fail( + &validator, + "Node48 available_children does not match actual children"); + } + while (used_children != 0) { + uint8_t child_idx = roaring_trailing_zeroes(used_children); + used_children &= used_children - 1; + + uint64_t other_children = used_children; + while (other_children != 0) { + uint8_t other_child_idx = roaring_trailing_zeroes(other_children); + if (node->children[child_idx] == node->children[other_child_idx]) { + return art_validate_fail(&validator, + "Node48 has duplicate children"); + } + other_children &= other_children - 1; + } + } + + validator.depth++; + for (int i = 0; i < 256; ++i) { + if (node->keys[i] != ART_NODE48_EMPTY_VAL) { + validator.current_key[validator.depth - 1] = i; + if (!art_internal_validate_at(node->children[node->keys[i]], + validator)) { + return false; + } + } + } + return true; +} + +static art_node256_t *art_node256_create(const art_key_chunk_t prefix[], + uint8_t prefix_size) { + art_node256_t *node = + (art_node256_t *)roaring_malloc(sizeof(art_node256_t)); + art_init_inner_node(&node->base, ART_NODE256_TYPE, prefix, prefix_size); + node->count = 0; + for (size_t i = 0; i < 256; ++i) { + node->children[i] = NULL; + } + return node; +} + +static void art_free_node256(art_node256_t *node) { + for (size_t i = 0; i < 256; ++i) { + if (node->children[i] != NULL) { + art_free_node(node->children[i]); + } + } + roaring_free(node); +} + +static inline art_node_t *art_node256_find_child(const art_node256_t *node, + art_key_chunk_t key) { + return node->children[key]; +} + +static art_node_t *art_node256_insert(art_node256_t *node, art_node_t *child, + uint8_t key) { + node->children[key] = child; + node->count++; + return (art_node_t *)node; +} + +static inline art_node_t *art_node256_erase(art_node256_t *node, + uint8_t key_chunk) { + node->children[key_chunk] = NULL; + node->count--; + if (node->count > 48) { + return (art_node_t *)node; + } + + art_node48_t *new_node = + art_node48_create(node->base.prefix, node->base.prefix_size); + for (size_t i = 0; i < 256; ++i) { + if (node->children[i] != NULL) { + art_node48_insert(new_node, node->children[i], i); + } + } + roaring_free(node); + return (art_node_t *)new_node; +} + +static inline void art_node256_replace(art_node256_t *node, + art_key_chunk_t key_chunk, + art_node_t *new_child) { + node->children[key_chunk] = new_child; +} + +static inline art_indexed_child_t art_node256_next_child( + const art_node256_t *node, int index) { + art_indexed_child_t indexed_child; + index++; + for (size_t i = index; i < 256; ++i) { + if (node->children[i] != NULL) { + indexed_child.index = i; + indexed_child.child = node->children[i]; + indexed_child.key_chunk = i; + return indexed_child; + } + } + indexed_child.child = NULL; + return indexed_child; +} + +static inline art_indexed_child_t art_node256_prev_child( + const art_node256_t *node, int index) { + if (index > 256) { + index = 256; + } + index--; + art_indexed_child_t indexed_child; + for (int i = index; i >= 0; --i) { + if (node->children[i] != NULL) { + indexed_child.index = i; + indexed_child.child = node->children[i]; + indexed_child.key_chunk = i; + return indexed_child; + } + } + indexed_child.child = NULL; + return indexed_child; +} + +static inline art_indexed_child_t art_node256_child_at( + const art_node256_t *node, int index) { + art_indexed_child_t indexed_child; + if (index < 0 || index >= 256) { + indexed_child.child = NULL; + return indexed_child; + } + indexed_child.index = index; + indexed_child.child = node->children[index]; + indexed_child.key_chunk = index; + return indexed_child; +} + +static inline art_indexed_child_t art_node256_lower_bound( + art_node256_t *node, art_key_chunk_t key_chunk) { + art_indexed_child_t indexed_child; + for (size_t i = key_chunk; i < 256; ++i) { + if (node->children[i] != NULL) { + indexed_child.index = i; + indexed_child.child = node->children[i]; + indexed_child.key_chunk = i; + return indexed_child; + } + } + indexed_child.child = NULL; + return indexed_child; +} + +static bool art_node256_internal_validate(const art_node256_t *node, + art_internal_validate_t validator) { + if (node->count <= 48) { + return art_validate_fail(&validator, "Node256 has too few children"); + } + if (node->count > 256) { + return art_validate_fail(&validator, "Node256 has too many children"); + } + validator.depth++; + int actual_count = 0; + for (int i = 0; i < 256; ++i) { + if (node->children[i] != NULL) { + actual_count++; + + for (int j = i + 1; j < 256; ++j) { + if (node->children[i] == node->children[j]) { + return art_validate_fail(&validator, + "Node256 has duplicate children"); + } + } + + validator.current_key[validator.depth - 1] = i; + if (!art_internal_validate_at(node->children[i], validator)) { + return false; + } + } + } + if (actual_count != node->count) { + return art_validate_fail( + &validator, "Node256 count does not match actual children"); + } + return true; +} + +// Finds the child with the given key chunk in the inner node, returns NULL if +// no such child is found. +static art_node_t *art_find_child(const art_inner_node_t *node, + art_key_chunk_t key_chunk) { + switch (art_get_type(node)) { + case ART_NODE4_TYPE: + return art_node4_find_child((art_node4_t *)node, key_chunk); + case ART_NODE16_TYPE: + return art_node16_find_child((art_node16_t *)node, key_chunk); + case ART_NODE48_TYPE: + return art_node48_find_child((art_node48_t *)node, key_chunk); + case ART_NODE256_TYPE: + return art_node256_find_child((art_node256_t *)node, key_chunk); + default: + assert(false); + return NULL; + } +} + +// Replaces the child with the given key chunk in the inner node. +static void art_replace(art_inner_node_t *node, art_key_chunk_t key_chunk, + art_node_t *new_child) { + switch (art_get_type(node)) { + case ART_NODE4_TYPE: + art_node4_replace((art_node4_t *)node, key_chunk, new_child); + break; + case ART_NODE16_TYPE: + art_node16_replace((art_node16_t *)node, key_chunk, new_child); + break; + case ART_NODE48_TYPE: + art_node48_replace((art_node48_t *)node, key_chunk, new_child); + break; + case ART_NODE256_TYPE: + art_node256_replace((art_node256_t *)node, key_chunk, new_child); + break; + default: + assert(false); + } +} + +// Erases the child with the given key chunk from the inner node, returns the +// updated node (the same as the initial node if it was not shrunk). +static art_node_t *art_node_erase(art_inner_node_t *node, + art_key_chunk_t key_chunk) { + switch (art_get_type(node)) { + case ART_NODE4_TYPE: + return art_node4_erase((art_node4_t *)node, key_chunk); + case ART_NODE16_TYPE: + return art_node16_erase((art_node16_t *)node, key_chunk); + case ART_NODE48_TYPE: + return art_node48_erase((art_node48_t *)node, key_chunk); + case ART_NODE256_TYPE: + return art_node256_erase((art_node256_t *)node, key_chunk); + default: + assert(false); + return NULL; + } +} + +// Inserts the leaf with the given key chunk in the inner node, returns a +// pointer to the (possibly expanded) node. +static art_node_t *art_node_insert_leaf(art_inner_node_t *node, + art_key_chunk_t key_chunk, + art_leaf_t *leaf) { + art_node_t *child = (art_node_t *)(SET_LEAF(leaf)); + switch (art_get_type(node)) { + case ART_NODE4_TYPE: + return art_node4_insert((art_node4_t *)node, child, key_chunk); + case ART_NODE16_TYPE: + return art_node16_insert((art_node16_t *)node, child, key_chunk); + case ART_NODE48_TYPE: + return art_node48_insert((art_node48_t *)node, child, key_chunk); + case ART_NODE256_TYPE: + return art_node256_insert((art_node256_t *)node, child, key_chunk); + default: + assert(false); + return NULL; + } +} + +// Frees the node and its children. Leaves are freed by the user. +static void art_free_node(art_node_t *node) { + if (art_is_leaf(node)) { + // We leave it up to the user to free leaves. + return; + } + switch (art_get_type((art_inner_node_t *)node)) { + case ART_NODE4_TYPE: + art_free_node4((art_node4_t *)node); + break; + case ART_NODE16_TYPE: + art_free_node16((art_node16_t *)node); + break; + case ART_NODE48_TYPE: + art_free_node48((art_node48_t *)node); + break; + case ART_NODE256_TYPE: + art_free_node256((art_node256_t *)node); + break; + default: + assert(false); + } +} + +// Returns the next child in key order, or NULL if called on a leaf. +// Provided index may be in the range [-1, 255]. +static art_indexed_child_t art_node_next_child(const art_node_t *node, + int index) { + if (art_is_leaf(node)) { + art_indexed_child_t indexed_child; + indexed_child.child = NULL; + return indexed_child; + } + switch (art_get_type((art_inner_node_t *)node)) { + case ART_NODE4_TYPE: + return art_node4_next_child((art_node4_t *)node, index); + case ART_NODE16_TYPE: + return art_node16_next_child((art_node16_t *)node, index); + case ART_NODE48_TYPE: + return art_node48_next_child((art_node48_t *)node, index); + case ART_NODE256_TYPE: + return art_node256_next_child((art_node256_t *)node, index); + default: + assert(false); + return (art_indexed_child_t){0}; + } +} + +// Returns the previous child in key order, or NULL if called on a leaf. +// Provided index may be in the range [0, 256]. +static art_indexed_child_t art_node_prev_child(const art_node_t *node, + int index) { + if (art_is_leaf(node)) { + art_indexed_child_t indexed_child; + indexed_child.child = NULL; + return indexed_child; + } + switch (art_get_type((art_inner_node_t *)node)) { + case ART_NODE4_TYPE: + return art_node4_prev_child((art_node4_t *)node, index); + case ART_NODE16_TYPE: + return art_node16_prev_child((art_node16_t *)node, index); + case ART_NODE48_TYPE: + return art_node48_prev_child((art_node48_t *)node, index); + case ART_NODE256_TYPE: + return art_node256_prev_child((art_node256_t *)node, index); + default: + assert(false); + return (art_indexed_child_t){0}; + } +} + +// Returns the child found at the provided index, or NULL if called on a leaf. +// Provided index is only valid if returned by art_node_(next|prev)_child. +static art_indexed_child_t art_node_child_at(const art_node_t *node, + int index) { + if (art_is_leaf(node)) { + art_indexed_child_t indexed_child; + indexed_child.child = NULL; + return indexed_child; + } + switch (art_get_type((art_inner_node_t *)node)) { + case ART_NODE4_TYPE: + return art_node4_child_at((art_node4_t *)node, index); + case ART_NODE16_TYPE: + return art_node16_child_at((art_node16_t *)node, index); + case ART_NODE48_TYPE: + return art_node48_child_at((art_node48_t *)node, index); + case ART_NODE256_TYPE: + return art_node256_child_at((art_node256_t *)node, index); + default: + assert(false); + return (art_indexed_child_t){0}; + } +} + +// Returns the child with the smallest key equal to or greater than the given +// key chunk, NULL if called on a leaf or no such child was found. +static art_indexed_child_t art_node_lower_bound(const art_node_t *node, + art_key_chunk_t key_chunk) { + if (art_is_leaf(node)) { + art_indexed_child_t indexed_child; + indexed_child.child = NULL; + return indexed_child; + } + switch (art_get_type((art_inner_node_t *)node)) { + case ART_NODE4_TYPE: + return art_node4_lower_bound((art_node4_t *)node, key_chunk); + case ART_NODE16_TYPE: + return art_node16_lower_bound((art_node16_t *)node, key_chunk); + case ART_NODE48_TYPE: + return art_node48_lower_bound((art_node48_t *)node, key_chunk); + case ART_NODE256_TYPE: + return art_node256_lower_bound((art_node256_t *)node, key_chunk); + default: + assert(false); + return (art_indexed_child_t){0}; + } +} + +// ====================== End of node-specific functions ======================= + +// Compares the given ranges of two keys, returns their relative order: +// * Key range 1 < key range 2: a negative value +// * Key range 1 == key range 2: 0 +// * Key range 1 > key range 2: a positive value +static inline int art_compare_prefix(const art_key_chunk_t key1[], + uint8_t key1_from, + const art_key_chunk_t key2[], + uint8_t key2_from, uint8_t length) { + return memcmp(key1 + key1_from, key2 + key2_from, length); +} + +// Compares two keys in full, see art_compare_prefix. +int art_compare_keys(const art_key_chunk_t key1[], + const art_key_chunk_t key2[]) { + return art_compare_prefix(key1, 0, key2, 0, ART_KEY_BYTES); +} + +// Returns the length of the common prefix between two key ranges. +static uint8_t art_common_prefix(const art_key_chunk_t key1[], + uint8_t key1_from, uint8_t key1_to, + const art_key_chunk_t key2[], + uint8_t key2_from, uint8_t key2_to) { + uint8_t min_len = key1_to - key1_from; + uint8_t key2_len = key2_to - key2_from; + if (key2_len < min_len) { + min_len = key2_len; + } + uint8_t offset = 0; + for (; offset < min_len; ++offset) { + if (key1[key1_from + offset] != key2[key2_from + offset]) { + return offset; + } + } + return offset; +} + +// Returns a pointer to the rootmost node where the value was inserted, may not +// be equal to `node`. +static art_node_t *art_insert_at(art_node_t *node, const art_key_chunk_t key[], + uint8_t depth, art_leaf_t *new_leaf) { + if (art_is_leaf(node)) { + art_leaf_t *leaf = CAST_LEAF(node); + uint8_t common_prefix = art_common_prefix( + leaf->key, depth, ART_KEY_BYTES, key, depth, ART_KEY_BYTES); + + // Previously this was a leaf, create an inner node instead and add both + // the existing and new leaf to it. + art_node_t *new_node = + (art_node_t *)art_node4_create(key + depth, common_prefix); + + new_node = art_node_insert_leaf((art_inner_node_t *)new_node, + leaf->key[depth + common_prefix], leaf); + new_node = art_node_insert_leaf((art_inner_node_t *)new_node, + key[depth + common_prefix], new_leaf); + + // The new inner node is now the rootmost node. + return new_node; + } + art_inner_node_t *inner_node = (art_inner_node_t *)node; + // Not a leaf: inner node + uint8_t common_prefix = + art_common_prefix(inner_node->prefix, 0, inner_node->prefix_size, key, + depth, ART_KEY_BYTES); + if (common_prefix != inner_node->prefix_size) { + // Partial prefix match. Create a new internal node to hold the common + // prefix. + art_node4_t *node4 = + art_node4_create(inner_node->prefix, common_prefix); + + // Make the existing internal node a child of the new internal node. + node4 = (art_node4_t *)art_node4_insert( + node4, node, inner_node->prefix[common_prefix]); + + // Correct the prefix of the moved internal node, trimming off the chunk + // inserted into the new internal node. + inner_node->prefix_size = inner_node->prefix_size - common_prefix - 1; + if (inner_node->prefix_size > 0) { + // Move the remaining prefix to the correct position. + memmove(inner_node->prefix, inner_node->prefix + common_prefix + 1, + inner_node->prefix_size); + } + + // Insert the value in the new internal node. + return art_node_insert_leaf(&node4->base, key[common_prefix + depth], + new_leaf); + } + // Prefix matches entirely or node has no prefix. Look for an existing + // child. + art_key_chunk_t key_chunk = key[depth + common_prefix]; + art_node_t *child = art_find_child(inner_node, key_chunk); + if (child != NULL) { + art_node_t *new_child = + art_insert_at(child, key, depth + common_prefix + 1, new_leaf); + if (new_child != child) { + // Node type changed. + art_replace(inner_node, key_chunk, new_child); + } + return node; + } + return art_node_insert_leaf(inner_node, key_chunk, new_leaf); +} + +// Erase helper struct. +typedef struct art_erase_result_s { + // The rootmost node where the value was erased, may not be equal to `node`. + // If no value was removed, this is null. + art_node_t *rootmost_node; + + // Value removed, null if not removed. + art_val_t *value_erased; +} art_erase_result_t; + +// Searches for the given key starting at `node`, erases it if found. +static art_erase_result_t art_erase_at(art_node_t *node, + const art_key_chunk_t *key, + uint8_t depth) { + art_erase_result_t result; + result.rootmost_node = NULL; + result.value_erased = NULL; + + if (art_is_leaf(node)) { + art_leaf_t *leaf = CAST_LEAF(node); + uint8_t common_prefix = art_common_prefix(leaf->key, 0, ART_KEY_BYTES, + key, 0, ART_KEY_BYTES); + if (common_prefix != ART_KEY_BYTES) { + // Leaf key mismatch. + return result; + } + result.value_erased = (art_val_t *)leaf; + return result; + } + art_inner_node_t *inner_node = (art_inner_node_t *)node; + uint8_t common_prefix = + art_common_prefix(inner_node->prefix, 0, inner_node->prefix_size, key, + depth, ART_KEY_BYTES); + if (common_prefix != inner_node->prefix_size) { + // Prefix mismatch. + return result; + } + art_key_chunk_t key_chunk = key[depth + common_prefix]; + art_node_t *child = art_find_child(inner_node, key_chunk); + if (child == NULL) { + // No child with key chunk. + return result; + } + // Try to erase the key further down. Skip the key chunk associated with the + // child in the node. + art_erase_result_t child_result = + art_erase_at(child, key, depth + common_prefix + 1); + if (child_result.value_erased == NULL) { + return result; + } + result.value_erased = child_result.value_erased; + result.rootmost_node = node; + if (child_result.rootmost_node == NULL) { + // Child node was fully erased, erase it from this node's children. + result.rootmost_node = art_node_erase(inner_node, key_chunk); + } else if (child_result.rootmost_node != child) { + // Child node was not fully erased, update the pointer to it in this + // node. + art_replace(inner_node, key_chunk, child_result.rootmost_node); + } + return result; +} + +// Searches for the given key starting at `node`, returns NULL if the key was +// not found. +static art_val_t *art_find_at(const art_node_t *node, + const art_key_chunk_t *key, uint8_t depth) { + while (!art_is_leaf(node)) { + art_inner_node_t *inner_node = (art_inner_node_t *)node; + uint8_t common_prefix = + art_common_prefix(inner_node->prefix, 0, inner_node->prefix_size, + key, depth, ART_KEY_BYTES); + if (common_prefix != inner_node->prefix_size) { + return NULL; + } + art_node_t *child = + art_find_child(inner_node, key[depth + inner_node->prefix_size]); + if (child == NULL) { + return NULL; + } + node = child; + // Include both the prefix and the child key chunk in the depth. + depth += inner_node->prefix_size + 1; + } + art_leaf_t *leaf = CAST_LEAF(node); + if (depth >= ART_KEY_BYTES) { + return (art_val_t *)leaf; + } + uint8_t common_prefix = + art_common_prefix(leaf->key, 0, ART_KEY_BYTES, key, 0, ART_KEY_BYTES); + if (common_prefix == ART_KEY_BYTES) { + return (art_val_t *)leaf; + } + return NULL; +} + +// Returns the size in bytes of the subtrie. +size_t art_size_in_bytes_at(const art_node_t *node) { + if (art_is_leaf(node)) { + return 0; + } + size_t size = 0; + switch (art_get_type((art_inner_node_t *)node)) { + case ART_NODE4_TYPE: { + size += sizeof(art_node4_t); + } break; + case ART_NODE16_TYPE: { + size += sizeof(art_node16_t); + } break; + case ART_NODE48_TYPE: { + size += sizeof(art_node48_t); + } break; + case ART_NODE256_TYPE: { + size += sizeof(art_node256_t); + } break; + default: + assert(false); + break; + } + art_indexed_child_t indexed_child = art_node_next_child(node, -1); + while (indexed_child.child != NULL) { + size += art_size_in_bytes_at(indexed_child.child); + indexed_child = art_node_next_child(node, indexed_child.index); + } + return size; +} + +static void art_node_print_type(const art_node_t *node) { + if (art_is_leaf(node)) { + printf("Leaf"); + return; + } + switch (art_get_type((art_inner_node_t *)node)) { + case ART_NODE4_TYPE: + printf("Node4"); + return; + case ART_NODE16_TYPE: + printf("Node16"); + return; + case ART_NODE48_TYPE: + printf("Node48"); + return; + case ART_NODE256_TYPE: + printf("Node256"); + return; + default: + assert(false); + return; + } +} + +void art_node_printf(const art_node_t *node, uint8_t depth) { + if (art_is_leaf(node)) { + printf("{ type: Leaf, key: "); + art_leaf_t *leaf = CAST_LEAF(node); + for (size_t i = 0; i < ART_KEY_BYTES; ++i) { + printf("%02x", leaf->key[i]); + } + printf(" }\n"); + return; + } + printf("{\n"); + depth++; + + printf("%*s", depth, ""); + printf("type: "); + art_node_print_type(node); + printf("\n"); + + art_inner_node_t *inner_node = (art_inner_node_t *)node; + printf("%*s", depth, ""); + printf("prefix_size: %d\n", inner_node->prefix_size); + + printf("%*s", depth, ""); + printf("prefix: "); + for (uint8_t i = 0; i < inner_node->prefix_size; ++i) { + printf("%02x", inner_node->prefix[i]); + } + printf("\n"); + + switch (art_get_type(inner_node)) { + case ART_NODE4_TYPE: { + art_node4_t *node4 = (art_node4_t *)node; + for (uint8_t i = 0; i < node4->count; ++i) { + printf("%*s", depth, ""); + printf("key: %02x ", node4->keys[i]); + art_node_printf(node4->children[i], depth); + } + } break; + case ART_NODE16_TYPE: { + art_node16_t *node16 = (art_node16_t *)node; + for (uint8_t i = 0; i < node16->count; ++i) { + printf("%*s", depth, ""); + printf("key: %02x ", node16->keys[i]); + art_node_printf(node16->children[i], depth); + } + } break; + case ART_NODE48_TYPE: { + art_node48_t *node48 = (art_node48_t *)node; + for (int i = 0; i < 256; ++i) { + if (node48->keys[i] != ART_NODE48_EMPTY_VAL) { + printf("%*s", depth, ""); + printf("key: %02x ", i); + printf("child: %02x ", node48->keys[i]); + art_node_printf(node48->children[node48->keys[i]], depth); + } + } + } break; + case ART_NODE256_TYPE: { + art_node256_t *node256 = (art_node256_t *)node; + for (int i = 0; i < 256; ++i) { + if (node256->children[i] != NULL) { + printf("%*s", depth, ""); + printf("key: %02x ", i); + art_node_printf(node256->children[i], depth); + } + } + } break; + default: + assert(false); + break; + } + depth--; + printf("%*s", depth, ""); + printf("}\n"); +} + +void art_insert(art_t *art, const art_key_chunk_t *key, art_val_t *val) { + art_leaf_t *leaf = (art_leaf_t *)val; + art_leaf_populate(leaf, key); + if (art->root == NULL) { + art->root = (art_node_t *)SET_LEAF(leaf); + return; + } + art->root = art_insert_at(art->root, key, 0, leaf); +} + +art_val_t *art_erase(art_t *art, const art_key_chunk_t *key) { + if (art->root == NULL) { + return NULL; + } + art_erase_result_t result = art_erase_at(art->root, key, 0); + if (result.value_erased == NULL) { + return NULL; + } + art->root = result.rootmost_node; + return result.value_erased; +} + +art_val_t *art_find(const art_t *art, const art_key_chunk_t *key) { + if (art->root == NULL) { + return NULL; + } + return art_find_at(art->root, key, 0); +} + +bool art_is_empty(const art_t *art) { return art->root == NULL; } + +void art_free(art_t *art) { + if (art->root == NULL) { + return; + } + art_free_node(art->root); +} + +size_t art_size_in_bytes(const art_t *art) { + size_t size = sizeof(art_t); + if (art->root != NULL) { + size += art_size_in_bytes_at(art->root); + } + return size; +} + +void art_printf(const art_t *art) { + if (art->root == NULL) { + return; + } + art_node_printf(art->root, 0); +} + +// Returns the current node that the iterator is positioned at. +static inline art_node_t *art_iterator_node(art_iterator_t *iterator) { + return iterator->frames[iterator->frame].node; +} + +// Sets the iterator key and value to the leaf's key and value. Always returns +// true for convenience. +static inline bool art_iterator_valid_loc(art_iterator_t *iterator, + art_leaf_t *leaf) { + iterator->frames[iterator->frame].node = SET_LEAF(leaf); + iterator->frames[iterator->frame].index_in_node = 0; + memcpy(iterator->key, leaf->key, ART_KEY_BYTES); + iterator->value = (art_val_t *)leaf; + return true; +} + +// Invalidates the iterator key and value. Always returns false for convenience. +static inline bool art_iterator_invalid_loc(art_iterator_t *iterator) { + memset(iterator->key, 0, ART_KEY_BYTES); + iterator->value = NULL; + return false; +} + +// Moves the iterator one level down in the tree, given a node at the current +// level and the index of the child that we're going down to. +// +// Note: does not set the index at the new level. +static void art_iterator_down(art_iterator_t *iterator, + const art_inner_node_t *node, + uint8_t index_in_node) { + iterator->frames[iterator->frame].node = (art_node_t *)node; + iterator->frames[iterator->frame].index_in_node = index_in_node; + iterator->frame++; + art_indexed_child_t indexed_child = + art_node_child_at((art_node_t *)node, index_in_node); + assert(indexed_child.child != NULL); + iterator->frames[iterator->frame].node = indexed_child.child; + iterator->depth += node->prefix_size + 1; +} + +// Moves the iterator to the next/previous child of the current node. Returns +// the child moved to, or NULL if there is no neighboring child. +static art_node_t *art_iterator_neighbor_child( + art_iterator_t *iterator, const art_inner_node_t *inner_node, + bool forward) { + art_iterator_frame_t frame = iterator->frames[iterator->frame]; + art_indexed_child_t indexed_child; + if (forward) { + indexed_child = art_node_next_child(frame.node, frame.index_in_node); + } else { + indexed_child = art_node_prev_child(frame.node, frame.index_in_node); + } + if (indexed_child.child != NULL) { + art_iterator_down(iterator, inner_node, indexed_child.index); + } + return indexed_child.child; +} + +// Moves the iterator one level up in the tree, returns false if not possible. +static bool art_iterator_up(art_iterator_t *iterator) { + if (iterator->frame == 0) { + return false; + } + iterator->frame--; + // We went up, so we are at an inner node. + iterator->depth -= + ((art_inner_node_t *)art_iterator_node(iterator))->prefix_size + 1; + return true; +} + +// Moves the iterator one level, followed by a move to the next / previous leaf. +// Sets the status of the iterator. +static bool art_iterator_up_and_move(art_iterator_t *iterator, bool forward) { + if (!art_iterator_up(iterator)) { + // We're at the root. + return art_iterator_invalid_loc(iterator); + } + return art_iterator_move(iterator, forward); +} + +// Initializes the iterator at the first / last leaf of the given node. +// Returns true for convenience. +static bool art_node_init_iterator(const art_node_t *node, + art_iterator_t *iterator, bool first) { + while (!art_is_leaf(node)) { + art_indexed_child_t indexed_child; + if (first) { + indexed_child = art_node_next_child(node, -1); + } else { + indexed_child = art_node_prev_child(node, 256); + } + art_iterator_down(iterator, (art_inner_node_t *)node, + indexed_child.index); + node = indexed_child.child; + } + // We're at a leaf. + iterator->frames[iterator->frame].node = (art_node_t *)node; + iterator->frames[iterator->frame].index_in_node = 0; // Should not matter. + return art_iterator_valid_loc(iterator, CAST_LEAF(node)); +} + +bool art_iterator_move(art_iterator_t *iterator, bool forward) { + if (art_is_leaf(art_iterator_node(iterator))) { + bool went_up = art_iterator_up(iterator); + if (!went_up) { + // This leaf is the root, we're done. + return art_iterator_invalid_loc(iterator); + } + } + // Advance within inner node. + art_node_t *neighbor_child = art_iterator_neighbor_child( + iterator, (art_inner_node_t *)art_iterator_node(iterator), forward); + if (neighbor_child != NULL) { + // There is another child at this level, go down to the first or last + // leaf. + return art_node_init_iterator(neighbor_child, iterator, forward); + } + // No more children at this level, go up. + return art_iterator_up_and_move(iterator, forward); +} + +// Assumes the iterator is positioned at a node with an equal prefix path up to +// the depth of the iterator. +static bool art_node_iterator_lower_bound(const art_node_t *node, + art_iterator_t *iterator, + const art_key_chunk_t key[]) { + while (!art_is_leaf(node)) { + art_inner_node_t *inner_node = (art_inner_node_t *)node; + int prefix_comparison = + art_compare_prefix(inner_node->prefix, 0, key, iterator->depth, + inner_node->prefix_size); + if (prefix_comparison < 0) { + // Prefix so far has been equal, but we've found a smaller key. + // Since we take the lower bound within each node, we can return the + // next leaf. + return art_iterator_up_and_move(iterator, true); + } else if (prefix_comparison > 0) { + // No key equal to the key we're looking for, return the first leaf. + return art_node_init_iterator(node, iterator, true); + } + // Prefix is equal, move to lower bound child. + art_key_chunk_t key_chunk = + key[iterator->depth + inner_node->prefix_size]; + art_indexed_child_t indexed_child = + art_node_lower_bound(node, key_chunk); + if (indexed_child.child == NULL) { + // Only smaller keys among children. + return art_iterator_up_and_move(iterator, true); + } + if (indexed_child.key_chunk > key_chunk) { + // Only larger children, return the first larger child. + art_iterator_down(iterator, inner_node, indexed_child.index); + return art_node_init_iterator(indexed_child.child, iterator, true); + } + // We found a child with an equal prefix. + art_iterator_down(iterator, inner_node, indexed_child.index); + node = indexed_child.child; + } + art_leaf_t *leaf = CAST_LEAF(node); + if (art_compare_keys(leaf->key, key) >= 0) { + // Leaf has an equal or larger key. + return art_iterator_valid_loc(iterator, leaf); + } + // Leaf has an equal prefix, but the full key is smaller. Move to the next + // leaf. + return art_iterator_up_and_move(iterator, true); +} + +art_iterator_t art_init_iterator(const art_t *art, bool first) { + art_iterator_t iterator = {0}; + if (art->root == NULL) { + return iterator; + } + art_node_init_iterator(art->root, &iterator, first); + return iterator; +} + +bool art_iterator_next(art_iterator_t *iterator) { + return art_iterator_move(iterator, true); +} + +bool art_iterator_prev(art_iterator_t *iterator) { + return art_iterator_move(iterator, false); +} + +bool art_iterator_lower_bound(art_iterator_t *iterator, + const art_key_chunk_t *key) { + if (iterator->value == NULL) { + // We're beyond the end / start of the ART so the iterator does not have + // a valid key. Start from the root. + iterator->frame = 0; + iterator->depth = 0; + return art_node_iterator_lower_bound(art_iterator_node(iterator), + iterator, key); + } + int compare_result = + art_compare_prefix(iterator->key, 0, key, 0, ART_KEY_BYTES); + // Move up until we have an equal prefix, after which we can do a normal + // lower bound search. + while (compare_result != 0) { + if (!art_iterator_up(iterator)) { + if (compare_result < 0) { + // Only smaller keys found. + return art_iterator_invalid_loc(iterator); + } else { + return art_node_init_iterator(art_iterator_node(iterator), + iterator, true); + } + } + // Since we're only moving up, we can keep comparing against the + // iterator key. + art_inner_node_t *inner_node = + (art_inner_node_t *)art_iterator_node(iterator); + compare_result = + art_compare_prefix(iterator->key, 0, key, 0, + iterator->depth + inner_node->prefix_size); + } + if (compare_result > 0) { + return art_node_init_iterator(art_iterator_node(iterator), iterator, + true); + } + return art_node_iterator_lower_bound(art_iterator_node(iterator), iterator, + key); +} + +art_iterator_t art_lower_bound(const art_t *art, const art_key_chunk_t *key) { + art_iterator_t iterator = {0}; + if (art->root != NULL) { + art_node_iterator_lower_bound(art->root, &iterator, key); + } + return iterator; +} + +art_iterator_t art_upper_bound(const art_t *art, const art_key_chunk_t *key) { + art_iterator_t iterator = {0}; + if (art->root != NULL) { + if (art_node_iterator_lower_bound(art->root, &iterator, key) && + art_compare_keys(iterator.key, key) == 0) { + art_iterator_next(&iterator); + } + } + return iterator; +} + +void art_iterator_insert(art_t *art, art_iterator_t *iterator, + const art_key_chunk_t *key, art_val_t *val) { + // TODO: This can likely be faster. + art_insert(art, key, val); + assert(art->root != NULL); + iterator->frame = 0; + iterator->depth = 0; + art_node_iterator_lower_bound(art->root, iterator, key); +} + +// TODO: consider keeping `art_t *art` in the iterator. +art_val_t *art_iterator_erase(art_t *art, art_iterator_t *iterator) { + if (iterator->value == NULL) { + return NULL; + } + art_key_chunk_t initial_key[ART_KEY_BYTES]; + memcpy(initial_key, iterator->key, ART_KEY_BYTES); + + art_val_t *value_erased = iterator->value; + bool went_up = art_iterator_up(iterator); + if (!went_up) { + // We're erasing the root. + art->root = NULL; + art_iterator_invalid_loc(iterator); + return value_erased; + } + + // Erase the leaf. + art_inner_node_t *parent_node = + (art_inner_node_t *)art_iterator_node(iterator); + art_key_chunk_t key_chunk_in_parent = + iterator->key[iterator->depth + parent_node->prefix_size]; + art_node_t *new_parent_node = + art_node_erase(parent_node, key_chunk_in_parent); + + if (new_parent_node != ((art_node_t *)parent_node)) { + // Replace the pointer to the inner node we erased from in its + // parent (it may be a leaf now). + iterator->frames[iterator->frame].node = new_parent_node; + went_up = art_iterator_up(iterator); + if (went_up) { + art_inner_node_t *grandparent_node = + (art_inner_node_t *)art_iterator_node(iterator); + art_key_chunk_t key_chunk_in_grandparent = + iterator->key[iterator->depth + grandparent_node->prefix_size]; + art_replace(grandparent_node, key_chunk_in_grandparent, + new_parent_node); + } else { + // We were already at the rootmost node. + art->root = new_parent_node; + } + } + + iterator->frame = 0; + iterator->depth = 0; + // Do a lower bound search for the initial key, which will find the first + // greater key if it exists. This can likely be mildly faster if we instead + // start from the current position. + art_node_iterator_lower_bound(art->root, iterator, initial_key); + return value_erased; +} + +static bool art_internal_validate_at(const art_node_t *node, + art_internal_validate_t validator) { + if (node == NULL) { + return art_validate_fail(&validator, "node is null"); + } + if (art_is_leaf(node)) { + art_leaf_t *leaf = CAST_LEAF(node); + if (art_compare_prefix(leaf->key, 0, validator.current_key, 0, + validator.depth) != 0) { + return art_validate_fail( + &validator, + "leaf key does not match its position's prefix in the tree"); + } + if (validator.validate_cb != NULL && + !validator.validate_cb(leaf, validator.reason)) { + if (*validator.reason == NULL) { + *validator.reason = "leaf validation failed"; + } + return false; + } + } else { + art_inner_node_t *inner_node = (art_inner_node_t *)node; + + if (validator.depth + inner_node->prefix_size + 1 > ART_KEY_BYTES) { + return art_validate_fail(&validator, + "node has too much prefix at given depth"); + } + memcpy(validator.current_key + validator.depth, inner_node->prefix, + inner_node->prefix_size); + validator.depth += inner_node->prefix_size; + + switch (inner_node->typecode) { + case ART_NODE4_TYPE: + if (!art_node4_internal_validate((art_node4_t *)inner_node, + validator)) { + return false; + } + break; + case ART_NODE16_TYPE: + if (!art_node16_internal_validate((art_node16_t *)inner_node, + validator)) { + return false; + } + break; + case ART_NODE48_TYPE: + if (!art_node48_internal_validate((art_node48_t *)inner_node, + validator)) { + return false; + } + break; + case ART_NODE256_TYPE: + if (!art_node256_internal_validate((art_node256_t *)inner_node, + validator)) { + return false; + } + break; + default: + return art_validate_fail(&validator, "invalid node type"); + } + } + return true; +} + +bool art_internal_validate(const art_t *art, const char **reason, + art_validate_cb_t validate_cb) { + const char *reason_local; + if (reason == NULL) { + // Always allow assigning through *reason + reason = &reason_local; + } + *reason = NULL; + if (art->root == NULL) { + return true; + } + art_internal_validate_t validator = { + .reason = reason, + .validate_cb = validate_cb, + .depth = 0, + .current_key = {0}, + }; + return art_internal_validate_at(art->root, validator); +} + +#ifdef __cplusplus +} // extern "C" +} // namespace roaring +} // namespace internal +#endif diff --git a/contrib/libs/croaring/src/bitset.c b/contrib/libs/croaring/src/bitset.c new file mode 100644 index 00000000000..a8380733032 --- /dev/null +++ b/contrib/libs/croaring/src/bitset.c @@ -0,0 +1,480 @@ +#include <limits.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <roaring/bitset/bitset.h> +#include <roaring/memory.h> +#include <roaring/portability.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +extern inline void bitset_print(const bitset_t *b); +extern inline bool bitset_for_each(const bitset_t *b, bitset_iterator iterator, + void *ptr); +extern inline size_t bitset_next_set_bits(const bitset_t *bitset, + size_t *buffer, size_t capacity, + size_t *startfrom); +extern inline void bitset_set_to_value(bitset_t *bitset, size_t i, bool flag); +extern inline bool bitset_next_set_bit(const bitset_t *bitset, size_t *i); +extern inline void bitset_set(bitset_t *bitset, size_t i); +extern inline bool bitset_get(const bitset_t *bitset, size_t i); +extern inline size_t bitset_size_in_words(const bitset_t *bitset); +extern inline size_t bitset_size_in_bits(const bitset_t *bitset); +extern inline size_t bitset_size_in_bytes(const bitset_t *bitset); + +/* Create a new bitset. Return NULL in case of failure. */ +bitset_t *bitset_create(void) { + bitset_t *bitset = NULL; + /* Allocate the bitset itself. */ + if ((bitset = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { + return NULL; + } + bitset->array = NULL; + bitset->arraysize = 0; + bitset->capacity = 0; + return bitset; +} + +/* Create a new bitset able to contain size bits. Return NULL in case of + * failure. */ +bitset_t *bitset_create_with_capacity(size_t size) { + bitset_t *bitset = NULL; + /* Allocate the bitset itself. */ + if ((bitset = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { + return NULL; + } + bitset->arraysize = + (size + sizeof(uint64_t) * 8 - 1) / (sizeof(uint64_t) * 8); + bitset->capacity = bitset->arraysize; + if ((bitset->array = (uint64_t *)roaring_calloc( + bitset->arraysize, sizeof(uint64_t))) == NULL) { + roaring_free(bitset); + return NULL; + } + return bitset; +} + +/* Create a copy */ +bitset_t *bitset_copy(const bitset_t *bitset) { + bitset_t *copy = NULL; + /* Allocate the bitset itself. */ + if ((copy = (bitset_t *)roaring_malloc(sizeof(bitset_t))) == NULL) { + return NULL; + } + memcpy(copy, bitset, sizeof(bitset_t)); + copy->capacity = copy->arraysize; + if ((copy->array = (uint64_t *)roaring_malloc(sizeof(uint64_t) * + bitset->arraysize)) == NULL) { + roaring_free(copy); + return NULL; + } + memcpy(copy->array, bitset->array, sizeof(uint64_t) * bitset->arraysize); + return copy; +} + +void bitset_clear(bitset_t *bitset) { + memset(bitset->array, 0, sizeof(uint64_t) * bitset->arraysize); +} + +void bitset_fill(bitset_t *bitset) { + memset(bitset->array, 0xff, sizeof(uint64_t) * bitset->arraysize); +} + +void bitset_shift_left(bitset_t *bitset, size_t s) { + size_t extra_words = s / 64; + int inword_shift = s % 64; + size_t as = bitset->arraysize; + if (inword_shift == 0) { + bitset_resize(bitset, as + extra_words, false); + // could be done with a memmove + for (size_t i = as + extra_words; i > extra_words; i--) { + bitset->array[i - 1] = bitset->array[i - 1 - extra_words]; + } + } else { + bitset_resize(bitset, as + extra_words + 1, true); + bitset->array[as + extra_words] = + bitset->array[as - 1] >> (64 - inword_shift); + for (size_t i = as + extra_words; i >= extra_words + 2; i--) { + bitset->array[i - 1] = + (bitset->array[i - 1 - extra_words] << inword_shift) | + (bitset->array[i - 2 - extra_words] >> (64 - inword_shift)); + } + bitset->array[extra_words] = bitset->array[0] << inword_shift; + } + for (size_t i = 0; i < extra_words; i++) { + bitset->array[i] = 0; + } +} + +void bitset_shift_right(bitset_t *bitset, size_t s) { + size_t extra_words = s / 64; + int inword_shift = s % 64; + size_t as = bitset->arraysize; + if (inword_shift == 0) { + // could be done with a memmove + for (size_t i = 0; i < as - extra_words; i++) { + bitset->array[i] = bitset->array[i + extra_words]; + } + bitset_resize(bitset, as - extra_words, false); + + } else { + for (size_t i = 0; i + extra_words + 1 < as; i++) { + bitset->array[i] = + (bitset->array[i + extra_words] >> inword_shift) | + (bitset->array[i + extra_words + 1] << (64 - inword_shift)); + } + bitset->array[as - extra_words - 1] = + (bitset->array[as - 1] >> inword_shift); + bitset_resize(bitset, as - extra_words, false); + } +} + +/* Free memory. */ +void bitset_free(bitset_t *bitset) { + if (bitset == NULL) { + return; + } + roaring_free(bitset->array); + roaring_free(bitset); +} + +/* Resize the bitset so that it can support newarraysize * 64 bits. Return true + * in case of success, false for failure. */ +bool bitset_resize(bitset_t *bitset, size_t newarraysize, bool padwithzeroes) { + if (newarraysize > SIZE_MAX / 64) { + return false; + } + size_t smallest = + newarraysize < bitset->arraysize ? newarraysize : bitset->arraysize; + if (bitset->capacity < newarraysize) { + uint64_t *newarray; + size_t newcapacity = bitset->capacity; + if (newcapacity == 0) { + newcapacity = 1; + } + while (newcapacity < newarraysize) { + newcapacity *= 2; + } + if ((newarray = (uint64_t *)roaring_realloc( + bitset->array, sizeof(uint64_t) * newcapacity)) == NULL) { + return false; + } + bitset->capacity = newcapacity; + bitset->array = newarray; + } + if (padwithzeroes && (newarraysize > smallest)) + memset(bitset->array + smallest, 0, + sizeof(uint64_t) * (newarraysize - smallest)); + bitset->arraysize = newarraysize; + return true; // success! +} + +size_t bitset_count(const bitset_t *bitset) { + size_t card = 0; + size_t k = 0; + for (; k + 7 < bitset->arraysize; k += 8) { + card += roaring_hamming(bitset->array[k]); + card += roaring_hamming(bitset->array[k + 1]); + card += roaring_hamming(bitset->array[k + 2]); + card += roaring_hamming(bitset->array[k + 3]); + card += roaring_hamming(bitset->array[k + 4]); + card += roaring_hamming(bitset->array[k + 5]); + card += roaring_hamming(bitset->array[k + 6]); + card += roaring_hamming(bitset->array[k + 7]); + } + for (; k + 3 < bitset->arraysize; k += 4) { + card += roaring_hamming(bitset->array[k]); + card += roaring_hamming(bitset->array[k + 1]); + card += roaring_hamming(bitset->array[k + 2]); + card += roaring_hamming(bitset->array[k + 3]); + } + for (; k < bitset->arraysize; k++) { + card += roaring_hamming(bitset->array[k]); + } + return card; +} + +bool bitset_inplace_union(bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + for (size_t k = 0; k < minlength; ++k) { + b1->array[k] |= b2->array[k]; + } + if (b2->arraysize > b1->arraysize) { + size_t oldsize = b1->arraysize; + if (!bitset_resize(b1, b2->arraysize, false)) return false; + memcpy(b1->array + oldsize, b2->array + oldsize, + (b2->arraysize - oldsize) * sizeof(uint64_t)); + } + return true; +} + +size_t bitset_minimum(const bitset_t *bitset) { + for (size_t k = 0; k < bitset->arraysize; k++) { + uint64_t w = bitset->array[k]; + if (w != 0) { + return roaring_trailing_zeroes(w) + k * 64; + } + } + return 0; +} + +bool bitset_grow(bitset_t *bitset, size_t newarraysize) { + if (newarraysize < bitset->arraysize) { + return false; + } + if (newarraysize > SIZE_MAX / 64) { + return false; + } + if (bitset->capacity < newarraysize) { + uint64_t *newarray; + size_t newcapacity = (UINT64_C(0xFFFFFFFFFFFFFFFF) >> + roaring_leading_zeroes(newarraysize)) + + 1; + while (newcapacity < newarraysize) { + newcapacity *= 2; + } + if ((newarray = (uint64_t *)roaring_realloc( + bitset->array, sizeof(uint64_t) * newcapacity)) == NULL) { + return false; + } + bitset->capacity = newcapacity; + bitset->array = newarray; + } + memset(bitset->array + bitset->arraysize, 0, + sizeof(uint64_t) * (newarraysize - bitset->arraysize)); + bitset->arraysize = newarraysize; + return true; // success! +} + +size_t bitset_maximum(const bitset_t *bitset) { + for (size_t k = bitset->arraysize; k > 0; k--) { + uint64_t w = bitset->array[k - 1]; + if (w != 0) { + return 63 - roaring_leading_zeroes(w) + (k - 1) * 64; + } + } + return 0; +} + +/* Returns true if bitsets share no common elements, false otherwise. + * + * Performs early-out if common element found. */ +bool bitsets_disjoint(const bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + + for (size_t k = 0; k < minlength; k++) { + if ((b1->array[k] & b2->array[k]) != 0) return false; + } + return true; +} + +/* Returns true if bitsets contain at least 1 common element, false if they are + * disjoint. + * + * Performs early-out if common element found. */ +bool bitsets_intersect(const bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + + for (size_t k = 0; k < minlength; k++) { + if ((b1->array[k] & b2->array[k]) != 0) return true; + } + return false; +} + +/* Returns true if b has any bits set in or after b->array[starting_loc]. */ +static bool any_bits_set(const bitset_t *b, size_t starting_loc) { + if (starting_loc >= b->arraysize) { + return false; + } + for (size_t k = starting_loc; k < b->arraysize; k++) { + if (b->array[k] != 0) return true; + } + return false; +} + +/* Returns true if b1 has all of b2's bits set. + * + * Performs early out if a bit is found in b2 that is not found in b1. */ +bool bitset_contains_all(const bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2) { + size_t min_size = b1->arraysize; + if (b1->arraysize > b2->arraysize) { + min_size = b2->arraysize; + } + for (size_t k = 0; k < min_size; k++) { + if ((b1->array[k] & b2->array[k]) != b2->array[k]) { + return false; + } + } + if (b2->arraysize > b1->arraysize) { + /* Need to check if b2 has any bits set beyond b1's array */ + return !any_bits_set(b2, b1->arraysize); + } + return true; +} + +size_t bitset_union_count(const bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2) { + size_t answer = 0; + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + for (; k + 3 < minlength; k += 4) { + answer += roaring_hamming(b1->array[k] | b2->array[k]); + answer += roaring_hamming(b1->array[k + 1] | b2->array[k + 1]); + answer += roaring_hamming(b1->array[k + 2] | b2->array[k + 2]); + answer += roaring_hamming(b1->array[k + 3] | b2->array[k + 3]); + } + for (; k < minlength; ++k) { + answer += roaring_hamming(b1->array[k] | b2->array[k]); + } + if (b2->arraysize > b1->arraysize) { + // k is equal to b1->arraysize + for (; k + 3 < b2->arraysize; k += 4) { + answer += roaring_hamming(b2->array[k]); + answer += roaring_hamming(b2->array[k + 1]); + answer += roaring_hamming(b2->array[k + 2]); + answer += roaring_hamming(b2->array[k + 3]); + } + for (; k < b2->arraysize; ++k) { + answer += roaring_hamming(b2->array[k]); + } + } else { + // k is equal to b2->arraysize + for (; k + 3 < b1->arraysize; k += 4) { + answer += roaring_hamming(b1->array[k]); + answer += roaring_hamming(b1->array[k + 1]); + answer += roaring_hamming(b1->array[k + 2]); + answer += roaring_hamming(b1->array[k + 3]); + } + for (; k < b1->arraysize; ++k) { + answer += roaring_hamming(b1->array[k]); + } + } + return answer; +} + +void bitset_inplace_intersection(bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + for (; k < minlength; ++k) { + b1->array[k] &= b2->array[k]; + } + for (; k < b1->arraysize; ++k) { + b1->array[k] = 0; // memset could, maybe, be a tiny bit faster + } +} + +size_t bitset_intersection_count(const bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2) { + size_t answer = 0; + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + for (size_t k = 0; k < minlength; ++k) { + answer += roaring_hamming(b1->array[k] & b2->array[k]); + } + return answer; +} + +void bitset_inplace_difference(bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + for (; k < minlength; ++k) { + b1->array[k] &= ~(b2->array[k]); + } +} + +size_t bitset_difference_count(const bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + size_t answer = 0; + for (; k < minlength; ++k) { + answer += roaring_hamming(b1->array[k] & ~(b2->array[k])); + } + for (; k < b1->arraysize; ++k) { + answer += roaring_hamming(b1->array[k]); + } + return answer; +} + +bool bitset_inplace_symmetric_difference(bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + for (; k < minlength; ++k) { + b1->array[k] ^= b2->array[k]; + } + if (b2->arraysize > b1->arraysize) { + size_t oldsize = b1->arraysize; + if (!bitset_resize(b1, b2->arraysize, false)) return false; + memcpy(b1->array + oldsize, b2->array + oldsize, + (b2->arraysize - oldsize) * sizeof(uint64_t)); + } + return true; +} + +size_t bitset_symmetric_difference_count(const bitset_t *CBITSET_RESTRICT b1, + const bitset_t *CBITSET_RESTRICT b2) { + size_t minlength = + b1->arraysize < b2->arraysize ? b1->arraysize : b2->arraysize; + size_t k = 0; + size_t answer = 0; + for (; k < minlength; ++k) { + answer += roaring_hamming(b1->array[k] ^ b2->array[k]); + } + if (b2->arraysize > b1->arraysize) { + for (; k < b2->arraysize; ++k) { + answer += roaring_hamming(b2->array[k]); + } + } else { + for (; k < b1->arraysize; ++k) { + answer += roaring_hamming(b1->array[k]); + } + } + return answer; +} + +bool bitset_trim(bitset_t *bitset) { + size_t newsize = bitset->arraysize; + while (newsize > 0) { + if (bitset->array[newsize - 1] == 0) + newsize -= 1; + else + break; + } + if (bitset->capacity == newsize) return true; // nothing to do + uint64_t *newarray; + if ((newarray = (uint64_t *)roaring_realloc( + bitset->array, sizeof(uint64_t) * newsize)) == NULL) { + return false; + } + bitset->array = newarray; + bitset->capacity = newsize; + bitset->arraysize = newsize; + return true; +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif diff --git a/contrib/libs/croaring/src/bitset_util.c b/contrib/libs/croaring/src/bitset_util.c new file mode 100644 index 00000000000..0ae7d925827 --- /dev/null +++ b/contrib/libs/croaring/src/bitset_util.c @@ -0,0 +1,1161 @@ +#include <assert.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <roaring/bitset_util.h> + +#if CROARING_IS_X64 +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif +#ifdef __cplusplus +using namespace ::roaring::internal; +extern "C" { +namespace roaring { +namespace api { +#endif + +#if CROARING_IS_X64 +static uint8_t lengthTable[256] = { + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, + 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, + 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, + 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, + 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, + 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}; +#endif + +#if CROARING_IS_X64 +ALIGNED(32) +static uint32_t vecDecodeTable[256][8] = { + {0, 0, 0, 0, 0, 0, 0, 0}, /* 0x00 (00000000) */ + {1, 0, 0, 0, 0, 0, 0, 0}, /* 0x01 (00000001) */ + {2, 0, 0, 0, 0, 0, 0, 0}, /* 0x02 (00000010) */ + {1, 2, 0, 0, 0, 0, 0, 0}, /* 0x03 (00000011) */ + {3, 0, 0, 0, 0, 0, 0, 0}, /* 0x04 (00000100) */ + {1, 3, 0, 0, 0, 0, 0, 0}, /* 0x05 (00000101) */ + {2, 3, 0, 0, 0, 0, 0, 0}, /* 0x06 (00000110) */ + {1, 2, 3, 0, 0, 0, 0, 0}, /* 0x07 (00000111) */ + {4, 0, 0, 0, 0, 0, 0, 0}, /* 0x08 (00001000) */ + {1, 4, 0, 0, 0, 0, 0, 0}, /* 0x09 (00001001) */ + {2, 4, 0, 0, 0, 0, 0, 0}, /* 0x0A (00001010) */ + {1, 2, 4, 0, 0, 0, 0, 0}, /* 0x0B (00001011) */ + {3, 4, 0, 0, 0, 0, 0, 0}, /* 0x0C (00001100) */ + {1, 3, 4, 0, 0, 0, 0, 0}, /* 0x0D (00001101) */ + {2, 3, 4, 0, 0, 0, 0, 0}, /* 0x0E (00001110) */ + {1, 2, 3, 4, 0, 0, 0, 0}, /* 0x0F (00001111) */ + {5, 0, 0, 0, 0, 0, 0, 0}, /* 0x10 (00010000) */ + {1, 5, 0, 0, 0, 0, 0, 0}, /* 0x11 (00010001) */ + {2, 5, 0, 0, 0, 0, 0, 0}, /* 0x12 (00010010) */ + {1, 2, 5, 0, 0, 0, 0, 0}, /* 0x13 (00010011) */ + {3, 5, 0, 0, 0, 0, 0, 0}, /* 0x14 (00010100) */ + {1, 3, 5, 0, 0, 0, 0, 0}, /* 0x15 (00010101) */ + {2, 3, 5, 0, 0, 0, 0, 0}, /* 0x16 (00010110) */ + {1, 2, 3, 5, 0, 0, 0, 0}, /* 0x17 (00010111) */ + {4, 5, 0, 0, 0, 0, 0, 0}, /* 0x18 (00011000) */ + {1, 4, 5, 0, 0, 0, 0, 0}, /* 0x19 (00011001) */ + {2, 4, 5, 0, 0, 0, 0, 0}, /* 0x1A (00011010) */ + {1, 2, 4, 5, 0, 0, 0, 0}, /* 0x1B (00011011) */ + {3, 4, 5, 0, 0, 0, 0, 0}, /* 0x1C (00011100) */ + {1, 3, 4, 5, 0, 0, 0, 0}, /* 0x1D (00011101) */ + {2, 3, 4, 5, 0, 0, 0, 0}, /* 0x1E (00011110) */ + {1, 2, 3, 4, 5, 0, 0, 0}, /* 0x1F (00011111) */ + {6, 0, 0, 0, 0, 0, 0, 0}, /* 0x20 (00100000) */ + {1, 6, 0, 0, 0, 0, 0, 0}, /* 0x21 (00100001) */ + {2, 6, 0, 0, 0, 0, 0, 0}, /* 0x22 (00100010) */ + {1, 2, 6, 0, 0, 0, 0, 0}, /* 0x23 (00100011) */ + {3, 6, 0, 0, 0, 0, 0, 0}, /* 0x24 (00100100) */ + {1, 3, 6, 0, 0, 0, 0, 0}, /* 0x25 (00100101) */ + {2, 3, 6, 0, 0, 0, 0, 0}, /* 0x26 (00100110) */ + {1, 2, 3, 6, 0, 0, 0, 0}, /* 0x27 (00100111) */ + {4, 6, 0, 0, 0, 0, 0, 0}, /* 0x28 (00101000) */ + {1, 4, 6, 0, 0, 0, 0, 0}, /* 0x29 (00101001) */ + {2, 4, 6, 0, 0, 0, 0, 0}, /* 0x2A (00101010) */ + {1, 2, 4, 6, 0, 0, 0, 0}, /* 0x2B (00101011) */ + {3, 4, 6, 0, 0, 0, 0, 0}, /* 0x2C (00101100) */ + {1, 3, 4, 6, 0, 0, 0, 0}, /* 0x2D (00101101) */ + {2, 3, 4, 6, 0, 0, 0, 0}, /* 0x2E (00101110) */ + {1, 2, 3, 4, 6, 0, 0, 0}, /* 0x2F (00101111) */ + {5, 6, 0, 0, 0, 0, 0, 0}, /* 0x30 (00110000) */ + {1, 5, 6, 0, 0, 0, 0, 0}, /* 0x31 (00110001) */ + {2, 5, 6, 0, 0, 0, 0, 0}, /* 0x32 (00110010) */ + {1, 2, 5, 6, 0, 0, 0, 0}, /* 0x33 (00110011) */ + {3, 5, 6, 0, 0, 0, 0, 0}, /* 0x34 (00110100) */ + {1, 3, 5, 6, 0, 0, 0, 0}, /* 0x35 (00110101) */ + {2, 3, 5, 6, 0, 0, 0, 0}, /* 0x36 (00110110) */ + {1, 2, 3, 5, 6, 0, 0, 0}, /* 0x37 (00110111) */ + {4, 5, 6, 0, 0, 0, 0, 0}, /* 0x38 (00111000) */ + {1, 4, 5, 6, 0, 0, 0, 0}, /* 0x39 (00111001) */ + {2, 4, 5, 6, 0, 0, 0, 0}, /* 0x3A (00111010) */ + {1, 2, 4, 5, 6, 0, 0, 0}, /* 0x3B (00111011) */ + {3, 4, 5, 6, 0, 0, 0, 0}, /* 0x3C (00111100) */ + {1, 3, 4, 5, 6, 0, 0, 0}, /* 0x3D (00111101) */ + {2, 3, 4, 5, 6, 0, 0, 0}, /* 0x3E (00111110) */ + {1, 2, 3, 4, 5, 6, 0, 0}, /* 0x3F (00111111) */ + {7, 0, 0, 0, 0, 0, 0, 0}, /* 0x40 (01000000) */ + {1, 7, 0, 0, 0, 0, 0, 0}, /* 0x41 (01000001) */ + {2, 7, 0, 0, 0, 0, 0, 0}, /* 0x42 (01000010) */ + {1, 2, 7, 0, 0, 0, 0, 0}, /* 0x43 (01000011) */ + {3, 7, 0, 0, 0, 0, 0, 0}, /* 0x44 (01000100) */ + {1, 3, 7, 0, 0, 0, 0, 0}, /* 0x45 (01000101) */ + {2, 3, 7, 0, 0, 0, 0, 0}, /* 0x46 (01000110) */ + {1, 2, 3, 7, 0, 0, 0, 0}, /* 0x47 (01000111) */ + {4, 7, 0, 0, 0, 0, 0, 0}, /* 0x48 (01001000) */ + {1, 4, 7, 0, 0, 0, 0, 0}, /* 0x49 (01001001) */ + {2, 4, 7, 0, 0, 0, 0, 0}, /* 0x4A (01001010) */ + {1, 2, 4, 7, 0, 0, 0, 0}, /* 0x4B (01001011) */ + {3, 4, 7, 0, 0, 0, 0, 0}, /* 0x4C (01001100) */ + {1, 3, 4, 7, 0, 0, 0, 0}, /* 0x4D (01001101) */ + {2, 3, 4, 7, 0, 0, 0, 0}, /* 0x4E (01001110) */ + {1, 2, 3, 4, 7, 0, 0, 0}, /* 0x4F (01001111) */ + {5, 7, 0, 0, 0, 0, 0, 0}, /* 0x50 (01010000) */ + {1, 5, 7, 0, 0, 0, 0, 0}, /* 0x51 (01010001) */ + {2, 5, 7, 0, 0, 0, 0, 0}, /* 0x52 (01010010) */ + {1, 2, 5, 7, 0, 0, 0, 0}, /* 0x53 (01010011) */ + {3, 5, 7, 0, 0, 0, 0, 0}, /* 0x54 (01010100) */ + {1, 3, 5, 7, 0, 0, 0, 0}, /* 0x55 (01010101) */ + {2, 3, 5, 7, 0, 0, 0, 0}, /* 0x56 (01010110) */ + {1, 2, 3, 5, 7, 0, 0, 0}, /* 0x57 (01010111) */ + {4, 5, 7, 0, 0, 0, 0, 0}, /* 0x58 (01011000) */ + {1, 4, 5, 7, 0, 0, 0, 0}, /* 0x59 (01011001) */ + {2, 4, 5, 7, 0, 0, 0, 0}, /* 0x5A (01011010) */ + {1, 2, 4, 5, 7, 0, 0, 0}, /* 0x5B (01011011) */ + {3, 4, 5, 7, 0, 0, 0, 0}, /* 0x5C (01011100) */ + {1, 3, 4, 5, 7, 0, 0, 0}, /* 0x5D (01011101) */ + {2, 3, 4, 5, 7, 0, 0, 0}, /* 0x5E (01011110) */ + {1, 2, 3, 4, 5, 7, 0, 0}, /* 0x5F (01011111) */ + {6, 7, 0, 0, 0, 0, 0, 0}, /* 0x60 (01100000) */ + {1, 6, 7, 0, 0, 0, 0, 0}, /* 0x61 (01100001) */ + {2, 6, 7, 0, 0, 0, 0, 0}, /* 0x62 (01100010) */ + {1, 2, 6, 7, 0, 0, 0, 0}, /* 0x63 (01100011) */ + {3, 6, 7, 0, 0, 0, 0, 0}, /* 0x64 (01100100) */ + {1, 3, 6, 7, 0, 0, 0, 0}, /* 0x65 (01100101) */ + {2, 3, 6, 7, 0, 0, 0, 0}, /* 0x66 (01100110) */ + {1, 2, 3, 6, 7, 0, 0, 0}, /* 0x67 (01100111) */ + {4, 6, 7, 0, 0, 0, 0, 0}, /* 0x68 (01101000) */ + {1, 4, 6, 7, 0, 0, 0, 0}, /* 0x69 (01101001) */ + {2, 4, 6, 7, 0, 0, 0, 0}, /* 0x6A (01101010) */ + {1, 2, 4, 6, 7, 0, 0, 0}, /* 0x6B (01101011) */ + {3, 4, 6, 7, 0, 0, 0, 0}, /* 0x6C (01101100) */ + {1, 3, 4, 6, 7, 0, 0, 0}, /* 0x6D (01101101) */ + {2, 3, 4, 6, 7, 0, 0, 0}, /* 0x6E (01101110) */ + {1, 2, 3, 4, 6, 7, 0, 0}, /* 0x6F (01101111) */ + {5, 6, 7, 0, 0, 0, 0, 0}, /* 0x70 (01110000) */ + {1, 5, 6, 7, 0, 0, 0, 0}, /* 0x71 (01110001) */ + {2, 5, 6, 7, 0, 0, 0, 0}, /* 0x72 (01110010) */ + {1, 2, 5, 6, 7, 0, 0, 0}, /* 0x73 (01110011) */ + {3, 5, 6, 7, 0, 0, 0, 0}, /* 0x74 (01110100) */ + {1, 3, 5, 6, 7, 0, 0, 0}, /* 0x75 (01110101) */ + {2, 3, 5, 6, 7, 0, 0, 0}, /* 0x76 (01110110) */ + {1, 2, 3, 5, 6, 7, 0, 0}, /* 0x77 (01110111) */ + {4, 5, 6, 7, 0, 0, 0, 0}, /* 0x78 (01111000) */ + {1, 4, 5, 6, 7, 0, 0, 0}, /* 0x79 (01111001) */ + {2, 4, 5, 6, 7, 0, 0, 0}, /* 0x7A (01111010) */ + {1, 2, 4, 5, 6, 7, 0, 0}, /* 0x7B (01111011) */ + {3, 4, 5, 6, 7, 0, 0, 0}, /* 0x7C (01111100) */ + {1, 3, 4, 5, 6, 7, 0, 0}, /* 0x7D (01111101) */ + {2, 3, 4, 5, 6, 7, 0, 0}, /* 0x7E (01111110) */ + {1, 2, 3, 4, 5, 6, 7, 0}, /* 0x7F (01111111) */ + {8, 0, 0, 0, 0, 0, 0, 0}, /* 0x80 (10000000) */ + {1, 8, 0, 0, 0, 0, 0, 0}, /* 0x81 (10000001) */ + {2, 8, 0, 0, 0, 0, 0, 0}, /* 0x82 (10000010) */ + {1, 2, 8, 0, 0, 0, 0, 0}, /* 0x83 (10000011) */ + {3, 8, 0, 0, 0, 0, 0, 0}, /* 0x84 (10000100) */ + {1, 3, 8, 0, 0, 0, 0, 0}, /* 0x85 (10000101) */ + {2, 3, 8, 0, 0, 0, 0, 0}, /* 0x86 (10000110) */ + {1, 2, 3, 8, 0, 0, 0, 0}, /* 0x87 (10000111) */ + {4, 8, 0, 0, 0, 0, 0, 0}, /* 0x88 (10001000) */ + {1, 4, 8, 0, 0, 0, 0, 0}, /* 0x89 (10001001) */ + {2, 4, 8, 0, 0, 0, 0, 0}, /* 0x8A (10001010) */ + {1, 2, 4, 8, 0, 0, 0, 0}, /* 0x8B (10001011) */ + {3, 4, 8, 0, 0, 0, 0, 0}, /* 0x8C (10001100) */ + {1, 3, 4, 8, 0, 0, 0, 0}, /* 0x8D (10001101) */ + {2, 3, 4, 8, 0, 0, 0, 0}, /* 0x8E (10001110) */ + {1, 2, 3, 4, 8, 0, 0, 0}, /* 0x8F (10001111) */ + {5, 8, 0, 0, 0, 0, 0, 0}, /* 0x90 (10010000) */ + {1, 5, 8, 0, 0, 0, 0, 0}, /* 0x91 (10010001) */ + {2, 5, 8, 0, 0, 0, 0, 0}, /* 0x92 (10010010) */ + {1, 2, 5, 8, 0, 0, 0, 0}, /* 0x93 (10010011) */ + {3, 5, 8, 0, 0, 0, 0, 0}, /* 0x94 (10010100) */ + {1, 3, 5, 8, 0, 0, 0, 0}, /* 0x95 (10010101) */ + {2, 3, 5, 8, 0, 0, 0, 0}, /* 0x96 (10010110) */ + {1, 2, 3, 5, 8, 0, 0, 0}, /* 0x97 (10010111) */ + {4, 5, 8, 0, 0, 0, 0, 0}, /* 0x98 (10011000) */ + {1, 4, 5, 8, 0, 0, 0, 0}, /* 0x99 (10011001) */ + {2, 4, 5, 8, 0, 0, 0, 0}, /* 0x9A (10011010) */ + {1, 2, 4, 5, 8, 0, 0, 0}, /* 0x9B (10011011) */ + {3, 4, 5, 8, 0, 0, 0, 0}, /* 0x9C (10011100) */ + {1, 3, 4, 5, 8, 0, 0, 0}, /* 0x9D (10011101) */ + {2, 3, 4, 5, 8, 0, 0, 0}, /* 0x9E (10011110) */ + {1, 2, 3, 4, 5, 8, 0, 0}, /* 0x9F (10011111) */ + {6, 8, 0, 0, 0, 0, 0, 0}, /* 0xA0 (10100000) */ + {1, 6, 8, 0, 0, 0, 0, 0}, /* 0xA1 (10100001) */ + {2, 6, 8, 0, 0, 0, 0, 0}, /* 0xA2 (10100010) */ + {1, 2, 6, 8, 0, 0, 0, 0}, /* 0xA3 (10100011) */ + {3, 6, 8, 0, 0, 0, 0, 0}, /* 0xA4 (10100100) */ + {1, 3, 6, 8, 0, 0, 0, 0}, /* 0xA5 (10100101) */ + {2, 3, 6, 8, 0, 0, 0, 0}, /* 0xA6 (10100110) */ + {1, 2, 3, 6, 8, 0, 0, 0}, /* 0xA7 (10100111) */ + {4, 6, 8, 0, 0, 0, 0, 0}, /* 0xA8 (10101000) */ + {1, 4, 6, 8, 0, 0, 0, 0}, /* 0xA9 (10101001) */ + {2, 4, 6, 8, 0, 0, 0, 0}, /* 0xAA (10101010) */ + {1, 2, 4, 6, 8, 0, 0, 0}, /* 0xAB (10101011) */ + {3, 4, 6, 8, 0, 0, 0, 0}, /* 0xAC (10101100) */ + {1, 3, 4, 6, 8, 0, 0, 0}, /* 0xAD (10101101) */ + {2, 3, 4, 6, 8, 0, 0, 0}, /* 0xAE (10101110) */ + {1, 2, 3, 4, 6, 8, 0, 0}, /* 0xAF (10101111) */ + {5, 6, 8, 0, 0, 0, 0, 0}, /* 0xB0 (10110000) */ + {1, 5, 6, 8, 0, 0, 0, 0}, /* 0xB1 (10110001) */ + {2, 5, 6, 8, 0, 0, 0, 0}, /* 0xB2 (10110010) */ + {1, 2, 5, 6, 8, 0, 0, 0}, /* 0xB3 (10110011) */ + {3, 5, 6, 8, 0, 0, 0, 0}, /* 0xB4 (10110100) */ + {1, 3, 5, 6, 8, 0, 0, 0}, /* 0xB5 (10110101) */ + {2, 3, 5, 6, 8, 0, 0, 0}, /* 0xB6 (10110110) */ + {1, 2, 3, 5, 6, 8, 0, 0}, /* 0xB7 (10110111) */ + {4, 5, 6, 8, 0, 0, 0, 0}, /* 0xB8 (10111000) */ + {1, 4, 5, 6, 8, 0, 0, 0}, /* 0xB9 (10111001) */ + {2, 4, 5, 6, 8, 0, 0, 0}, /* 0xBA (10111010) */ + {1, 2, 4, 5, 6, 8, 0, 0}, /* 0xBB (10111011) */ + {3, 4, 5, 6, 8, 0, 0, 0}, /* 0xBC (10111100) */ + {1, 3, 4, 5, 6, 8, 0, 0}, /* 0xBD (10111101) */ + {2, 3, 4, 5, 6, 8, 0, 0}, /* 0xBE (10111110) */ + {1, 2, 3, 4, 5, 6, 8, 0}, /* 0xBF (10111111) */ + {7, 8, 0, 0, 0, 0, 0, 0}, /* 0xC0 (11000000) */ + {1, 7, 8, 0, 0, 0, 0, 0}, /* 0xC1 (11000001) */ + {2, 7, 8, 0, 0, 0, 0, 0}, /* 0xC2 (11000010) */ + {1, 2, 7, 8, 0, 0, 0, 0}, /* 0xC3 (11000011) */ + {3, 7, 8, 0, 0, 0, 0, 0}, /* 0xC4 (11000100) */ + {1, 3, 7, 8, 0, 0, 0, 0}, /* 0xC5 (11000101) */ + {2, 3, 7, 8, 0, 0, 0, 0}, /* 0xC6 (11000110) */ + {1, 2, 3, 7, 8, 0, 0, 0}, /* 0xC7 (11000111) */ + {4, 7, 8, 0, 0, 0, 0, 0}, /* 0xC8 (11001000) */ + {1, 4, 7, 8, 0, 0, 0, 0}, /* 0xC9 (11001001) */ + {2, 4, 7, 8, 0, 0, 0, 0}, /* 0xCA (11001010) */ + {1, 2, 4, 7, 8, 0, 0, 0}, /* 0xCB (11001011) */ + {3, 4, 7, 8, 0, 0, 0, 0}, /* 0xCC (11001100) */ + {1, 3, 4, 7, 8, 0, 0, 0}, /* 0xCD (11001101) */ + {2, 3, 4, 7, 8, 0, 0, 0}, /* 0xCE (11001110) */ + {1, 2, 3, 4, 7, 8, 0, 0}, /* 0xCF (11001111) */ + {5, 7, 8, 0, 0, 0, 0, 0}, /* 0xD0 (11010000) */ + {1, 5, 7, 8, 0, 0, 0, 0}, /* 0xD1 (11010001) */ + {2, 5, 7, 8, 0, 0, 0, 0}, /* 0xD2 (11010010) */ + {1, 2, 5, 7, 8, 0, 0, 0}, /* 0xD3 (11010011) */ + {3, 5, 7, 8, 0, 0, 0, 0}, /* 0xD4 (11010100) */ + {1, 3, 5, 7, 8, 0, 0, 0}, /* 0xD5 (11010101) */ + {2, 3, 5, 7, 8, 0, 0, 0}, /* 0xD6 (11010110) */ + {1, 2, 3, 5, 7, 8, 0, 0}, /* 0xD7 (11010111) */ + {4, 5, 7, 8, 0, 0, 0, 0}, /* 0xD8 (11011000) */ + {1, 4, 5, 7, 8, 0, 0, 0}, /* 0xD9 (11011001) */ + {2, 4, 5, 7, 8, 0, 0, 0}, /* 0xDA (11011010) */ + {1, 2, 4, 5, 7, 8, 0, 0}, /* 0xDB (11011011) */ + {3, 4, 5, 7, 8, 0, 0, 0}, /* 0xDC (11011100) */ + {1, 3, 4, 5, 7, 8, 0, 0}, /* 0xDD (11011101) */ + {2, 3, 4, 5, 7, 8, 0, 0}, /* 0xDE (11011110) */ + {1, 2, 3, 4, 5, 7, 8, 0}, /* 0xDF (11011111) */ + {6, 7, 8, 0, 0, 0, 0, 0}, /* 0xE0 (11100000) */ + {1, 6, 7, 8, 0, 0, 0, 0}, /* 0xE1 (11100001) */ + {2, 6, 7, 8, 0, 0, 0, 0}, /* 0xE2 (11100010) */ + {1, 2, 6, 7, 8, 0, 0, 0}, /* 0xE3 (11100011) */ + {3, 6, 7, 8, 0, 0, 0, 0}, /* 0xE4 (11100100) */ + {1, 3, 6, 7, 8, 0, 0, 0}, /* 0xE5 (11100101) */ + {2, 3, 6, 7, 8, 0, 0, 0}, /* 0xE6 (11100110) */ + {1, 2, 3, 6, 7, 8, 0, 0}, /* 0xE7 (11100111) */ + {4, 6, 7, 8, 0, 0, 0, 0}, /* 0xE8 (11101000) */ + {1, 4, 6, 7, 8, 0, 0, 0}, /* 0xE9 (11101001) */ + {2, 4, 6, 7, 8, 0, 0, 0}, /* 0xEA (11101010) */ + {1, 2, 4, 6, 7, 8, 0, 0}, /* 0xEB (11101011) */ + {3, 4, 6, 7, 8, 0, 0, 0}, /* 0xEC (11101100) */ + {1, 3, 4, 6, 7, 8, 0, 0}, /* 0xED (11101101) */ + {2, 3, 4, 6, 7, 8, 0, 0}, /* 0xEE (11101110) */ + {1, 2, 3, 4, 6, 7, 8, 0}, /* 0xEF (11101111) */ + {5, 6, 7, 8, 0, 0, 0, 0}, /* 0xF0 (11110000) */ + {1, 5, 6, 7, 8, 0, 0, 0}, /* 0xF1 (11110001) */ + {2, 5, 6, 7, 8, 0, 0, 0}, /* 0xF2 (11110010) */ + {1, 2, 5, 6, 7, 8, 0, 0}, /* 0xF3 (11110011) */ + {3, 5, 6, 7, 8, 0, 0, 0}, /* 0xF4 (11110100) */ + {1, 3, 5, 6, 7, 8, 0, 0}, /* 0xF5 (11110101) */ + {2, 3, 5, 6, 7, 8, 0, 0}, /* 0xF6 (11110110) */ + {1, 2, 3, 5, 6, 7, 8, 0}, /* 0xF7 (11110111) */ + {4, 5, 6, 7, 8, 0, 0, 0}, /* 0xF8 (11111000) */ + {1, 4, 5, 6, 7, 8, 0, 0}, /* 0xF9 (11111001) */ + {2, 4, 5, 6, 7, 8, 0, 0}, /* 0xFA (11111010) */ + {1, 2, 4, 5, 6, 7, 8, 0}, /* 0xFB (11111011) */ + {3, 4, 5, 6, 7, 8, 0, 0}, /* 0xFC (11111100) */ + {1, 3, 4, 5, 6, 7, 8, 0}, /* 0xFD (11111101) */ + {2, 3, 4, 5, 6, 7, 8, 0}, /* 0xFE (11111110) */ + {1, 2, 3, 4, 5, 6, 7, 8} /* 0xFF (11111111) */ +}; + +#endif // #if CROARING_IS_X64 + +#if CROARING_IS_X64 +// same as vecDecodeTable but in 16 bits +ALIGNED(32) +static uint16_t vecDecodeTable_uint16[256][8] = { + {0, 0, 0, 0, 0, 0, 0, 0}, /* 0x00 (00000000) */ + {1, 0, 0, 0, 0, 0, 0, 0}, /* 0x01 (00000001) */ + {2, 0, 0, 0, 0, 0, 0, 0}, /* 0x02 (00000010) */ + {1, 2, 0, 0, 0, 0, 0, 0}, /* 0x03 (00000011) */ + {3, 0, 0, 0, 0, 0, 0, 0}, /* 0x04 (00000100) */ + {1, 3, 0, 0, 0, 0, 0, 0}, /* 0x05 (00000101) */ + {2, 3, 0, 0, 0, 0, 0, 0}, /* 0x06 (00000110) */ + {1, 2, 3, 0, 0, 0, 0, 0}, /* 0x07 (00000111) */ + {4, 0, 0, 0, 0, 0, 0, 0}, /* 0x08 (00001000) */ + {1, 4, 0, 0, 0, 0, 0, 0}, /* 0x09 (00001001) */ + {2, 4, 0, 0, 0, 0, 0, 0}, /* 0x0A (00001010) */ + {1, 2, 4, 0, 0, 0, 0, 0}, /* 0x0B (00001011) */ + {3, 4, 0, 0, 0, 0, 0, 0}, /* 0x0C (00001100) */ + {1, 3, 4, 0, 0, 0, 0, 0}, /* 0x0D (00001101) */ + {2, 3, 4, 0, 0, 0, 0, 0}, /* 0x0E (00001110) */ + {1, 2, 3, 4, 0, 0, 0, 0}, /* 0x0F (00001111) */ + {5, 0, 0, 0, 0, 0, 0, 0}, /* 0x10 (00010000) */ + {1, 5, 0, 0, 0, 0, 0, 0}, /* 0x11 (00010001) */ + {2, 5, 0, 0, 0, 0, 0, 0}, /* 0x12 (00010010) */ + {1, 2, 5, 0, 0, 0, 0, 0}, /* 0x13 (00010011) */ + {3, 5, 0, 0, 0, 0, 0, 0}, /* 0x14 (00010100) */ + {1, 3, 5, 0, 0, 0, 0, 0}, /* 0x15 (00010101) */ + {2, 3, 5, 0, 0, 0, 0, 0}, /* 0x16 (00010110) */ + {1, 2, 3, 5, 0, 0, 0, 0}, /* 0x17 (00010111) */ + {4, 5, 0, 0, 0, 0, 0, 0}, /* 0x18 (00011000) */ + {1, 4, 5, 0, 0, 0, 0, 0}, /* 0x19 (00011001) */ + {2, 4, 5, 0, 0, 0, 0, 0}, /* 0x1A (00011010) */ + {1, 2, 4, 5, 0, 0, 0, 0}, /* 0x1B (00011011) */ + {3, 4, 5, 0, 0, 0, 0, 0}, /* 0x1C (00011100) */ + {1, 3, 4, 5, 0, 0, 0, 0}, /* 0x1D (00011101) */ + {2, 3, 4, 5, 0, 0, 0, 0}, /* 0x1E (00011110) */ + {1, 2, 3, 4, 5, 0, 0, 0}, /* 0x1F (00011111) */ + {6, 0, 0, 0, 0, 0, 0, 0}, /* 0x20 (00100000) */ + {1, 6, 0, 0, 0, 0, 0, 0}, /* 0x21 (00100001) */ + {2, 6, 0, 0, 0, 0, 0, 0}, /* 0x22 (00100010) */ + {1, 2, 6, 0, 0, 0, 0, 0}, /* 0x23 (00100011) */ + {3, 6, 0, 0, 0, 0, 0, 0}, /* 0x24 (00100100) */ + {1, 3, 6, 0, 0, 0, 0, 0}, /* 0x25 (00100101) */ + {2, 3, 6, 0, 0, 0, 0, 0}, /* 0x26 (00100110) */ + {1, 2, 3, 6, 0, 0, 0, 0}, /* 0x27 (00100111) */ + {4, 6, 0, 0, 0, 0, 0, 0}, /* 0x28 (00101000) */ + {1, 4, 6, 0, 0, 0, 0, 0}, /* 0x29 (00101001) */ + {2, 4, 6, 0, 0, 0, 0, 0}, /* 0x2A (00101010) */ + {1, 2, 4, 6, 0, 0, 0, 0}, /* 0x2B (00101011) */ + {3, 4, 6, 0, 0, 0, 0, 0}, /* 0x2C (00101100) */ + {1, 3, 4, 6, 0, 0, 0, 0}, /* 0x2D (00101101) */ + {2, 3, 4, 6, 0, 0, 0, 0}, /* 0x2E (00101110) */ + {1, 2, 3, 4, 6, 0, 0, 0}, /* 0x2F (00101111) */ + {5, 6, 0, 0, 0, 0, 0, 0}, /* 0x30 (00110000) */ + {1, 5, 6, 0, 0, 0, 0, 0}, /* 0x31 (00110001) */ + {2, 5, 6, 0, 0, 0, 0, 0}, /* 0x32 (00110010) */ + {1, 2, 5, 6, 0, 0, 0, 0}, /* 0x33 (00110011) */ + {3, 5, 6, 0, 0, 0, 0, 0}, /* 0x34 (00110100) */ + {1, 3, 5, 6, 0, 0, 0, 0}, /* 0x35 (00110101) */ + {2, 3, 5, 6, 0, 0, 0, 0}, /* 0x36 (00110110) */ + {1, 2, 3, 5, 6, 0, 0, 0}, /* 0x37 (00110111) */ + {4, 5, 6, 0, 0, 0, 0, 0}, /* 0x38 (00111000) */ + {1, 4, 5, 6, 0, 0, 0, 0}, /* 0x39 (00111001) */ + {2, 4, 5, 6, 0, 0, 0, 0}, /* 0x3A (00111010) */ + {1, 2, 4, 5, 6, 0, 0, 0}, /* 0x3B (00111011) */ + {3, 4, 5, 6, 0, 0, 0, 0}, /* 0x3C (00111100) */ + {1, 3, 4, 5, 6, 0, 0, 0}, /* 0x3D (00111101) */ + {2, 3, 4, 5, 6, 0, 0, 0}, /* 0x3E (00111110) */ + {1, 2, 3, 4, 5, 6, 0, 0}, /* 0x3F (00111111) */ + {7, 0, 0, 0, 0, 0, 0, 0}, /* 0x40 (01000000) */ + {1, 7, 0, 0, 0, 0, 0, 0}, /* 0x41 (01000001) */ + {2, 7, 0, 0, 0, 0, 0, 0}, /* 0x42 (01000010) */ + {1, 2, 7, 0, 0, 0, 0, 0}, /* 0x43 (01000011) */ + {3, 7, 0, 0, 0, 0, 0, 0}, /* 0x44 (01000100) */ + {1, 3, 7, 0, 0, 0, 0, 0}, /* 0x45 (01000101) */ + {2, 3, 7, 0, 0, 0, 0, 0}, /* 0x46 (01000110) */ + {1, 2, 3, 7, 0, 0, 0, 0}, /* 0x47 (01000111) */ + {4, 7, 0, 0, 0, 0, 0, 0}, /* 0x48 (01001000) */ + {1, 4, 7, 0, 0, 0, 0, 0}, /* 0x49 (01001001) */ + {2, 4, 7, 0, 0, 0, 0, 0}, /* 0x4A (01001010) */ + {1, 2, 4, 7, 0, 0, 0, 0}, /* 0x4B (01001011) */ + {3, 4, 7, 0, 0, 0, 0, 0}, /* 0x4C (01001100) */ + {1, 3, 4, 7, 0, 0, 0, 0}, /* 0x4D (01001101) */ + {2, 3, 4, 7, 0, 0, 0, 0}, /* 0x4E (01001110) */ + {1, 2, 3, 4, 7, 0, 0, 0}, /* 0x4F (01001111) */ + {5, 7, 0, 0, 0, 0, 0, 0}, /* 0x50 (01010000) */ + {1, 5, 7, 0, 0, 0, 0, 0}, /* 0x51 (01010001) */ + {2, 5, 7, 0, 0, 0, 0, 0}, /* 0x52 (01010010) */ + {1, 2, 5, 7, 0, 0, 0, 0}, /* 0x53 (01010011) */ + {3, 5, 7, 0, 0, 0, 0, 0}, /* 0x54 (01010100) */ + {1, 3, 5, 7, 0, 0, 0, 0}, /* 0x55 (01010101) */ + {2, 3, 5, 7, 0, 0, 0, 0}, /* 0x56 (01010110) */ + {1, 2, 3, 5, 7, 0, 0, 0}, /* 0x57 (01010111) */ + {4, 5, 7, 0, 0, 0, 0, 0}, /* 0x58 (01011000) */ + {1, 4, 5, 7, 0, 0, 0, 0}, /* 0x59 (01011001) */ + {2, 4, 5, 7, 0, 0, 0, 0}, /* 0x5A (01011010) */ + {1, 2, 4, 5, 7, 0, 0, 0}, /* 0x5B (01011011) */ + {3, 4, 5, 7, 0, 0, 0, 0}, /* 0x5C (01011100) */ + {1, 3, 4, 5, 7, 0, 0, 0}, /* 0x5D (01011101) */ + {2, 3, 4, 5, 7, 0, 0, 0}, /* 0x5E (01011110) */ + {1, 2, 3, 4, 5, 7, 0, 0}, /* 0x5F (01011111) */ + {6, 7, 0, 0, 0, 0, 0, 0}, /* 0x60 (01100000) */ + {1, 6, 7, 0, 0, 0, 0, 0}, /* 0x61 (01100001) */ + {2, 6, 7, 0, 0, 0, 0, 0}, /* 0x62 (01100010) */ + {1, 2, 6, 7, 0, 0, 0, 0}, /* 0x63 (01100011) */ + {3, 6, 7, 0, 0, 0, 0, 0}, /* 0x64 (01100100) */ + {1, 3, 6, 7, 0, 0, 0, 0}, /* 0x65 (01100101) */ + {2, 3, 6, 7, 0, 0, 0, 0}, /* 0x66 (01100110) */ + {1, 2, 3, 6, 7, 0, 0, 0}, /* 0x67 (01100111) */ + {4, 6, 7, 0, 0, 0, 0, 0}, /* 0x68 (01101000) */ + {1, 4, 6, 7, 0, 0, 0, 0}, /* 0x69 (01101001) */ + {2, 4, 6, 7, 0, 0, 0, 0}, /* 0x6A (01101010) */ + {1, 2, 4, 6, 7, 0, 0, 0}, /* 0x6B (01101011) */ + {3, 4, 6, 7, 0, 0, 0, 0}, /* 0x6C (01101100) */ + {1, 3, 4, 6, 7, 0, 0, 0}, /* 0x6D (01101101) */ + {2, 3, 4, 6, 7, 0, 0, 0}, /* 0x6E (01101110) */ + {1, 2, 3, 4, 6, 7, 0, 0}, /* 0x6F (01101111) */ + {5, 6, 7, 0, 0, 0, 0, 0}, /* 0x70 (01110000) */ + {1, 5, 6, 7, 0, 0, 0, 0}, /* 0x71 (01110001) */ + {2, 5, 6, 7, 0, 0, 0, 0}, /* 0x72 (01110010) */ + {1, 2, 5, 6, 7, 0, 0, 0}, /* 0x73 (01110011) */ + {3, 5, 6, 7, 0, 0, 0, 0}, /* 0x74 (01110100) */ + {1, 3, 5, 6, 7, 0, 0, 0}, /* 0x75 (01110101) */ + {2, 3, 5, 6, 7, 0, 0, 0}, /* 0x76 (01110110) */ + {1, 2, 3, 5, 6, 7, 0, 0}, /* 0x77 (01110111) */ + {4, 5, 6, 7, 0, 0, 0, 0}, /* 0x78 (01111000) */ + {1, 4, 5, 6, 7, 0, 0, 0}, /* 0x79 (01111001) */ + {2, 4, 5, 6, 7, 0, 0, 0}, /* 0x7A (01111010) */ + {1, 2, 4, 5, 6, 7, 0, 0}, /* 0x7B (01111011) */ + {3, 4, 5, 6, 7, 0, 0, 0}, /* 0x7C (01111100) */ + {1, 3, 4, 5, 6, 7, 0, 0}, /* 0x7D (01111101) */ + {2, 3, 4, 5, 6, 7, 0, 0}, /* 0x7E (01111110) */ + {1, 2, 3, 4, 5, 6, 7, 0}, /* 0x7F (01111111) */ + {8, 0, 0, 0, 0, 0, 0, 0}, /* 0x80 (10000000) */ + {1, 8, 0, 0, 0, 0, 0, 0}, /* 0x81 (10000001) */ + {2, 8, 0, 0, 0, 0, 0, 0}, /* 0x82 (10000010) */ + {1, 2, 8, 0, 0, 0, 0, 0}, /* 0x83 (10000011) */ + {3, 8, 0, 0, 0, 0, 0, 0}, /* 0x84 (10000100) */ + {1, 3, 8, 0, 0, 0, 0, 0}, /* 0x85 (10000101) */ + {2, 3, 8, 0, 0, 0, 0, 0}, /* 0x86 (10000110) */ + {1, 2, 3, 8, 0, 0, 0, 0}, /* 0x87 (10000111) */ + {4, 8, 0, 0, 0, 0, 0, 0}, /* 0x88 (10001000) */ + {1, 4, 8, 0, 0, 0, 0, 0}, /* 0x89 (10001001) */ + {2, 4, 8, 0, 0, 0, 0, 0}, /* 0x8A (10001010) */ + {1, 2, 4, 8, 0, 0, 0, 0}, /* 0x8B (10001011) */ + {3, 4, 8, 0, 0, 0, 0, 0}, /* 0x8C (10001100) */ + {1, 3, 4, 8, 0, 0, 0, 0}, /* 0x8D (10001101) */ + {2, 3, 4, 8, 0, 0, 0, 0}, /* 0x8E (10001110) */ + {1, 2, 3, 4, 8, 0, 0, 0}, /* 0x8F (10001111) */ + {5, 8, 0, 0, 0, 0, 0, 0}, /* 0x90 (10010000) */ + {1, 5, 8, 0, 0, 0, 0, 0}, /* 0x91 (10010001) */ + {2, 5, 8, 0, 0, 0, 0, 0}, /* 0x92 (10010010) */ + {1, 2, 5, 8, 0, 0, 0, 0}, /* 0x93 (10010011) */ + {3, 5, 8, 0, 0, 0, 0, 0}, /* 0x94 (10010100) */ + {1, 3, 5, 8, 0, 0, 0, 0}, /* 0x95 (10010101) */ + {2, 3, 5, 8, 0, 0, 0, 0}, /* 0x96 (10010110) */ + {1, 2, 3, 5, 8, 0, 0, 0}, /* 0x97 (10010111) */ + {4, 5, 8, 0, 0, 0, 0, 0}, /* 0x98 (10011000) */ + {1, 4, 5, 8, 0, 0, 0, 0}, /* 0x99 (10011001) */ + {2, 4, 5, 8, 0, 0, 0, 0}, /* 0x9A (10011010) */ + {1, 2, 4, 5, 8, 0, 0, 0}, /* 0x9B (10011011) */ + {3, 4, 5, 8, 0, 0, 0, 0}, /* 0x9C (10011100) */ + {1, 3, 4, 5, 8, 0, 0, 0}, /* 0x9D (10011101) */ + {2, 3, 4, 5, 8, 0, 0, 0}, /* 0x9E (10011110) */ + {1, 2, 3, 4, 5, 8, 0, 0}, /* 0x9F (10011111) */ + {6, 8, 0, 0, 0, 0, 0, 0}, /* 0xA0 (10100000) */ + {1, 6, 8, 0, 0, 0, 0, 0}, /* 0xA1 (10100001) */ + {2, 6, 8, 0, 0, 0, 0, 0}, /* 0xA2 (10100010) */ + {1, 2, 6, 8, 0, 0, 0, 0}, /* 0xA3 (10100011) */ + {3, 6, 8, 0, 0, 0, 0, 0}, /* 0xA4 (10100100) */ + {1, 3, 6, 8, 0, 0, 0, 0}, /* 0xA5 (10100101) */ + {2, 3, 6, 8, 0, 0, 0, 0}, /* 0xA6 (10100110) */ + {1, 2, 3, 6, 8, 0, 0, 0}, /* 0xA7 (10100111) */ + {4, 6, 8, 0, 0, 0, 0, 0}, /* 0xA8 (10101000) */ + {1, 4, 6, 8, 0, 0, 0, 0}, /* 0xA9 (10101001) */ + {2, 4, 6, 8, 0, 0, 0, 0}, /* 0xAA (10101010) */ + {1, 2, 4, 6, 8, 0, 0, 0}, /* 0xAB (10101011) */ + {3, 4, 6, 8, 0, 0, 0, 0}, /* 0xAC (10101100) */ + {1, 3, 4, 6, 8, 0, 0, 0}, /* 0xAD (10101101) */ + {2, 3, 4, 6, 8, 0, 0, 0}, /* 0xAE (10101110) */ + {1, 2, 3, 4, 6, 8, 0, 0}, /* 0xAF (10101111) */ + {5, 6, 8, 0, 0, 0, 0, 0}, /* 0xB0 (10110000) */ + {1, 5, 6, 8, 0, 0, 0, 0}, /* 0xB1 (10110001) */ + {2, 5, 6, 8, 0, 0, 0, 0}, /* 0xB2 (10110010) */ + {1, 2, 5, 6, 8, 0, 0, 0}, /* 0xB3 (10110011) */ + {3, 5, 6, 8, 0, 0, 0, 0}, /* 0xB4 (10110100) */ + {1, 3, 5, 6, 8, 0, 0, 0}, /* 0xB5 (10110101) */ + {2, 3, 5, 6, 8, 0, 0, 0}, /* 0xB6 (10110110) */ + {1, 2, 3, 5, 6, 8, 0, 0}, /* 0xB7 (10110111) */ + {4, 5, 6, 8, 0, 0, 0, 0}, /* 0xB8 (10111000) */ + {1, 4, 5, 6, 8, 0, 0, 0}, /* 0xB9 (10111001) */ + {2, 4, 5, 6, 8, 0, 0, 0}, /* 0xBA (10111010) */ + {1, 2, 4, 5, 6, 8, 0, 0}, /* 0xBB (10111011) */ + {3, 4, 5, 6, 8, 0, 0, 0}, /* 0xBC (10111100) */ + {1, 3, 4, 5, 6, 8, 0, 0}, /* 0xBD (10111101) */ + {2, 3, 4, 5, 6, 8, 0, 0}, /* 0xBE (10111110) */ + {1, 2, 3, 4, 5, 6, 8, 0}, /* 0xBF (10111111) */ + {7, 8, 0, 0, 0, 0, 0, 0}, /* 0xC0 (11000000) */ + {1, 7, 8, 0, 0, 0, 0, 0}, /* 0xC1 (11000001) */ + {2, 7, 8, 0, 0, 0, 0, 0}, /* 0xC2 (11000010) */ + {1, 2, 7, 8, 0, 0, 0, 0}, /* 0xC3 (11000011) */ + {3, 7, 8, 0, 0, 0, 0, 0}, /* 0xC4 (11000100) */ + {1, 3, 7, 8, 0, 0, 0, 0}, /* 0xC5 (11000101) */ + {2, 3, 7, 8, 0, 0, 0, 0}, /* 0xC6 (11000110) */ + {1, 2, 3, 7, 8, 0, 0, 0}, /* 0xC7 (11000111) */ + {4, 7, 8, 0, 0, 0, 0, 0}, /* 0xC8 (11001000) */ + {1, 4, 7, 8, 0, 0, 0, 0}, /* 0xC9 (11001001) */ + {2, 4, 7, 8, 0, 0, 0, 0}, /* 0xCA (11001010) */ + {1, 2, 4, 7, 8, 0, 0, 0}, /* 0xCB (11001011) */ + {3, 4, 7, 8, 0, 0, 0, 0}, /* 0xCC (11001100) */ + {1, 3, 4, 7, 8, 0, 0, 0}, /* 0xCD (11001101) */ + {2, 3, 4, 7, 8, 0, 0, 0}, /* 0xCE (11001110) */ + {1, 2, 3, 4, 7, 8, 0, 0}, /* 0xCF (11001111) */ + {5, 7, 8, 0, 0, 0, 0, 0}, /* 0xD0 (11010000) */ + {1, 5, 7, 8, 0, 0, 0, 0}, /* 0xD1 (11010001) */ + {2, 5, 7, 8, 0, 0, 0, 0}, /* 0xD2 (11010010) */ + {1, 2, 5, 7, 8, 0, 0, 0}, /* 0xD3 (11010011) */ + {3, 5, 7, 8, 0, 0, 0, 0}, /* 0xD4 (11010100) */ + {1, 3, 5, 7, 8, 0, 0, 0}, /* 0xD5 (11010101) */ + {2, 3, 5, 7, 8, 0, 0, 0}, /* 0xD6 (11010110) */ + {1, 2, 3, 5, 7, 8, 0, 0}, /* 0xD7 (11010111) */ + {4, 5, 7, 8, 0, 0, 0, 0}, /* 0xD8 (11011000) */ + {1, 4, 5, 7, 8, 0, 0, 0}, /* 0xD9 (11011001) */ + {2, 4, 5, 7, 8, 0, 0, 0}, /* 0xDA (11011010) */ + {1, 2, 4, 5, 7, 8, 0, 0}, /* 0xDB (11011011) */ + {3, 4, 5, 7, 8, 0, 0, 0}, /* 0xDC (11011100) */ + {1, 3, 4, 5, 7, 8, 0, 0}, /* 0xDD (11011101) */ + {2, 3, 4, 5, 7, 8, 0, 0}, /* 0xDE (11011110) */ + {1, 2, 3, 4, 5, 7, 8, 0}, /* 0xDF (11011111) */ + {6, 7, 8, 0, 0, 0, 0, 0}, /* 0xE0 (11100000) */ + {1, 6, 7, 8, 0, 0, 0, 0}, /* 0xE1 (11100001) */ + {2, 6, 7, 8, 0, 0, 0, 0}, /* 0xE2 (11100010) */ + {1, 2, 6, 7, 8, 0, 0, 0}, /* 0xE3 (11100011) */ + {3, 6, 7, 8, 0, 0, 0, 0}, /* 0xE4 (11100100) */ + {1, 3, 6, 7, 8, 0, 0, 0}, /* 0xE5 (11100101) */ + {2, 3, 6, 7, 8, 0, 0, 0}, /* 0xE6 (11100110) */ + {1, 2, 3, 6, 7, 8, 0, 0}, /* 0xE7 (11100111) */ + {4, 6, 7, 8, 0, 0, 0, 0}, /* 0xE8 (11101000) */ + {1, 4, 6, 7, 8, 0, 0, 0}, /* 0xE9 (11101001) */ + {2, 4, 6, 7, 8, 0, 0, 0}, /* 0xEA (11101010) */ + {1, 2, 4, 6, 7, 8, 0, 0}, /* 0xEB (11101011) */ + {3, 4, 6, 7, 8, 0, 0, 0}, /* 0xEC (11101100) */ + {1, 3, 4, 6, 7, 8, 0, 0}, /* 0xED (11101101) */ + {2, 3, 4, 6, 7, 8, 0, 0}, /* 0xEE (11101110) */ + {1, 2, 3, 4, 6, 7, 8, 0}, /* 0xEF (11101111) */ + {5, 6, 7, 8, 0, 0, 0, 0}, /* 0xF0 (11110000) */ + {1, 5, 6, 7, 8, 0, 0, 0}, /* 0xF1 (11110001) */ + {2, 5, 6, 7, 8, 0, 0, 0}, /* 0xF2 (11110010) */ + {1, 2, 5, 6, 7, 8, 0, 0}, /* 0xF3 (11110011) */ + {3, 5, 6, 7, 8, 0, 0, 0}, /* 0xF4 (11110100) */ + {1, 3, 5, 6, 7, 8, 0, 0}, /* 0xF5 (11110101) */ + {2, 3, 5, 6, 7, 8, 0, 0}, /* 0xF6 (11110110) */ + {1, 2, 3, 5, 6, 7, 8, 0}, /* 0xF7 (11110111) */ + {4, 5, 6, 7, 8, 0, 0, 0}, /* 0xF8 (11111000) */ + {1, 4, 5, 6, 7, 8, 0, 0}, /* 0xF9 (11111001) */ + {2, 4, 5, 6, 7, 8, 0, 0}, /* 0xFA (11111010) */ + {1, 2, 4, 5, 6, 7, 8, 0}, /* 0xFB (11111011) */ + {3, 4, 5, 6, 7, 8, 0, 0}, /* 0xFC (11111100) */ + {1, 3, 4, 5, 6, 7, 8, 0}, /* 0xFD (11111101) */ + {2, 3, 4, 5, 6, 7, 8, 0}, /* 0xFE (11111110) */ + {1, 2, 3, 4, 5, 6, 7, 8} /* 0xFF (11111111) */ +}; + +#endif + +#if CROARING_IS_X64 +#if CROARING_COMPILER_SUPPORTS_AVX512 +CROARING_TARGET_AVX512 +const uint8_t vbmi2_table[64] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63}; +size_t bitset_extract_setbits_avx512(const uint64_t *words, size_t length, + uint32_t *vout, size_t outcapacity, + uint32_t base) { + uint32_t *out = (uint32_t *)vout; + uint32_t *initout = out; + uint32_t *safeout = out + outcapacity; + __m512i base_v = _mm512_set1_epi32(base); + __m512i index_table = _mm512_loadu_si512(vbmi2_table); + size_t i = 0; + + for (; (i < length) && ((out + 64) < safeout); i += 1) { + uint64_t v = words[i]; + __m512i vec = _mm512_maskz_compress_epi8(v, index_table); + + uint8_t advance = (uint8_t)roaring_hamming(v); + + __m512i vbase = + _mm512_add_epi32(base_v, _mm512_set1_epi32((int)(i * 64))); + __m512i r1 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec, 0)); + __m512i r2 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec, 1)); + __m512i r3 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec, 2)); + __m512i r4 = _mm512_cvtepi8_epi32(_mm512_extracti32x4_epi32(vec, 3)); + + r1 = _mm512_add_epi32(r1, vbase); + r2 = _mm512_add_epi32(r2, vbase); + r3 = _mm512_add_epi32(r3, vbase); + r4 = _mm512_add_epi32(r4, vbase); + _mm512_storeu_si512((__m512i *)out, r1); + _mm512_storeu_si512((__m512i *)(out + 16), r2); + _mm512_storeu_si512((__m512i *)(out + 32), r3); + _mm512_storeu_si512((__m512i *)(out + 48), r4); + + out += advance; + } + + base += i * 64; + + for (; (i < length) && (out < safeout); ++i) { + uint64_t w = words[i]; + while ((w != 0) && (out < safeout)) { + uint64_t t = + w & (~w + 1); // on x64, should compile to BLSI (careful: the + // Intel compiler seems to fail) + int r = + roaring_trailing_zeroes(w); // on x64, should compile to TZCNT + uint32_t val = r + base; + memcpy(out, &val, + sizeof(uint32_t)); // should be compiled as a MOV on x64 + out++; + w ^= t; + } + base += 64; + } + + return out - initout; +} + +// Reference: +// https://lemire.me/blog/2022/05/10/faster-bitset-decoding-using-intel-avx-512/ +size_t bitset_extract_setbits_avx512_uint16(const uint64_t *array, + size_t length, uint16_t *vout, + size_t capacity, uint16_t base) { + uint16_t *out = (uint16_t *)vout; + uint16_t *initout = out; + uint16_t *safeout = vout + capacity; + + __m512i base_v = _mm512_set1_epi16(base); + __m512i index_table = _mm512_loadu_si512(vbmi2_table); + size_t i = 0; + + for (; (i < length) && ((out + 64) < safeout); i++) { + uint64_t v = array[i]; + __m512i vec = _mm512_maskz_compress_epi8(v, index_table); + + uint8_t advance = (uint8_t)roaring_hamming(v); + + __m512i vbase = + _mm512_add_epi16(base_v, _mm512_set1_epi16((short)(i * 64))); + __m512i r1 = _mm512_cvtepi8_epi16(_mm512_extracti32x8_epi32(vec, 0)); + __m512i r2 = _mm512_cvtepi8_epi16(_mm512_extracti32x8_epi32(vec, 1)); + + r1 = _mm512_add_epi16(r1, vbase); + r2 = _mm512_add_epi16(r2, vbase); + + _mm512_storeu_si512((__m512i *)out, r1); + _mm512_storeu_si512((__m512i *)(out + 32), r2); + out += advance; + } + + base += i * 64; + + for (; (i < length) && (out < safeout); ++i) { + uint64_t w = array[i]; + while ((w != 0) && (out < safeout)) { + uint64_t t = + w & (~w + 1); // on x64, should compile to BLSI (careful: the + // Intel compiler seems to fail) + int r = + roaring_trailing_zeroes(w); // on x64, should compile to TZCNT + uint32_t val = r + base; + memcpy(out, &val, sizeof(uint16_t)); + out++; + w ^= t; + } + base += 64; + } + + return out - initout; +} +CROARING_UNTARGET_AVX512 +#endif + +CROARING_TARGET_AVX2 +size_t bitset_extract_setbits_avx2(const uint64_t *words, size_t length, + uint32_t *out, size_t outcapacity, + uint32_t base) { + uint32_t *initout = out; + __m256i baseVec = _mm256_set1_epi32(base - 1); + __m256i incVec = _mm256_set1_epi32(64); + __m256i add8 = _mm256_set1_epi32(8); + uint32_t *safeout = out + outcapacity; + size_t i = 0; + for (; (i < length) && (out + 64 <= safeout); ++i) { + uint64_t w = words[i]; + if (w == 0) { + baseVec = _mm256_add_epi32(baseVec, incVec); + } else { + for (int k = 0; k < 4; ++k) { + uint8_t byteA = (uint8_t)w; + uint8_t byteB = (uint8_t)(w >> 8); + w >>= 16; + __m256i vecA = + _mm256_loadu_si256((const __m256i *)vecDecodeTable[byteA]); + __m256i vecB = + _mm256_loadu_si256((const __m256i *)vecDecodeTable[byteB]); + uint8_t advanceA = lengthTable[byteA]; + uint8_t advanceB = lengthTable[byteB]; + vecA = _mm256_add_epi32(baseVec, vecA); + baseVec = _mm256_add_epi32(baseVec, add8); + vecB = _mm256_add_epi32(baseVec, vecB); + baseVec = _mm256_add_epi32(baseVec, add8); + _mm256_storeu_si256((__m256i *)out, vecA); + out += advanceA; + _mm256_storeu_si256((__m256i *)out, vecB); + out += advanceB; + } + } + } + base += i * 64; + for (; (i < length) && (out < safeout); ++i) { + uint64_t w = words[i]; + while ((w != 0) && (out < safeout)) { + uint64_t t = + w & (~w + 1); // on x64, should compile to BLSI (careful: the + // Intel compiler seems to fail) + int r = + roaring_trailing_zeroes(w); // on x64, should compile to TZCNT + uint32_t val = r + base; + memcpy(out, &val, + sizeof(uint32_t)); // should be compiled as a MOV on x64 + out++; + w ^= t; + } + base += 64; + } + return out - initout; +} +CROARING_UNTARGET_AVX2 +#endif // CROARING_IS_X64 + +size_t bitset_extract_setbits(const uint64_t *words, size_t length, + uint32_t *out, uint32_t base) { + int outpos = 0; + for (size_t i = 0; i < length; ++i) { + uint64_t w = words[i]; + while (w != 0) { + uint64_t t = + w & (~w + 1); // on x64, should compile to BLSI (careful: the + // Intel compiler seems to fail) + int r = + roaring_trailing_zeroes(w); // on x64, should compile to TZCNT + uint32_t val = r + base; + memcpy(out + outpos, &val, + sizeof(uint32_t)); // should be compiled as a MOV on x64 + outpos++; + w ^= t; + } + base += 64; + } + return outpos; +} + +size_t bitset_extract_intersection_setbits_uint16( + const uint64_t *__restrict__ words1, const uint64_t *__restrict__ words2, + size_t length, uint16_t *out, uint16_t base) { + int outpos = 0; + for (size_t i = 0; i < length; ++i) { + uint64_t w = words1[i] & words2[i]; + while (w != 0) { + uint64_t t = w & (~w + 1); + int r = roaring_trailing_zeroes(w); + out[outpos++] = (uint16_t)(r + base); + w ^= t; + } + base += 64; + } + return outpos; +} + +#if CROARING_IS_X64 +/* + * Given a bitset containing "length" 64-bit words, write out the position + * of all the set bits to "out" as 16-bit integers, values start at "base" (can + *be set to zero). + * + * The "out" pointer should be sufficient to store the actual number of bits + *set. + * + * Returns how many values were actually decoded. + * + * This function uses SSE decoding. + */ +CROARING_TARGET_AVX2 +size_t bitset_extract_setbits_sse_uint16(const uint64_t *words, size_t length, + uint16_t *out, size_t outcapacity, + uint16_t base) { + uint16_t *initout = out; + __m128i baseVec = _mm_set1_epi16(base - 1); + __m128i incVec = _mm_set1_epi16(64); + __m128i add8 = _mm_set1_epi16(8); + uint16_t *safeout = out + outcapacity; + const int numberofbytes = 2; // process two bytes at a time + size_t i = 0; + for (; (i < length) && (out + numberofbytes * 8 <= safeout); ++i) { + uint64_t w = words[i]; + if (w == 0) { + baseVec = _mm_add_epi16(baseVec, incVec); + } else { + for (int k = 0; k < 4; ++k) { + uint8_t byteA = (uint8_t)w; + uint8_t byteB = (uint8_t)(w >> 8); + w >>= 16; + __m128i vecA = _mm_loadu_si128( + (const __m128i *)vecDecodeTable_uint16[byteA]); + __m128i vecB = _mm_loadu_si128( + (const __m128i *)vecDecodeTable_uint16[byteB]); + uint8_t advanceA = lengthTable[byteA]; + uint8_t advanceB = lengthTable[byteB]; + vecA = _mm_add_epi16(baseVec, vecA); + baseVec = _mm_add_epi16(baseVec, add8); + vecB = _mm_add_epi16(baseVec, vecB); + baseVec = _mm_add_epi16(baseVec, add8); + _mm_storeu_si128((__m128i *)out, vecA); + out += advanceA; + _mm_storeu_si128((__m128i *)out, vecB); + out += advanceB; + } + } + } + base += (uint16_t)(i * 64); + for (; (i < length) && (out < safeout); ++i) { + uint64_t w = words[i]; + while ((w != 0) && (out < safeout)) { + uint64_t t = w & (~w + 1); + int r = roaring_trailing_zeroes(w); + *out = (uint16_t)(r + base); + out++; + w ^= t; + } + base += 64; + } + return out - initout; +} +CROARING_UNTARGET_AVX2 +#endif + +/* + * Given a bitset containing "length" 64-bit words, write out the position + * of all the set bits to "out", values start at "base" (can be set to zero). + * + * The "out" pointer should be sufficient to store the actual number of bits + *set. + * + * Returns how many values were actually decoded. + */ +size_t bitset_extract_setbits_uint16(const uint64_t *words, size_t length, + uint16_t *out, uint16_t base) { + int outpos = 0; + for (size_t i = 0; i < length; ++i) { + uint64_t w = words[i]; + while (w != 0) { + uint64_t t = w & (~w + 1); + int r = roaring_trailing_zeroes(w); + out[outpos++] = (uint16_t)(r + base); + w ^= t; + } + base += 64; + } + return outpos; +} + +#if defined(CROARING_ASMBITMANIPOPTIMIZATION) && defined(CROARING_IS_X64) + +static inline uint64_t _asm_bitset_set_list_withcard(uint64_t *words, + uint64_t card, + const uint16_t *list, + uint64_t length) { + uint64_t offset, load, pos; + uint64_t shift = 6; + const uint16_t *end = list + length; + if (!length) return card; + // TODO: could unroll for performance, see bitset_set_list + // bts is not available as an intrinsic in GCC + __asm volatile( + "1:\n" + "movzwq (%[list]), %[pos]\n" + "shrx %[shift], %[pos], %[offset]\n" + "mov (%[words],%[offset],8), %[load]\n" + "bts %[pos], %[load]\n" + "mov %[load], (%[words],%[offset],8)\n" + "sbb $-1, %[card]\n" + "add $2, %[list]\n" + "cmp %[list], %[end]\n" + "jnz 1b" + : [card] "+&r"(card), [list] "+&r"(list), [load] "=&r"(load), + [pos] "=&r"(pos), [offset] "=&r"(offset) + : [end] "r"(end), [words] "r"(words), [shift] "r"(shift)); + return card; +} + +static inline void _asm_bitset_set_list(uint64_t *words, const uint16_t *list, + uint64_t length) { + uint64_t pos; + const uint16_t *end = list + length; + + uint64_t shift = 6; + uint64_t offset; + uint64_t load; + for (; list + 3 < end; list += 4) { + pos = list[0]; + __asm volatile( + "shrx %[shift], %[pos], %[offset]\n" + "mov (%[words],%[offset],8), %[load]\n" + "bts %[pos], %[load]\n" + "mov %[load], (%[words],%[offset],8)" + : [load] "=&r"(load), [offset] "=&r"(offset) + : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); + pos = list[1]; + __asm volatile( + "shrx %[shift], %[pos], %[offset]\n" + "mov (%[words],%[offset],8), %[load]\n" + "bts %[pos], %[load]\n" + "mov %[load], (%[words],%[offset],8)" + : [load] "=&r"(load), [offset] "=&r"(offset) + : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); + pos = list[2]; + __asm volatile( + "shrx %[shift], %[pos], %[offset]\n" + "mov (%[words],%[offset],8), %[load]\n" + "bts %[pos], %[load]\n" + "mov %[load], (%[words],%[offset],8)" + : [load] "=&r"(load), [offset] "=&r"(offset) + : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); + pos = list[3]; + __asm volatile( + "shrx %[shift], %[pos], %[offset]\n" + "mov (%[words],%[offset],8), %[load]\n" + "bts %[pos], %[load]\n" + "mov %[load], (%[words],%[offset],8)" + : [load] "=&r"(load), [offset] "=&r"(offset) + : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); + } + + while (list != end) { + pos = list[0]; + __asm volatile( + "shrx %[shift], %[pos], %[offset]\n" + "mov (%[words],%[offset],8), %[load]\n" + "bts %[pos], %[load]\n" + "mov %[load], (%[words],%[offset],8)" + : [load] "=&r"(load), [offset] "=&r"(offset) + : [words] "r"(words), [shift] "r"(shift), [pos] "r"(pos)); + list++; + } +} + +static inline uint64_t _asm_bitset_clear_list(uint64_t *words, uint64_t card, + const uint16_t *list, + uint64_t length) { + uint64_t offset, load, pos; + uint64_t shift = 6; + const uint16_t *end = list + length; + if (!length) return card; + // btr is not available as an intrinsic in GCC + __asm volatile( + "1:\n" + "movzwq (%[list]), %[pos]\n" + "shrx %[shift], %[pos], %[offset]\n" + "mov (%[words],%[offset],8), %[load]\n" + "btr %[pos], %[load]\n" + "mov %[load], (%[words],%[offset],8)\n" + "sbb $0, %[card]\n" + "add $2, %[list]\n" + "cmp %[list], %[end]\n" + "jnz 1b" + : [card] "+&r"(card), [list] "+&r"(list), [load] "=&r"(load), + [pos] "=&r"(pos), [offset] "=&r"(offset) + : [end] "r"(end), [words] "r"(words), [shift] "r"(shift) + : + /* clobbers */ "memory"); + return card; +} + +static inline uint64_t _scalar_bitset_clear_list(uint64_t *words, uint64_t card, + const uint16_t *list, + uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *(const uint16_t *)list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load & ~(UINT64_C(1) << index); + card -= (load ^ newload) >> index; + words[offset] = newload; + list++; + } + return card; +} + +static inline uint64_t _scalar_bitset_set_list_withcard(uint64_t *words, + uint64_t card, + const uint16_t *list, + uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load | (UINT64_C(1) << index); + card += (load ^ newload) >> index; + words[offset] = newload; + list++; + } + return card; +} + +static inline void _scalar_bitset_set_list(uint64_t *words, + const uint16_t *list, + uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load | (UINT64_C(1) << index); + words[offset] = newload; + list++; + } +} + +uint64_t bitset_clear_list(uint64_t *words, uint64_t card, const uint16_t *list, + uint64_t length) { + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + return _asm_bitset_clear_list(words, card, list, length); + } else { + return _scalar_bitset_clear_list(words, card, list, length); + } +} + +uint64_t bitset_set_list_withcard(uint64_t *words, uint64_t card, + const uint16_t *list, uint64_t length) { + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + return _asm_bitset_set_list_withcard(words, card, list, length); + } else { + return _scalar_bitset_set_list_withcard(words, card, list, length); + } +} + +void bitset_set_list(uint64_t *words, const uint16_t *list, uint64_t length) { + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + _asm_bitset_set_list(words, list, length); + } else { + _scalar_bitset_set_list(words, list, length); + } +} +#else +uint64_t bitset_clear_list(uint64_t *words, uint64_t card, const uint16_t *list, + uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *(const uint16_t *)list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load & ~(UINT64_C(1) << index); + card -= (load ^ newload) >> index; + words[offset] = newload; + list++; + } + return card; +} + +uint64_t bitset_set_list_withcard(uint64_t *words, uint64_t card, + const uint16_t *list, uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load | (UINT64_C(1) << index); + card += (load ^ newload) >> index; + words[offset] = newload; + list++; + } + return card; +} + +void bitset_set_list(uint64_t *words, const uint16_t *list, uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load | (UINT64_C(1) << index); + words[offset] = newload; + list++; + } +} + +#endif + +/* flip specified bits */ +/* TODO: consider whether worthwhile to make an asm version */ + +uint64_t bitset_flip_list_withcard(uint64_t *words, uint64_t card, + const uint16_t *list, uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load ^ (UINT64_C(1) << index); + // todo: is a branch here all that bad? + card += + (1 - 2 * (((UINT64_C(1) << index) & load) >> index)); // +1 or -1 + words[offset] = newload; + list++; + } + return card; +} + +void bitset_flip_list(uint64_t *words, const uint16_t *list, uint64_t length) { + uint64_t offset, load, newload, pos, index; + const uint16_t *end = list + length; + while (list != end) { + pos = *list; + offset = pos >> 6; + index = pos % 64; + load = words[offset]; + newload = load ^ (UINT64_C(1) << index); + words[offset] = newload; + list++; + } +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace api { +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif
\ No newline at end of file diff --git a/contrib/libs/croaring/src/containers/array.c b/contrib/libs/croaring/src/containers/array.c new file mode 100644 index 00000000000..0a24482a32b --- /dev/null +++ b/contrib/libs/croaring/src/containers/array.c @@ -0,0 +1,575 @@ +/* + * array.c + * + */ + +#include <assert.h> +#include <stdio.h> +#include <stdlib.h> + +#include <roaring/containers/array.h> +#include <roaring/memory.h> + +#if CROARING_IS_X64 +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +extern inline uint16_t array_container_minimum(const array_container_t *arr); +extern inline uint16_t array_container_maximum(const array_container_t *arr); +extern inline int array_container_index_equalorlarger( + const array_container_t *arr, uint16_t x); + +extern inline int array_container_rank(const array_container_t *arr, + uint16_t x); +extern inline uint32_t array_container_rank_many(const array_container_t *arr, + uint64_t start_rank, + const uint32_t *begin, + const uint32_t *end, + uint64_t *ans); +extern inline int array_container_get_index(const array_container_t *arr, + uint16_t x); +extern inline bool array_container_contains(const array_container_t *arr, + uint16_t pos); +extern inline int array_container_cardinality(const array_container_t *array); +extern inline bool array_container_nonzero_cardinality( + const array_container_t *array); +extern inline int32_t array_container_serialized_size_in_bytes(int32_t card); +extern inline bool array_container_empty(const array_container_t *array); +extern inline bool array_container_full(const array_container_t *array); + +/* Create a new array with capacity size. Return NULL in case of failure. */ +array_container_t *array_container_create_given_capacity(int32_t size) { + array_container_t *container; + + if ((container = (array_container_t *)roaring_malloc( + sizeof(array_container_t))) == NULL) { + return NULL; + } + + if (size <= 0) { // we don't want to rely on malloc(0) + container->array = NULL; + } else if ((container->array = (uint16_t *)roaring_malloc(sizeof(uint16_t) * + size)) == NULL) { + roaring_free(container); + return NULL; + } + + container->capacity = size; + container->cardinality = 0; + + return container; +} + +/* Create a new array. Return NULL in case of failure. */ +array_container_t *array_container_create(void) { + return array_container_create_given_capacity(ARRAY_DEFAULT_INIT_SIZE); +} + +/* Create a new array containing all values in [min,max). */ +array_container_t *array_container_create_range(uint32_t min, uint32_t max) { + array_container_t *answer = + array_container_create_given_capacity(max - min + 1); + if (answer == NULL) return answer; + answer->cardinality = 0; + for (uint32_t k = min; k < max; k++) { + answer->array[answer->cardinality++] = k; + } + return answer; +} + +/* Duplicate container */ +ALLOW_UNALIGNED +array_container_t *array_container_clone(const array_container_t *src) { + array_container_t *newcontainer = + array_container_create_given_capacity(src->capacity); + if (newcontainer == NULL) return NULL; + + newcontainer->cardinality = src->cardinality; + + memcpy(newcontainer->array, src->array, + src->cardinality * sizeof(uint16_t)); + + return newcontainer; +} + +void array_container_offset(const array_container_t *c, container_t **loc, + container_t **hic, uint16_t offset) { + array_container_t *lo = NULL, *hi = NULL; + int top, lo_cap, hi_cap; + + top = (1 << 16) - offset; + + lo_cap = count_less(c->array, c->cardinality, top); + if (loc && lo_cap) { + lo = array_container_create_given_capacity(lo_cap); + for (int i = 0; i < lo_cap; ++i) { + array_container_add(lo, c->array[i] + offset); + } + *loc = (container_t *)lo; + } + + hi_cap = c->cardinality - lo_cap; + if (hic && hi_cap) { + hi = array_container_create_given_capacity(hi_cap); + for (int i = lo_cap; i < c->cardinality; ++i) { + array_container_add(hi, c->array[i] + offset); + } + *hic = (container_t *)hi; + } +} + +int array_container_shrink_to_fit(array_container_t *src) { + if (src->cardinality == src->capacity) return 0; // nothing to do + int savings = src->capacity - src->cardinality; + src->capacity = src->cardinality; + if (src->capacity == + 0) { // we do not want to rely on realloc for zero allocs + roaring_free(src->array); + src->array = NULL; + } else { + uint16_t *oldarray = src->array; + src->array = (uint16_t *)roaring_realloc( + oldarray, src->capacity * sizeof(uint16_t)); + if (src->array == NULL) roaring_free(oldarray); // should never happen? + } + return savings; +} + +/* Free memory. */ +void array_container_free(array_container_t *arr) { + if (arr->array != + NULL) { // Jon Strabala reports that some tools complain otherwise + roaring_free(arr->array); + arr->array = NULL; // pedantic + } + roaring_free(arr); +} + +static inline int32_t grow_capacity(int32_t capacity) { + return (capacity <= 0) ? ARRAY_DEFAULT_INIT_SIZE + : capacity < 64 ? capacity * 2 + : capacity < 1024 ? capacity * 3 / 2 + : capacity * 5 / 4; +} + +static inline int32_t clamp(int32_t val, int32_t min, int32_t max) { + return ((val < min) ? min : (val > max) ? max : val); +} + +void array_container_grow(array_container_t *container, int32_t min, + bool preserve) { + int32_t max = (min <= DEFAULT_MAX_SIZE ? DEFAULT_MAX_SIZE : 65536); + int32_t new_capacity = clamp(grow_capacity(container->capacity), min, max); + + container->capacity = new_capacity; + uint16_t *array = container->array; + + if (preserve) { + container->array = + (uint16_t *)roaring_realloc(array, new_capacity * sizeof(uint16_t)); + if (container->array == NULL) roaring_free(array); + } else { + // Jon Strabala reports that some tools complain otherwise + if (array != NULL) { + roaring_free(array); + } + container->array = + (uint16_t *)roaring_malloc(new_capacity * sizeof(uint16_t)); + } + + // if realloc fails, we have container->array == NULL. +} + +/* Copy one container into another. We assume that they are distinct. */ +void array_container_copy(const array_container_t *src, + array_container_t *dst) { + const int32_t cardinality = src->cardinality; + if (cardinality > dst->capacity) { + array_container_grow(dst, cardinality, false); + } + + dst->cardinality = cardinality; + memcpy(dst->array, src->array, cardinality * sizeof(uint16_t)); +} + +void array_container_add_from_range(array_container_t *arr, uint32_t min, + uint32_t max, uint16_t step) { + for (uint32_t value = min; value < max; value += step) { + array_container_append(arr, value); + } +} + +/* Computes the union of array1 and array2 and write the result to arrayout. + * It is assumed that arrayout is distinct from both array1 and array2. + */ +void array_container_union(const array_container_t *array_1, + const array_container_t *array_2, + array_container_t *out) { + const int32_t card_1 = array_1->cardinality, card_2 = array_2->cardinality; + const int32_t max_cardinality = card_1 + card_2; + + if (out->capacity < max_cardinality) { + array_container_grow(out, max_cardinality, false); + } + out->cardinality = (int32_t)fast_union_uint16( + array_1->array, card_1, array_2->array, card_2, out->array); +} + +/* Computes the difference of array1 and array2 and write the result + * to array out. + * Array out does not need to be distinct from array_1 + */ +void array_container_andnot(const array_container_t *array_1, + const array_container_t *array_2, + array_container_t *out) { + if (out->capacity < array_1->cardinality) + array_container_grow(out, array_1->cardinality, false); +#if CROARING_IS_X64 + if ((croaring_hardware_support() & ROARING_SUPPORTS_AVX2) && + (out != array_1) && (out != array_2)) { + out->cardinality = difference_vector16( + array_1->array, array_1->cardinality, array_2->array, + array_2->cardinality, out->array); + } else { + out->cardinality = + difference_uint16(array_1->array, array_1->cardinality, + array_2->array, array_2->cardinality, out->array); + } +#else + out->cardinality = + difference_uint16(array_1->array, array_1->cardinality, array_2->array, + array_2->cardinality, out->array); +#endif +} + +/* Computes the symmetric difference of array1 and array2 and write the + * result + * to arrayout. + * It is assumed that arrayout is distinct from both array1 and array2. + */ +void array_container_xor(const array_container_t *array_1, + const array_container_t *array_2, + array_container_t *out) { + const int32_t card_1 = array_1->cardinality, card_2 = array_2->cardinality; + const int32_t max_cardinality = card_1 + card_2; + if (out->capacity < max_cardinality) { + array_container_grow(out, max_cardinality, false); + } + +#if CROARING_IS_X64 + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + out->cardinality = + xor_vector16(array_1->array, array_1->cardinality, array_2->array, + array_2->cardinality, out->array); + } else { + out->cardinality = + xor_uint16(array_1->array, array_1->cardinality, array_2->array, + array_2->cardinality, out->array); + } +#else + out->cardinality = + xor_uint16(array_1->array, array_1->cardinality, array_2->array, + array_2->cardinality, out->array); +#endif +} + +static inline int32_t minimum_int32(int32_t a, int32_t b) { + return (a < b) ? a : b; +} + +/* computes the intersection of array1 and array2 and write the result to + * arrayout. + * It is assumed that arrayout is distinct from both array1 and array2. + * */ +void array_container_intersection(const array_container_t *array1, + const array_container_t *array2, + array_container_t *out) { + int32_t card_1 = array1->cardinality, card_2 = array2->cardinality, + min_card = minimum_int32(card_1, card_2); + const int threshold = 64; // subject to tuning +#if CROARING_IS_X64 + if (out->capacity < min_card) { + array_container_grow(out, min_card + sizeof(__m128i) / sizeof(uint16_t), + false); + } +#else + if (out->capacity < min_card) { + array_container_grow(out, min_card, false); + } +#endif + + if (card_1 * threshold < card_2) { + out->cardinality = intersect_skewed_uint16( + array1->array, card_1, array2->array, card_2, out->array); + } else if (card_2 * threshold < card_1) { + out->cardinality = intersect_skewed_uint16( + array2->array, card_2, array1->array, card_1, out->array); + } else { +#if CROARING_IS_X64 + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + out->cardinality = intersect_vector16( + array1->array, card_1, array2->array, card_2, out->array); + } else { + out->cardinality = intersect_uint16( + array1->array, card_1, array2->array, card_2, out->array); + } +#else + out->cardinality = intersect_uint16(array1->array, card_1, + array2->array, card_2, out->array); +#endif + } +} + +/* computes the size of the intersection of array1 and array2 + * */ +int array_container_intersection_cardinality(const array_container_t *array1, + const array_container_t *array2) { + int32_t card_1 = array1->cardinality, card_2 = array2->cardinality; + const int threshold = 64; // subject to tuning + if (card_1 * threshold < card_2) { + return intersect_skewed_uint16_cardinality(array1->array, card_1, + array2->array, card_2); + } else if (card_2 * threshold < card_1) { + return intersect_skewed_uint16_cardinality(array2->array, card_2, + array1->array, card_1); + } else { +#if CROARING_IS_X64 + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + return intersect_vector16_cardinality(array1->array, card_1, + array2->array, card_2); + } else { + return intersect_uint16_cardinality(array1->array, card_1, + array2->array, card_2); + } +#else + return intersect_uint16_cardinality(array1->array, card_1, + array2->array, card_2); +#endif + } +} + +bool array_container_intersect(const array_container_t *array1, + const array_container_t *array2) { + int32_t card_1 = array1->cardinality, card_2 = array2->cardinality; + const int threshold = 64; // subject to tuning + if (card_1 * threshold < card_2) { + return intersect_skewed_uint16_nonempty(array1->array, card_1, + array2->array, card_2); + } else if (card_2 * threshold < card_1) { + return intersect_skewed_uint16_nonempty(array2->array, card_2, + array1->array, card_1); + } else { + // we do not bother vectorizing + return intersect_uint16_nonempty(array1->array, card_1, array2->array, + card_2); + } +} + +/* computes the intersection of array1 and array2 and write the result to + * array1. + * */ +void array_container_intersection_inplace(array_container_t *src_1, + const array_container_t *src_2) { + int32_t card_1 = src_1->cardinality, card_2 = src_2->cardinality; + const int threshold = 64; // subject to tuning + if (card_1 * threshold < card_2) { + src_1->cardinality = intersect_skewed_uint16( + src_1->array, card_1, src_2->array, card_2, src_1->array); + } else if (card_2 * threshold < card_1) { + src_1->cardinality = intersect_skewed_uint16( + src_2->array, card_2, src_1->array, card_1, src_1->array); + } else { +#if CROARING_IS_X64 + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + src_1->cardinality = intersect_vector16_inplace( + src_1->array, card_1, src_2->array, card_2); + } else { + src_1->cardinality = intersect_uint16( + src_1->array, card_1, src_2->array, card_2, src_1->array); + } +#else + src_1->cardinality = intersect_uint16( + src_1->array, card_1, src_2->array, card_2, src_1->array); +#endif + } +} + +ALLOW_UNALIGNED +int array_container_to_uint32_array(void *vout, const array_container_t *cont, + uint32_t base) { +#if CROARING_IS_X64 + int support = croaring_hardware_support(); +#if CROARING_COMPILER_SUPPORTS_AVX512 + if (support & ROARING_SUPPORTS_AVX512) { + return avx512_array_container_to_uint32_array(vout, cont->array, + cont->cardinality, base); + } +#endif + if (support & ROARING_SUPPORTS_AVX2) { + return array_container_to_uint32_array_vector16( + vout, cont->array, cont->cardinality, base); + } +#endif // CROARING_IS_X64 + int outpos = 0; + uint32_t *out = (uint32_t *)vout; + size_t i = 0; + for (; i < (size_t)cont->cardinality; ++i) { + const uint32_t val = base + cont->array[i]; + memcpy(out + outpos, &val, + sizeof(uint32_t)); // should be compiled as a MOV on x64 + outpos++; + } + return outpos; +} + +void array_container_printf(const array_container_t *v) { + if (v->cardinality == 0) { + printf("{}"); + return; + } + printf("{"); + printf("%d", v->array[0]); + for (int i = 1; i < v->cardinality; ++i) { + printf(",%d", v->array[i]); + } + printf("}"); +} + +void array_container_printf_as_uint32_array(const array_container_t *v, + uint32_t base) { + if (v->cardinality == 0) { + return; + } + printf("%u", v->array[0] + base); + for (int i = 1; i < v->cardinality; ++i) { + printf(",%u", v->array[i] + base); + } +} + +/* + * Validate the container. Returns true if valid. + */ +bool array_container_validate(const array_container_t *v, const char **reason) { + if (v->capacity < 0) { + *reason = "negative capacity"; + return false; + } + if (v->cardinality < 0) { + *reason = "negative cardinality"; + return false; + } + if (v->cardinality > v->capacity) { + *reason = "cardinality exceeds capacity"; + return false; + } + if (v->cardinality > DEFAULT_MAX_SIZE) { + *reason = "cardinality exceeds DEFAULT_MAX_SIZE"; + return false; + } + if (v->cardinality == 0) { + *reason = "zero cardinality"; + return false; + } + + if (v->array == NULL) { + *reason = "NULL array pointer"; + return false; + } + uint16_t prev = v->array[0]; + for (int i = 1; i < v->cardinality; ++i) { + if (v->array[i] <= prev) { + *reason = "array elements not strictly increasing"; + return false; + } + prev = v->array[i]; + } + + return true; +} + +/* Compute the number of runs */ +int32_t array_container_number_of_runs(const array_container_t *ac) { + // Can SIMD work here? + int32_t nr_runs = 0; + int32_t prev = -2; + for (const uint16_t *p = ac->array; p != ac->array + ac->cardinality; ++p) { + if (*p != prev + 1) nr_runs++; + prev = *p; + } + return nr_runs; +} + +/** + * Writes the underlying array to buf, outputs how many bytes were written. + * The number of bytes written should be + * array_container_size_in_bytes(container). + * + */ +int32_t array_container_write(const array_container_t *container, char *buf) { + memcpy(buf, container->array, container->cardinality * sizeof(uint16_t)); + return array_container_size_in_bytes(container); +} + +bool array_container_is_subset(const array_container_t *container1, + const array_container_t *container2) { + if (container1->cardinality > container2->cardinality) { + return false; + } + int i1 = 0, i2 = 0; + while (i1 < container1->cardinality && i2 < container2->cardinality) { + if (container1->array[i1] == container2->array[i2]) { + i1++; + i2++; + } else if (container1->array[i1] > container2->array[i2]) { + i2++; + } else { // container1->array[i1] < container2->array[i2] + return false; + } + } + if (i1 == container1->cardinality) { + return true; + } else { + return false; + } +} + +int32_t array_container_read(int32_t cardinality, array_container_t *container, + const char *buf) { + if (container->capacity < cardinality) { + array_container_grow(container, cardinality, false); + } + container->cardinality = cardinality; + memcpy(container->array, buf, container->cardinality * sizeof(uint16_t)); + + return array_container_size_in_bytes(container); +} + +bool array_container_iterate(const array_container_t *cont, uint32_t base, + roaring_iterator iterator, void *ptr) { + for (int i = 0; i < cont->cardinality; i++) + if (!iterator(cont->array[i] + base, ptr)) return false; + return true; +} + +bool array_container_iterate64(const array_container_t *cont, uint32_t base, + roaring_iterator64 iterator, uint64_t high_bits, + void *ptr) { + for (int i = 0; i < cont->cardinality; i++) + if (!iterator(high_bits | (uint64_t)(cont->array[i] + base), ptr)) + return false; + return true; +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif diff --git a/contrib/libs/croaring/src/containers/bitset.c b/contrib/libs/croaring/src/containers/bitset.c new file mode 100644 index 00000000000..9be73ba21fe --- /dev/null +++ b/contrib/libs/croaring/src/containers/bitset.c @@ -0,0 +1,1309 @@ +/* + * bitset.c + * + */ +#ifndef _POSIX_C_SOURCE +#define _POSIX_C_SOURCE 200809L +#endif +#include <assert.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <roaring/bitset_util.h> +#include <roaring/containers/array.h> +#include <roaring/containers/bitset.h> +#include <roaring/memory.h> +#include <roaring/portability.h> +#include <roaring/utilasm.h> + +#if CROARING_IS_X64 +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif + +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +extern inline int bitset_container_cardinality( + const bitset_container_t *bitset); +extern inline void bitset_container_set(bitset_container_t *bitset, + uint16_t pos); +// unused at this time: +// extern inline void bitset_container_unset(bitset_container_t *bitset, +// uint16_t pos); +extern inline bool bitset_container_get(const bitset_container_t *bitset, + uint16_t pos); +extern inline int32_t bitset_container_serialized_size_in_bytes(void); +extern inline bool bitset_container_add(bitset_container_t *bitset, + uint16_t pos); +extern inline bool bitset_container_remove(bitset_container_t *bitset, + uint16_t pos); +extern inline bool bitset_container_contains(const bitset_container_t *bitset, + uint16_t pos); + +void bitset_container_clear(bitset_container_t *bitset) { + memset(bitset->words, 0, sizeof(uint64_t) * BITSET_CONTAINER_SIZE_IN_WORDS); + bitset->cardinality = 0; +} + +void bitset_container_set_all(bitset_container_t *bitset) { + memset(bitset->words, INT64_C(-1), + sizeof(uint64_t) * BITSET_CONTAINER_SIZE_IN_WORDS); + bitset->cardinality = (1 << 16); +} + +/* Create a new bitset. Return NULL in case of failure. */ +bitset_container_t *bitset_container_create(void) { + bitset_container_t *bitset = + (bitset_container_t *)roaring_malloc(sizeof(bitset_container_t)); + + if (!bitset) { + return NULL; + } + + size_t align_size = 32; +#if CROARING_IS_X64 + int support = croaring_hardware_support(); + if (support & ROARING_SUPPORTS_AVX512) { + // sizeof(__m512i) == 64 + align_size = 64; + } else { + // sizeof(__m256i) == 32 + align_size = 32; + } +#endif + bitset->words = (uint64_t *)roaring_aligned_malloc( + align_size, sizeof(uint64_t) * BITSET_CONTAINER_SIZE_IN_WORDS); + if (!bitset->words) { + roaring_free(bitset); + return NULL; + } + bitset_container_clear(bitset); + return bitset; +} + +/* Copy one container into another. We assume that they are distinct. */ +void bitset_container_copy(const bitset_container_t *source, + bitset_container_t *dest) { + dest->cardinality = source->cardinality; + memcpy(dest->words, source->words, + sizeof(uint64_t) * BITSET_CONTAINER_SIZE_IN_WORDS); +} + +void bitset_container_add_from_range(bitset_container_t *bitset, uint32_t min, + uint32_t max, uint16_t step) { + if (step == 0) return; // refuse to crash + if ((64 % step) == 0) { // step divides 64 + uint64_t mask = 0; // construct the repeated mask + for (uint32_t value = (min % step); value < 64; value += step) { + mask |= ((uint64_t)1 << value); + } + uint32_t firstword = min / 64; + uint32_t endword = (max - 1) / 64; + bitset->cardinality = (max - min + step - 1) / step; + if (firstword == endword) { + bitset->words[firstword] |= + mask & (((~UINT64_C(0)) << (min % 64)) & + ((~UINT64_C(0)) >> ((~max + 1) % 64))); + return; + } + bitset->words[firstword] = mask & ((~UINT64_C(0)) << (min % 64)); + for (uint32_t i = firstword + 1; i < endword; i++) + bitset->words[i] = mask; + bitset->words[endword] = mask & ((~UINT64_C(0)) >> ((~max + 1) % 64)); + } else { + for (uint32_t value = min; value < max; value += step) { + bitset_container_add(bitset, value); + } + } +} + +/* Free memory. */ +void bitset_container_free(bitset_container_t *bitset) { + if (bitset->words != + NULL) { // Jon Strabala reports that some tools complain otherwise + roaring_aligned_free(bitset->words); + bitset->words = NULL; // pedantic + } + roaring_free(bitset); +} + +/* duplicate container. */ +ALLOW_UNALIGNED +bitset_container_t *bitset_container_clone(const bitset_container_t *src) { + bitset_container_t *bitset = + (bitset_container_t *)roaring_malloc(sizeof(bitset_container_t)); + + if (!bitset) { + return NULL; + } + + size_t align_size = 32; +#if CROARING_IS_X64 + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX512) { + // sizeof(__m512i) == 64 + align_size = 64; + } else { + // sizeof(__m256i) == 32 + align_size = 32; + } +#endif + bitset->words = (uint64_t *)roaring_aligned_malloc( + align_size, sizeof(uint64_t) * BITSET_CONTAINER_SIZE_IN_WORDS); + if (!bitset->words) { + roaring_free(bitset); + return NULL; + } + bitset->cardinality = src->cardinality; + memcpy(bitset->words, src->words, + sizeof(uint64_t) * BITSET_CONTAINER_SIZE_IN_WORDS); + return bitset; +} + +void bitset_container_offset(const bitset_container_t *c, container_t **loc, + container_t **hic, uint16_t offset) { + bitset_container_t *bc = NULL; + uint64_t val; + uint16_t b, i, end; + + b = offset >> 6; + i = offset % 64; + end = 1024 - b; + + if (loc != NULL) { + bc = bitset_container_create(); + if (i == 0) { + memcpy(bc->words + b, c->words, 8 * end); + } else { + bc->words[b] = c->words[0] << i; + for (uint32_t k = 1; k < end; ++k) { + val = c->words[k] << i; + val |= c->words[k - 1] >> (64 - i); + bc->words[b + k] = val; + } + } + + bc->cardinality = bitset_container_compute_cardinality(bc); + if (bc->cardinality != 0) { + *loc = bc; + } + if (bc->cardinality == c->cardinality) { + return; + } + } + + if (hic == NULL) { + // Both hic and loc can't be NULL, so bc is never NULL here + if (bc->cardinality == 0) { + bitset_container_free(bc); + } + return; + } + + if (bc == NULL || bc->cardinality != 0) { + bc = bitset_container_create(); + } + + if (i == 0) { + memcpy(bc->words, c->words + end, 8 * b); + } else { + for (uint32_t k = end; k < 1024; ++k) { + val = c->words[k] << i; + val |= c->words[k - 1] >> (64 - i); + bc->words[k - end] = val; + } + bc->words[b] = c->words[1023] >> (64 - i); + } + + bc->cardinality = bitset_container_compute_cardinality(bc); + if (bc->cardinality == 0) { + bitset_container_free(bc); + return; + } + *hic = bc; +} + +void bitset_container_set_range(bitset_container_t *bitset, uint32_t begin, + uint32_t end) { + bitset_set_range(bitset->words, begin, end); + bitset->cardinality = + bitset_container_compute_cardinality(bitset); // could be smarter +} + +bool bitset_container_intersect(const bitset_container_t *src_1, + const bitset_container_t *src_2) { + // could vectorize, but this is probably already quite fast in practice + const uint64_t *__restrict__ words_1 = src_1->words; + const uint64_t *__restrict__ words_2 = src_2->words; + for (int i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i++) { + if ((words_1[i] & words_2[i]) != 0) return true; + } + return false; +} + +#if CROARING_IS_X64 +#ifndef WORDS_IN_AVX2_REG +#define WORDS_IN_AVX2_REG sizeof(__m256i) / sizeof(uint64_t) +#endif +#ifndef WORDS_IN_AVX512_REG +#define WORDS_IN_AVX512_REG sizeof(__m512i) / sizeof(uint64_t) +#endif +/* Get the number of bits set (force computation) */ +static inline int _scalar_bitset_container_compute_cardinality( + const bitset_container_t *bitset) { + const uint64_t *words = bitset->words; + int32_t sum = 0; + for (int i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i += 4) { + sum += roaring_hamming(words[i]); + sum += roaring_hamming(words[i + 1]); + sum += roaring_hamming(words[i + 2]); + sum += roaring_hamming(words[i + 3]); + } + return sum; +} +/* Get the number of bits set (force computation) */ +int bitset_container_compute_cardinality(const bitset_container_t *bitset) { + int support = croaring_hardware_support(); +#if CROARING_COMPILER_SUPPORTS_AVX512 + if (support & ROARING_SUPPORTS_AVX512) { + return (int)avx512_vpopcount( + (const __m512i *)bitset->words, + BITSET_CONTAINER_SIZE_IN_WORDS / (WORDS_IN_AVX512_REG)); + } else +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + if (support & ROARING_SUPPORTS_AVX2) { + return (int)avx2_harley_seal_popcount256( + (const __m256i *)bitset->words, + BITSET_CONTAINER_SIZE_IN_WORDS / (WORDS_IN_AVX2_REG)); + } else { + return _scalar_bitset_container_compute_cardinality(bitset); + } +} + +#elif defined(CROARING_USENEON) +int bitset_container_compute_cardinality(const bitset_container_t *bitset) { + uint16x8_t n0 = vdupq_n_u16(0); + uint16x8_t n1 = vdupq_n_u16(0); + uint16x8_t n2 = vdupq_n_u16(0); + uint16x8_t n3 = vdupq_n_u16(0); + for (size_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i += 8) { + uint64x2_t c0 = vld1q_u64(&bitset->words[i + 0]); + n0 = vaddq_u16(n0, vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u64(c0)))); + uint64x2_t c1 = vld1q_u64(&bitset->words[i + 2]); + n1 = vaddq_u16(n1, vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u64(c1)))); + uint64x2_t c2 = vld1q_u64(&bitset->words[i + 4]); + n2 = vaddq_u16(n2, vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u64(c2)))); + uint64x2_t c3 = vld1q_u64(&bitset->words[i + 6]); + n3 = vaddq_u16(n3, vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u64(c3)))); + } + uint64x2_t n = vdupq_n_u64(0); + n = vaddq_u64(n, vpaddlq_u32(vpaddlq_u16(n0))); + n = vaddq_u64(n, vpaddlq_u32(vpaddlq_u16(n1))); + n = vaddq_u64(n, vpaddlq_u32(vpaddlq_u16(n2))); + n = vaddq_u64(n, vpaddlq_u32(vpaddlq_u16(n3))); + return vgetq_lane_u64(n, 0) + vgetq_lane_u64(n, 1); +} + +#else // CROARING_IS_X64 + +/* Get the number of bits set (force computation) */ +int bitset_container_compute_cardinality(const bitset_container_t *bitset) { + const uint64_t *words = bitset->words; + int32_t sum = 0; + for (int i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i += 4) { + sum += roaring_hamming(words[i]); + sum += roaring_hamming(words[i + 1]); + sum += roaring_hamming(words[i + 2]); + sum += roaring_hamming(words[i + 3]); + } + return sum; +} + +#endif // CROARING_IS_X64 + +#if CROARING_IS_X64 + +#define BITSET_CONTAINER_FN_REPEAT 8 +#ifndef WORDS_IN_AVX512_REG +#define WORDS_IN_AVX512_REG sizeof(__m512i) / sizeof(uint64_t) +#endif // WORDS_IN_AVX512_REG + +/* Computes a binary operation (eg union) on bitset1 and bitset2 and write the + result to bitsetout */ +// clang-format off +#define AVX512_BITSET_CONTAINER_FN1(before, opname, opsymbol, avx_intrinsic, \ + neon_intrinsic, after) \ + static inline int _avx512_bitset_container_##opname##_nocard( \ + const bitset_container_t *src_1, const bitset_container_t *src_2, \ + bitset_container_t *dst) { \ + const uint8_t * __restrict__ words_1 = (const uint8_t *)src_1->words; \ + const uint8_t * __restrict__ words_2 = (const uint8_t *)src_2->words; \ + /* not using the blocking optimization for some reason*/ \ + uint8_t *out = (uint8_t*)dst->words; \ + const int innerloop = 8; \ + for (size_t i = 0; \ + i < BITSET_CONTAINER_SIZE_IN_WORDS / (WORDS_IN_AVX512_REG); \ + i+=innerloop) { \ + __m512i A1, A2, AO; \ + A1 = _mm512_loadu_si512((const __m512i *)(words_1)); \ + A2 = _mm512_loadu_si512((const __m512i *)(words_2)); \ + AO = avx_intrinsic(A2, A1); \ + _mm512_storeu_si512((__m512i *)out, AO); \ + A1 = _mm512_loadu_si512((const __m512i *)(words_1 + 64)); \ + A2 = _mm512_loadu_si512((const __m512i *)(words_2 + 64)); \ + AO = avx_intrinsic(A2, A1); \ + _mm512_storeu_si512((__m512i *)(out+64), AO); \ + A1 = _mm512_loadu_si512((const __m512i *)(words_1 + 128)); \ + A2 = _mm512_loadu_si512((const __m512i *)(words_2 + 128)); \ + AO = avx_intrinsic(A2, A1); \ + _mm512_storeu_si512((__m512i *)(out+128), AO); \ + A1 = _mm512_loadu_si512((const __m512i *)(words_1 + 192)); \ + A2 = _mm512_loadu_si512((const __m512i *)(words_2 + 192)); \ + AO = avx_intrinsic(A2, A1); \ + _mm512_storeu_si512((__m512i *)(out+192), AO); \ + A1 = _mm512_loadu_si512((const __m512i *)(words_1 + 256)); \ + A2 = _mm512_loadu_si512((const __m512i *)(words_2 + 256)); \ + AO = avx_intrinsic(A2, A1); \ + _mm512_storeu_si512((__m512i *)(out+256), AO); \ + A1 = _mm512_loadu_si512((const __m512i *)(words_1 + 320)); \ + A2 = _mm512_loadu_si512((const __m512i *)(words_2 + 320)); \ + AO = avx_intrinsic(A2, A1); \ + _mm512_storeu_si512((__m512i *)(out+320), AO); \ + A1 = _mm512_loadu_si512((const __m512i *)(words_1 + 384)); \ + A2 = _mm512_loadu_si512((const __m512i *)(words_2 + 384)); \ + AO = avx_intrinsic(A2, A1); \ + _mm512_storeu_si512((__m512i *)(out+384), AO); \ + A1 = _mm512_loadu_si512((const __m512i *)(words_1 + 448)); \ + A2 = _mm512_loadu_si512((const __m512i *)(words_2 + 448)); \ + AO = avx_intrinsic(A2, A1); \ + _mm512_storeu_si512((__m512i *)(out+448), AO); \ + out+=512; \ + words_1 += 512; \ + words_2 += 512; \ + } \ + dst->cardinality = BITSET_UNKNOWN_CARDINALITY; \ + return dst->cardinality; \ + } + +#define AVX512_BITSET_CONTAINER_FN2(before, opname, opsymbol, avx_intrinsic, \ + neon_intrinsic, after) \ + /* next, a version that updates cardinality*/ \ + static inline int _avx512_bitset_container_##opname(const bitset_container_t *src_1, \ + const bitset_container_t *src_2, \ + bitset_container_t *dst) { \ + const __m512i * __restrict__ words_1 = (const __m512i *) src_1->words; \ + const __m512i * __restrict__ words_2 = (const __m512i *) src_2->words; \ + __m512i *out = (__m512i *) dst->words; \ + dst->cardinality = (int32_t)avx512_harley_seal_popcount512andstore_##opname(words_2,\ + words_1, out,BITSET_CONTAINER_SIZE_IN_WORDS / (WORDS_IN_AVX512_REG)); \ + return dst->cardinality; \ + } + +#define AVX512_BITSET_CONTAINER_FN3(before, opname, opsymbol, avx_intrinsic, \ + neon_intrinsic, after) \ + /* next, a version that just computes the cardinality*/ \ + static inline int _avx512_bitset_container_##opname##_justcard( \ + const bitset_container_t *src_1, const bitset_container_t *src_2) { \ + const __m512i * __restrict__ data1 = (const __m512i *) src_1->words; \ + const __m512i * __restrict__ data2 = (const __m512i *) src_2->words; \ + return (int)avx512_harley_seal_popcount512_##opname(data2, \ + data1, BITSET_CONTAINER_SIZE_IN_WORDS / (WORDS_IN_AVX512_REG)); \ + } + + +// we duplicate the function because other containers use the "or" term, makes API more consistent +#if CROARING_COMPILER_SUPPORTS_AVX512 +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, or, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, union, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 + +// we duplicate the function because other containers use the "intersection" term, makes API more consistent +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, and, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, intersection, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 + +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, xor, ^, _mm512_xor_si512, veorq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX512, andnot, &~, _mm512_andnot_si512, vbicq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 + +// we duplicate the function because other containers use the "or" term, makes API more consistent +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, or, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, union, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 + +// we duplicate the function because other containers use the "intersection" term, makes API more consistent +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, and, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, intersection, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 + +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, xor, ^, _mm512_xor_si512, veorq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX512, andnot, &~, _mm512_andnot_si512, vbicq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 + +// we duplicate the function because other containers use the "or" term, makes API more consistent +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, or, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, union, |, _mm512_or_si512, vorrq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 + +// we duplicate the function because other containers use the "intersection" term, makes API more consistent +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, and, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, intersection, &, _mm512_and_si512, vandq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 + +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, xor, ^, _mm512_xor_si512, veorq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 +CROARING_TARGET_AVX512 +AVX512_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX512, andnot, &~, _mm512_andnot_si512, vbicq_u64, CROARING_UNTARGET_AVX512) +CROARING_UNTARGET_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + +#ifndef WORDS_IN_AVX2_REG +#define WORDS_IN_AVX2_REG sizeof(__m256i) / sizeof(uint64_t) +#endif // WORDS_IN_AVX2_REG +#define LOOP_SIZE \ + BITSET_CONTAINER_SIZE_IN_WORDS / \ + ((WORDS_IN_AVX2_REG)*BITSET_CONTAINER_FN_REPEAT) + +/* Computes a binary operation (eg union) on bitset1 and bitset2 and write the + result to bitsetout */ +// clang-format off +#define AVX_BITSET_CONTAINER_FN1(before, opname, opsymbol, avx_intrinsic, \ + neon_intrinsic, after) \ + static inline int _avx2_bitset_container_##opname##_nocard( \ + const bitset_container_t *src_1, const bitset_container_t *src_2, \ + bitset_container_t *dst) { \ + const uint8_t *__restrict__ words_1 = (const uint8_t *)src_1->words; \ + const uint8_t *__restrict__ words_2 = (const uint8_t *)src_2->words; \ + /* not using the blocking optimization for some reason*/ \ + uint8_t *out = (uint8_t *)dst->words; \ + const int innerloop = 8; \ + for (size_t i = 0; \ + i < BITSET_CONTAINER_SIZE_IN_WORDS / (WORDS_IN_AVX2_REG); \ + i += innerloop) { \ + __m256i A1, A2, AO; \ + A1 = _mm256_lddqu_si256((const __m256i *)(words_1)); \ + A2 = _mm256_lddqu_si256((const __m256i *)(words_2)); \ + AO = avx_intrinsic(A2, A1); \ + _mm256_storeu_si256((__m256i *)out, AO); \ + A1 = _mm256_lddqu_si256((const __m256i *)(words_1 + 32)); \ + A2 = _mm256_lddqu_si256((const __m256i *)(words_2 + 32)); \ + AO = avx_intrinsic(A2, A1); \ + _mm256_storeu_si256((__m256i *)(out + 32), AO); \ + A1 = _mm256_lddqu_si256((const __m256i *)(words_1 + 64)); \ + A2 = _mm256_lddqu_si256((const __m256i *)(words_2 + 64)); \ + AO = avx_intrinsic(A2, A1); \ + _mm256_storeu_si256((__m256i *)(out + 64), AO); \ + A1 = _mm256_lddqu_si256((const __m256i *)(words_1 + 96)); \ + A2 = _mm256_lddqu_si256((const __m256i *)(words_2 + 96)); \ + AO = avx_intrinsic(A2, A1); \ + _mm256_storeu_si256((__m256i *)(out + 96), AO); \ + A1 = _mm256_lddqu_si256((const __m256i *)(words_1 + 128)); \ + A2 = _mm256_lddqu_si256((const __m256i *)(words_2 + 128)); \ + AO = avx_intrinsic(A2, A1); \ + _mm256_storeu_si256((__m256i *)(out + 128), AO); \ + A1 = _mm256_lddqu_si256((const __m256i *)(words_1 + 160)); \ + A2 = _mm256_lddqu_si256((const __m256i *)(words_2 + 160)); \ + AO = avx_intrinsic(A2, A1); \ + _mm256_storeu_si256((__m256i *)(out + 160), AO); \ + A1 = _mm256_lddqu_si256((const __m256i *)(words_1 + 192)); \ + A2 = _mm256_lddqu_si256((const __m256i *)(words_2 + 192)); \ + AO = avx_intrinsic(A2, A1); \ + _mm256_storeu_si256((__m256i *)(out + 192), AO); \ + A1 = _mm256_lddqu_si256((const __m256i *)(words_1 + 224)); \ + A2 = _mm256_lddqu_si256((const __m256i *)(words_2 + 224)); \ + AO = avx_intrinsic(A2, A1); \ + _mm256_storeu_si256((__m256i *)(out + 224), AO); \ + out += 256; \ + words_1 += 256; \ + words_2 += 256; \ + } \ + dst->cardinality = BITSET_UNKNOWN_CARDINALITY; \ + return dst->cardinality; \ + } + +#define AVX_BITSET_CONTAINER_FN2(before, opname, opsymbol, avx_intrinsic, \ + neon_intrinsic, after) \ + /* next, a version that updates cardinality*/ \ + static inline int _avx2_bitset_container_##opname(const bitset_container_t *src_1, \ + const bitset_container_t *src_2, \ + bitset_container_t *dst) { \ + const __m256i *__restrict__ words_1 = (const __m256i *)src_1->words; \ + const __m256i *__restrict__ words_2 = (const __m256i *)src_2->words; \ + __m256i *out = (__m256i *)dst->words; \ + dst->cardinality = (int32_t)avx2_harley_seal_popcount256andstore_##opname( \ + words_2, words_1, out, \ + BITSET_CONTAINER_SIZE_IN_WORDS / (WORDS_IN_AVX2_REG)); \ + return dst->cardinality; \ + } \ + +#define AVX_BITSET_CONTAINER_FN3(before, opname, opsymbol, avx_intrinsic, \ + neon_intrinsic, after) \ + /* next, a version that just computes the cardinality*/ \ + static inline int _avx2_bitset_container_##opname##_justcard( \ + const bitset_container_t *src_1, const bitset_container_t *src_2) { \ + const __m256i *__restrict__ data1 = (const __m256i *)src_1->words; \ + const __m256i *__restrict__ data2 = (const __m256i *)src_2->words; \ + return (int)avx2_harley_seal_popcount256_##opname( \ + data2, data1, BITSET_CONTAINER_SIZE_IN_WORDS / (WORDS_IN_AVX2_REG)); \ + } + + +// we duplicate the function because other containers use the "or" term, makes API more consistent +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, or, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, union, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 + +// we duplicate the function because other containers use the "intersection" term, makes API more consistent +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, and, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, intersection, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 + +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, xor, ^, _mm256_xor_si256, veorq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN1(CROARING_TARGET_AVX2, andnot, &~, _mm256_andnot_si256, vbicq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 + +// we duplicate the function because other containers use the "or" term, makes API more consistent +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, or, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, union, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 + +// we duplicate the function because other containers use the "intersection" term, makes API more consistent +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, and, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, intersection, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 + +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, xor, ^, _mm256_xor_si256, veorq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN2(CROARING_TARGET_AVX2, andnot, &~, _mm256_andnot_si256, vbicq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 + +// we duplicate the function because other containers use the "or" term, makes API more consistent +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, or, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, union, |, _mm256_or_si256, vorrq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 + +// we duplicate the function because other containers use the "intersection" term, makes API more consistent +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, and, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, intersection, &, _mm256_and_si256, vandq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 + +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, xor, ^, _mm256_xor_si256, veorq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 +CROARING_TARGET_AVX2 +AVX_BITSET_CONTAINER_FN3(CROARING_TARGET_AVX2, andnot, &~, _mm256_andnot_si256, vbicq_u64, CROARING_UNTARGET_AVX2) +CROARING_UNTARGET_AVX2 + + +#define SCALAR_BITSET_CONTAINER_FN(opname, opsymbol, avx_intrinsic, \ + neon_intrinsic) \ + static inline int _scalar_bitset_container_##opname(const bitset_container_t *src_1, \ + const bitset_container_t *src_2, \ + bitset_container_t *dst) { \ + const uint64_t *__restrict__ words_1 = src_1->words; \ + const uint64_t *__restrict__ words_2 = src_2->words; \ + uint64_t *out = dst->words; \ + int32_t sum = 0; \ + for (size_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i += 2) { \ + const uint64_t word_1 = (words_1[i])opsymbol(words_2[i]), \ + word_2 = (words_1[i + 1]) opsymbol(words_2[i + 1]); \ + out[i] = word_1; \ + out[i + 1] = word_2; \ + sum += roaring_hamming(word_1); \ + sum += roaring_hamming(word_2); \ + } \ + dst->cardinality = sum; \ + return dst->cardinality; \ + } \ + static inline int _scalar_bitset_container_##opname##_nocard( \ + const bitset_container_t *src_1, const bitset_container_t *src_2, \ + bitset_container_t *dst) { \ + const uint64_t *__restrict__ words_1 = src_1->words; \ + const uint64_t *__restrict__ words_2 = src_2->words; \ + uint64_t *out = dst->words; \ + for (size_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i++) { \ + out[i] = (words_1[i])opsymbol(words_2[i]); \ + } \ + dst->cardinality = BITSET_UNKNOWN_CARDINALITY; \ + return dst->cardinality; \ + } \ + static inline int _scalar_bitset_container_##opname##_justcard( \ + const bitset_container_t *src_1, const bitset_container_t *src_2) { \ + const uint64_t *__restrict__ words_1 = src_1->words; \ + const uint64_t *__restrict__ words_2 = src_2->words; \ + int32_t sum = 0; \ + for (size_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i += 2) { \ + const uint64_t word_1 = (words_1[i])opsymbol(words_2[i]), \ + word_2 = (words_1[i + 1]) opsymbol(words_2[i + 1]); \ + sum += roaring_hamming(word_1); \ + sum += roaring_hamming(word_2); \ + } \ + return sum; \ + } + +// we duplicate the function because other containers use the "or" term, makes API more consistent +SCALAR_BITSET_CONTAINER_FN(or, |, _mm256_or_si256, vorrq_u64) +SCALAR_BITSET_CONTAINER_FN(union, |, _mm256_or_si256, vorrq_u64) + +// we duplicate the function because other containers use the "intersection" term, makes API more consistent +SCALAR_BITSET_CONTAINER_FN(and, &, _mm256_and_si256, vandq_u64) +SCALAR_BITSET_CONTAINER_FN(intersection, &, _mm256_and_si256, vandq_u64) + +SCALAR_BITSET_CONTAINER_FN(xor, ^, _mm256_xor_si256, veorq_u64) +SCALAR_BITSET_CONTAINER_FN(andnot, &~, _mm256_andnot_si256, vbicq_u64) + +#if CROARING_COMPILER_SUPPORTS_AVX512 +#define BITSET_CONTAINER_FN(opname, opsymbol, avx_intrinsic, neon_intrinsic) \ + int bitset_container_##opname(const bitset_container_t *src_1, \ + const bitset_container_t *src_2, \ + bitset_container_t *dst) { \ + int support = croaring_hardware_support(); \ + if ( support & ROARING_SUPPORTS_AVX512 ) { \ + return _avx512_bitset_container_##opname(src_1, src_2, dst); \ + } \ + else if ( support & ROARING_SUPPORTS_AVX2 ) { \ + return _avx2_bitset_container_##opname(src_1, src_2, dst); \ + } else { \ + return _scalar_bitset_container_##opname(src_1, src_2, dst); \ + } \ + } \ + int bitset_container_##opname##_nocard(const bitset_container_t *src_1, \ + const bitset_container_t *src_2, \ + bitset_container_t *dst) { \ + int support = croaring_hardware_support(); \ + if ( support & ROARING_SUPPORTS_AVX512 ) { \ + return _avx512_bitset_container_##opname##_nocard(src_1, src_2, dst); \ + } \ + else if ( support & ROARING_SUPPORTS_AVX2 ) { \ + return _avx2_bitset_container_##opname##_nocard(src_1, src_2, dst); \ + } else { \ + return _scalar_bitset_container_##opname##_nocard(src_1, src_2, dst); \ + } \ + } \ + int bitset_container_##opname##_justcard(const bitset_container_t *src_1, \ + const bitset_container_t *src_2) { \ + int support = croaring_hardware_support(); \ + if ( support & ROARING_SUPPORTS_AVX512 ) { \ + return _avx512_bitset_container_##opname##_justcard(src_1, src_2); \ + } \ + else if ( support & ROARING_SUPPORTS_AVX2 ) { \ + return _avx2_bitset_container_##opname##_justcard(src_1, src_2); \ + } else { \ + return _scalar_bitset_container_##opname##_justcard(src_1, src_2); \ + } \ + } + +#else // CROARING_COMPILER_SUPPORTS_AVX512 + + +#define BITSET_CONTAINER_FN(opname, opsymbol, avx_intrinsic, neon_intrinsic) \ + int bitset_container_##opname(const bitset_container_t *src_1, \ + const bitset_container_t *src_2, \ + bitset_container_t *dst) { \ + if ( croaring_hardware_support() & ROARING_SUPPORTS_AVX2 ) { \ + return _avx2_bitset_container_##opname(src_1, src_2, dst); \ + } else { \ + return _scalar_bitset_container_##opname(src_1, src_2, dst); \ + } \ + } \ + int bitset_container_##opname##_nocard(const bitset_container_t *src_1, \ + const bitset_container_t *src_2, \ + bitset_container_t *dst) { \ + if ( croaring_hardware_support() & ROARING_SUPPORTS_AVX2 ) { \ + return _avx2_bitset_container_##opname##_nocard(src_1, src_2, dst); \ + } else { \ + return _scalar_bitset_container_##opname##_nocard(src_1, src_2, dst); \ + } \ + } \ + int bitset_container_##opname##_justcard(const bitset_container_t *src_1, \ + const bitset_container_t *src_2) { \ + if ( croaring_hardware_support() & ROARING_SUPPORTS_AVX2 ) { \ + return _avx2_bitset_container_##opname##_justcard(src_1, src_2); \ + } else { \ + return _scalar_bitset_container_##opname##_justcard(src_1, src_2); \ + } \ + } + +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + +#elif defined(CROARING_USENEON) + +#define BITSET_CONTAINER_FN(opname, opsymbol, avx_intrinsic, neon_intrinsic) \ +int bitset_container_##opname(const bitset_container_t *src_1, \ + const bitset_container_t *src_2, \ + bitset_container_t *dst) { \ + const uint64_t * __restrict__ words_1 = src_1->words; \ + const uint64_t * __restrict__ words_2 = src_2->words; \ + uint64_t *out = dst->words; \ + uint16x8_t n0 = vdupq_n_u16(0); \ + uint16x8_t n1 = vdupq_n_u16(0); \ + uint16x8_t n2 = vdupq_n_u16(0); \ + uint16x8_t n3 = vdupq_n_u16(0); \ + for (size_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i += 8) { \ + uint64x2_t c0 = neon_intrinsic(vld1q_u64(&words_1[i + 0]), \ + vld1q_u64(&words_2[i + 0])); \ + n0 = vaddq_u16(n0, vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u64(c0)))); \ + vst1q_u64(&out[i + 0], c0); \ + uint64x2_t c1 = neon_intrinsic(vld1q_u64(&words_1[i + 2]), \ + vld1q_u64(&words_2[i + 2])); \ + n1 = vaddq_u16(n1, vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u64(c1)))); \ + vst1q_u64(&out[i + 2], c1); \ + uint64x2_t c2 = neon_intrinsic(vld1q_u64(&words_1[i + 4]), \ + vld1q_u64(&words_2[i + 4])); \ + n2 = vaddq_u16(n2, vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u64(c2)))); \ + vst1q_u64(&out[i + 4], c2); \ + uint64x2_t c3 = neon_intrinsic(vld1q_u64(&words_1[i + 6]), \ + vld1q_u64(&words_2[i + 6])); \ + n3 = vaddq_u16(n3, vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u64(c3)))); \ + vst1q_u64(&out[i + 6], c3); \ + } \ + uint64x2_t n = vdupq_n_u64(0); \ + n = vaddq_u64(n, vpaddlq_u32(vpaddlq_u16(n0))); \ + n = vaddq_u64(n, vpaddlq_u32(vpaddlq_u16(n1))); \ + n = vaddq_u64(n, vpaddlq_u32(vpaddlq_u16(n2))); \ + n = vaddq_u64(n, vpaddlq_u32(vpaddlq_u16(n3))); \ + dst->cardinality = vgetq_lane_u64(n, 0) + vgetq_lane_u64(n, 1); \ + return dst->cardinality; \ +} \ +int bitset_container_##opname##_nocard(const bitset_container_t *src_1, \ + const bitset_container_t *src_2, \ + bitset_container_t *dst) { \ + const uint64_t * __restrict__ words_1 = src_1->words; \ + const uint64_t * __restrict__ words_2 = src_2->words; \ + uint64_t *out = dst->words; \ + for (size_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i += 8) { \ + vst1q_u64(&out[i + 0], neon_intrinsic(vld1q_u64(&words_1[i + 0]), \ + vld1q_u64(&words_2[i + 0]))); \ + vst1q_u64(&out[i + 2], neon_intrinsic(vld1q_u64(&words_1[i + 2]), \ + vld1q_u64(&words_2[i + 2]))); \ + vst1q_u64(&out[i + 4], neon_intrinsic(vld1q_u64(&words_1[i + 4]), \ + vld1q_u64(&words_2[i + 4]))); \ + vst1q_u64(&out[i + 6], neon_intrinsic(vld1q_u64(&words_1[i + 6]), \ + vld1q_u64(&words_2[i + 6]))); \ + } \ + dst->cardinality = BITSET_UNKNOWN_CARDINALITY; \ + return dst->cardinality; \ +} \ +int bitset_container_##opname##_justcard(const bitset_container_t *src_1, \ + const bitset_container_t *src_2) { \ + const uint64_t * __restrict__ words_1 = src_1->words; \ + const uint64_t * __restrict__ words_2 = src_2->words; \ + uint16x8_t n0 = vdupq_n_u16(0); \ + uint16x8_t n1 = vdupq_n_u16(0); \ + uint16x8_t n2 = vdupq_n_u16(0); \ + uint16x8_t n3 = vdupq_n_u16(0); \ + for (size_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i += 8) { \ + uint64x2_t c0 = neon_intrinsic(vld1q_u64(&words_1[i + 0]), \ + vld1q_u64(&words_2[i + 0])); \ + n0 = vaddq_u16(n0, vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u64(c0)))); \ + uint64x2_t c1 = neon_intrinsic(vld1q_u64(&words_1[i + 2]), \ + vld1q_u64(&words_2[i + 2])); \ + n1 = vaddq_u16(n1, vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u64(c1)))); \ + uint64x2_t c2 = neon_intrinsic(vld1q_u64(&words_1[i + 4]), \ + vld1q_u64(&words_2[i + 4])); \ + n2 = vaddq_u16(n2, vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u64(c2)))); \ + uint64x2_t c3 = neon_intrinsic(vld1q_u64(&words_1[i + 6]), \ + vld1q_u64(&words_2[i + 6])); \ + n3 = vaddq_u16(n3, vpaddlq_u8(vcntq_u8(vreinterpretq_u8_u64(c3)))); \ + } \ + uint64x2_t n = vdupq_n_u64(0); \ + n = vaddq_u64(n, vpaddlq_u32(vpaddlq_u16(n0))); \ + n = vaddq_u64(n, vpaddlq_u32(vpaddlq_u16(n1))); \ + n = vaddq_u64(n, vpaddlq_u32(vpaddlq_u16(n2))); \ + n = vaddq_u64(n, vpaddlq_u32(vpaddlq_u16(n3))); \ + return vgetq_lane_u64(n, 0) + vgetq_lane_u64(n, 1); \ +} + +#else + +#define BITSET_CONTAINER_FN(opname, opsymbol, avx_intrinsic, neon_intrinsic) \ +int bitset_container_##opname(const bitset_container_t *src_1, \ + const bitset_container_t *src_2, \ + bitset_container_t *dst) { \ + const uint64_t * __restrict__ words_1 = src_1->words; \ + const uint64_t * __restrict__ words_2 = src_2->words; \ + uint64_t *out = dst->words; \ + int32_t sum = 0; \ + for (size_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i += 2) { \ + const uint64_t word_1 = (words_1[i])opsymbol(words_2[i]), \ + word_2 = (words_1[i + 1])opsymbol(words_2[i + 1]); \ + out[i] = word_1; \ + out[i + 1] = word_2; \ + sum += roaring_hamming(word_1); \ + sum += roaring_hamming(word_2); \ + } \ + dst->cardinality = sum; \ + return dst->cardinality; \ +} \ +int bitset_container_##opname##_nocard(const bitset_container_t *src_1, \ + const bitset_container_t *src_2, \ + bitset_container_t *dst) { \ + const uint64_t * __restrict__ words_1 = src_1->words; \ + const uint64_t * __restrict__ words_2 = src_2->words; \ + uint64_t *out = dst->words; \ + for (size_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i++) { \ + out[i] = (words_1[i])opsymbol(words_2[i]); \ + } \ + dst->cardinality = BITSET_UNKNOWN_CARDINALITY; \ + return dst->cardinality; \ +} \ +int bitset_container_##opname##_justcard(const bitset_container_t *src_1, \ + const bitset_container_t *src_2) { \ + printf("A1\n"); const uint64_t * __restrict__ words_1 = src_1->words; \ + const uint64_t * __restrict__ words_2 = src_2->words; \ + int32_t sum = 0; \ + for (size_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i += 2) { \ + const uint64_t word_1 = (words_1[i])opsymbol(words_2[i]), \ + word_2 = (words_1[i + 1])opsymbol(words_2[i + 1]); \ + sum += roaring_hamming(word_1); \ + sum += roaring_hamming(word_2); \ + } \ + return sum; \ +} + +#endif // CROARING_IS_X64 + +// we duplicate the function because other containers use the "or" term, makes API more consistent +BITSET_CONTAINER_FN(or, |, _mm256_or_si256, vorrq_u64) +BITSET_CONTAINER_FN(union, |, _mm256_or_si256, vorrq_u64) + +// we duplicate the function because other containers use the "intersection" term, makes API more consistent +BITSET_CONTAINER_FN(and, &, _mm256_and_si256, vandq_u64) +BITSET_CONTAINER_FN(intersection, &, _mm256_and_si256, vandq_u64) + +BITSET_CONTAINER_FN(xor, ^, _mm256_xor_si256, veorq_u64) +BITSET_CONTAINER_FN(andnot, &~, _mm256_andnot_si256, vbicq_u64) +// clang-format On + + +ALLOW_UNALIGNED +int bitset_container_to_uint32_array( + uint32_t *out, + const bitset_container_t *bc, + uint32_t base +){ +#if CROARING_IS_X64 + int support = croaring_hardware_support(); +#if CROARING_COMPILER_SUPPORTS_AVX512 + if(( support & ROARING_SUPPORTS_AVX512 ) && (bc->cardinality >= 8192)) // heuristic + return (int) bitset_extract_setbits_avx512(bc->words, + BITSET_CONTAINER_SIZE_IN_WORDS, out, bc->cardinality, base); + else +#endif + if(( support & ROARING_SUPPORTS_AVX2 ) && (bc->cardinality >= 8192)) // heuristic + return (int) bitset_extract_setbits_avx2(bc->words, + BITSET_CONTAINER_SIZE_IN_WORDS, out, bc->cardinality, base); + else + return (int) bitset_extract_setbits(bc->words, + BITSET_CONTAINER_SIZE_IN_WORDS, out, base); +#else + return (int) bitset_extract_setbits(bc->words, + BITSET_CONTAINER_SIZE_IN_WORDS, out, base); +#endif +} + +/* + * Print this container using printf (useful for debugging). + */ +void bitset_container_printf(const bitset_container_t * v) { + printf("{"); + uint32_t base = 0; + bool iamfirst = true;// TODO: rework so that this is not necessary yet still readable + for (int i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; ++i) { + uint64_t w = v->words[i]; + while (w != 0) { + uint64_t t = w & (~w + 1); + int r = roaring_trailing_zeroes(w); + if(iamfirst) {// predicted to be false + printf("%u",base + r); + iamfirst = false; + } else { + printf(",%u",base + r); + } + w ^= t; + } + base += 64; + } + printf("}"); +} + + +/* + * Print this container using printf as a comma-separated list of 32-bit integers starting at base. + */ +void bitset_container_printf_as_uint32_array(const bitset_container_t * v, uint32_t base) { + bool iamfirst = true;// TODO: rework so that this is not necessary yet still readable + for (int i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; ++i) { + uint64_t w = v->words[i]; + while (w != 0) { + uint64_t t = w & (~w + 1); + int r = roaring_trailing_zeroes(w); + if(iamfirst) {// predicted to be false + printf("%u", r + base); + iamfirst = false; + } else { + printf(",%u",r + base); + } + w ^= t; + } + base += 64; + } +} + +/* + * Validate the container. Returns true if valid. + */ +bool bitset_container_validate(const bitset_container_t *v, const char **reason) { + if (v->words == NULL) { + *reason = "words is NULL"; + return false; + } + if (v->cardinality != bitset_container_compute_cardinality(v)) { + *reason = "cardinality is incorrect"; + return false; + } + if (v->cardinality <= DEFAULT_MAX_SIZE) { + *reason = "cardinality is too small for a bitmap container"; + return false; + } + // Attempt to forcibly load the first and last words, hopefully causing + // a segfault or an address sanitizer error if words is not allocated. + volatile uint64_t *words = v->words; + (void) words[0]; + (void) words[BITSET_CONTAINER_SIZE_IN_WORDS - 1]; + return true; +} + + +// TODO: use the fast lower bound, also +int bitset_container_number_of_runs(bitset_container_t *bc) { + int num_runs = 0; + uint64_t next_word = bc->words[0]; + + for (int i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS-1; ++i) { + uint64_t word = next_word; + next_word = bc->words[i+1]; + num_runs += roaring_hamming((~word) & (word << 1)) + ( (word >> 63) & ~next_word); + } + + uint64_t word = next_word; + num_runs += roaring_hamming((~word) & (word << 1)); + if((word & 0x8000000000000000ULL) != 0) + num_runs++; + return num_runs; +} + + +int32_t bitset_container_write(const bitset_container_t *container, + char *buf) { + memcpy(buf, container->words, BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t)); + return bitset_container_size_in_bytes(container); +} + + +int32_t bitset_container_read(int32_t cardinality, bitset_container_t *container, + const char *buf) { + container->cardinality = cardinality; + memcpy(container->words, buf, BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t)); + return bitset_container_size_in_bytes(container); +} + +bool bitset_container_iterate(const bitset_container_t *cont, uint32_t base, roaring_iterator iterator, void *ptr) { + for (int32_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; ++i ) { + uint64_t w = cont->words[i]; + while (w != 0) { + uint64_t t = w & (~w + 1); + int r = roaring_trailing_zeroes(w); + if(!iterator(r + base, ptr)) return false; + w ^= t; + } + base += 64; + } + return true; +} + +bool bitset_container_iterate64(const bitset_container_t *cont, uint32_t base, roaring_iterator64 iterator, uint64_t high_bits, void *ptr) { + for (int32_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; ++i ) { + uint64_t w = cont->words[i]; + while (w != 0) { + uint64_t t = w & (~w + 1); + int r = roaring_trailing_zeroes(w); + if(!iterator(high_bits | (uint64_t)(r + base), ptr)) return false; + w ^= t; + } + base += 64; + } + return true; +} + +#if CROARING_IS_X64 +#if CROARING_COMPILER_SUPPORTS_AVX512 +CROARING_TARGET_AVX512 +ALLOW_UNALIGNED +static inline bool _avx512_bitset_container_equals(const bitset_container_t *container1, const bitset_container_t *container2) { + const __m512i *ptr1 = (const __m512i*)container1->words; + const __m512i *ptr2 = (const __m512i*)container2->words; + for (size_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS*sizeof(uint64_t)/64; i++) { + __m512i r1 = _mm512_loadu_si512(ptr1+i); + __m512i r2 = _mm512_loadu_si512(ptr2+i); + __mmask64 mask = _mm512_cmpeq_epi8_mask(r1, r2); + if ((uint64_t)mask != UINT64_MAX) { + return false; + } + } + return true; +} +CROARING_UNTARGET_AVX512 +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +CROARING_TARGET_AVX2 +ALLOW_UNALIGNED +static inline bool _avx2_bitset_container_equals(const bitset_container_t *container1, const bitset_container_t *container2) { + const __m256i *ptr1 = (const __m256i*)container1->words; + const __m256i *ptr2 = (const __m256i*)container2->words; + for (size_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS*sizeof(uint64_t)/32; i++) { + __m256i r1 = _mm256_loadu_si256(ptr1+i); + __m256i r2 = _mm256_loadu_si256(ptr2+i); + int mask = _mm256_movemask_epi8(_mm256_cmpeq_epi8(r1, r2)); + if ((uint32_t)mask != UINT32_MAX) { + return false; + } + } + return true; +} +CROARING_UNTARGET_AVX2 +#endif // CROARING_IS_X64 + +ALLOW_UNALIGNED +bool bitset_container_equals(const bitset_container_t *container1, const bitset_container_t *container2) { + if((container1->cardinality != BITSET_UNKNOWN_CARDINALITY) && (container2->cardinality != BITSET_UNKNOWN_CARDINALITY)) { + if(container1->cardinality != container2->cardinality) { + return false; + } + if (container1->cardinality == INT32_C(0x10000)) { + return true; + } + } +#if CROARING_IS_X64 + int support = croaring_hardware_support(); +#if CROARING_COMPILER_SUPPORTS_AVX512 + if( support & ROARING_SUPPORTS_AVX512 ) { + return _avx512_bitset_container_equals(container1, container2); + } + else +#endif + if( support & ROARING_SUPPORTS_AVX2 ) { + return _avx2_bitset_container_equals(container1, container2); + } +#endif + return memcmp(container1->words, + container2->words, + BITSET_CONTAINER_SIZE_IN_WORDS*sizeof(uint64_t)) == 0; +} + +bool bitset_container_is_subset(const bitset_container_t *container1, + const bitset_container_t *container2) { + if((container1->cardinality != BITSET_UNKNOWN_CARDINALITY) && (container2->cardinality != BITSET_UNKNOWN_CARDINALITY)) { + if(container1->cardinality > container2->cardinality) { + return false; + } + } + for(int32_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; ++i ) { + if((container1->words[i] & container2->words[i]) != container1->words[i]) { + return false; + } + } + return true; +} + +bool bitset_container_select(const bitset_container_t *container, uint32_t *start_rank, uint32_t rank, uint32_t *element) { + int card = bitset_container_cardinality(container); + if(rank >= *start_rank + card) { + *start_rank += card; + return false; + } + const uint64_t *words = container->words; + int32_t size; + for (int i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; i += 1) { + size = roaring_hamming(words[i]); + if(rank <= *start_rank + size) { + uint64_t w = container->words[i]; + uint16_t base = i*64; + while (w != 0) { + uint64_t t = w & (~w + 1); + int r = roaring_trailing_zeroes(w); + if(*start_rank == rank) { + *element = r+base; + return true; + } + w ^= t; + *start_rank += 1; + } + } + else + *start_rank += size; + } + assert(false); + roaring_unreachable; +} + + +/* Returns the smallest value (assumes not empty) */ +uint16_t bitset_container_minimum(const bitset_container_t *container) { + for (int32_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; ++i ) { + uint64_t w = container->words[i]; + if (w != 0) { + int r = roaring_trailing_zeroes(w); + return r + i * 64; + } + } + return UINT16_MAX; +} + +/* Returns the largest value (assumes not empty) */ +uint16_t bitset_container_maximum(const bitset_container_t *container) { + for (int32_t i = BITSET_CONTAINER_SIZE_IN_WORDS - 1; i > 0; --i ) { + uint64_t w = container->words[i]; + if (w != 0) { + int r = roaring_leading_zeroes(w); + return i * 64 + 63 - r; + } + } + return 0; +} + +/* Returns the number of values equal or smaller than x */ +int bitset_container_rank(const bitset_container_t *container, uint16_t x) { + // credit: aqrit + int sum = 0; + int i = 0; + for (int end = x / 64; i < end; i++){ + sum += roaring_hamming(container->words[i]); + } + uint64_t lastword = container->words[i]; + uint64_t lastpos = UINT64_C(1) << (x % 64); + uint64_t mask = lastpos + lastpos - 1; // smear right + sum += roaring_hamming(lastword & mask); + return sum; +} + +uint32_t bitset_container_rank_many(const bitset_container_t *container, uint64_t start_rank, const uint32_t* begin, const uint32_t* end, uint64_t* ans){ + const uint16_t high = (uint16_t)((*begin) >> 16); + int i = 0; + int sum = 0; + const uint32_t* iter = begin; + for(; iter != end; iter++) { + uint32_t x = *iter; + uint16_t xhigh = (uint16_t)(x >> 16); + if(xhigh != high) return iter - begin; // stop at next container + + uint16_t xlow = (uint16_t)x; + for(int count = xlow / 64; i < count; i++){ + sum += roaring_hamming(container->words[i]); + } + uint64_t lastword = container->words[i]; + uint64_t lastpos = UINT64_C(1) << (xlow % 64); + uint64_t mask = lastpos + lastpos - 1; // smear right + *(ans++) = start_rank + sum + roaring_hamming(lastword & mask); + } + return iter - begin; +} + + +/* Returns the index of x , if not exsist return -1 */ +int bitset_container_get_index(const bitset_container_t *container, uint16_t x) { + if (bitset_container_get(container, x)) { + // credit: aqrit + int sum = 0; + int i = 0; + for (int end = x / 64; i < end; i++){ + sum += roaring_hamming(container->words[i]); + } + uint64_t lastword = container->words[i]; + uint64_t lastpos = UINT64_C(1) << (x % 64); + uint64_t mask = lastpos + lastpos - 1; // smear right + sum += roaring_hamming(lastword & mask); + return sum - 1; + } else { + return -1; + } +} + +/* Returns the index of the first value equal or larger than x, or -1 */ +int bitset_container_index_equalorlarger(const bitset_container_t *container, uint16_t x) { + uint32_t x32 = x; + uint32_t k = x32 / 64; + uint64_t word = container->words[k]; + const int diff = x32 - k * 64; // in [0,64) + word = (word >> diff) << diff; // a mask is faster, but we don't care + while(word == 0) { + k++; + if(k == BITSET_CONTAINER_SIZE_IN_WORDS) return -1; + word = container->words[k]; + } + return k * 64 + roaring_trailing_zeroes(word); +} + +#ifdef __cplusplus +} } } // extern "C" { namespace roaring { namespace internal { +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif
\ No newline at end of file diff --git a/contrib/libs/croaring/src/containers/containers.c b/contrib/libs/croaring/src/containers/containers.c new file mode 100644 index 00000000000..eb5a346be59 --- /dev/null +++ b/contrib/libs/croaring/src/containers/containers.c @@ -0,0 +1,715 @@ + +#include <roaring/containers/containers.h> +#include <roaring/memory.h> + +#ifdef __cplusplus +extern "C" { +// In Windows MSVC C++ compiler, (type){init} does not compile, +// it causes C4576: a parenthesized type followed by an initializer list is a +// non-standard explicit type conversion syntax The correct syntax is type{init} +#define ROARING_INIT_ROARING_CONTAINER_ITERATOR_T roaring_container_iterator_t +namespace roaring { +namespace internal { +#else +#define ROARING_INIT_ROARING_CONTAINER_ITERATOR_T (roaring_container_iterator_t) +#endif + +static inline uint32_t minimum_uint32(uint32_t a, uint32_t b) { + return (a < b) ? a : b; +} + +extern inline const container_t *container_unwrap_shared( + const container_t *candidate_shared_container, uint8_t *type); + +extern inline container_t *container_mutable_unwrap_shared( + container_t *candidate_shared_container, uint8_t *type); + +extern inline int container_get_cardinality(const container_t *c, + uint8_t typecode); + +extern inline container_t *container_iand(container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type); + +extern inline container_t *container_ior(container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type); + +extern inline container_t *container_ixor(container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type); + +extern inline container_t *container_iandnot(container_t *c1, uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type); + +void container_free(container_t *c, uint8_t type) { + switch (type) { + case BITSET_CONTAINER_TYPE: + bitset_container_free(CAST_bitset(c)); + break; + case ARRAY_CONTAINER_TYPE: + array_container_free(CAST_array(c)); + break; + case RUN_CONTAINER_TYPE: + run_container_free(CAST_run(c)); + break; + case SHARED_CONTAINER_TYPE: + shared_container_free(CAST_shared(c)); + break; + default: + assert(false); + roaring_unreachable; + } +} + +void container_printf(const container_t *c, uint8_t type) { + c = container_unwrap_shared(c, &type); + switch (type) { + case BITSET_CONTAINER_TYPE: + bitset_container_printf(const_CAST_bitset(c)); + return; + case ARRAY_CONTAINER_TYPE: + array_container_printf(const_CAST_array(c)); + return; + case RUN_CONTAINER_TYPE: + run_container_printf(const_CAST_run(c)); + return; + default: + roaring_unreachable; + } +} + +void container_printf_as_uint32_array(const container_t *c, uint8_t typecode, + uint32_t base) { + c = container_unwrap_shared(c, &typecode); + switch (typecode) { + case BITSET_CONTAINER_TYPE: + bitset_container_printf_as_uint32_array(const_CAST_bitset(c), base); + return; + case ARRAY_CONTAINER_TYPE: + array_container_printf_as_uint32_array(const_CAST_array(c), base); + return; + case RUN_CONTAINER_TYPE: + run_container_printf_as_uint32_array(const_CAST_run(c), base); + return; + default: + roaring_unreachable; + } +} + +bool container_internal_validate(const container_t *container, uint8_t typecode, + const char **reason) { + if (container == NULL) { + *reason = "container is NULL"; + return false; + } + // Not using container_unwrap_shared because it asserts if shared containers + // are nested + if (typecode == SHARED_CONTAINER_TYPE) { + const shared_container_t *shared_container = + const_CAST_shared(container); + if (croaring_refcount_get(&shared_container->counter) == 0) { + *reason = "shared container has zero refcount"; + return false; + } + if (shared_container->typecode == SHARED_CONTAINER_TYPE) { + *reason = "shared container is nested"; + return false; + } + if (shared_container->container == NULL) { + *reason = "shared container has NULL container"; + return false; + } + container = shared_container->container; + typecode = shared_container->typecode; + } + switch (typecode) { + case BITSET_CONTAINER_TYPE: + return bitset_container_validate(const_CAST_bitset(container), + reason); + case ARRAY_CONTAINER_TYPE: + return array_container_validate(const_CAST_array(container), + reason); + case RUN_CONTAINER_TYPE: + return run_container_validate(const_CAST_run(container), reason); + default: + *reason = "invalid typecode"; + return false; + } +} + +extern inline bool container_nonzero_cardinality(const container_t *c, + uint8_t typecode); + +extern inline int container_to_uint32_array(uint32_t *output, + const container_t *c, + uint8_t typecode, uint32_t base); + +extern inline container_t *container_add(container_t *c, uint16_t val, + uint8_t typecode, // !!! 2nd arg? + uint8_t *new_typecode); + +extern inline bool container_contains(const container_t *c, uint16_t val, + uint8_t typecode); // !!! 2nd arg? + +extern inline container_t *container_and(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type); + +extern inline container_t *container_or(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type); + +extern inline container_t *container_xor(const container_t *c1, uint8_t type1, + const container_t *c2, uint8_t type2, + uint8_t *result_type); + +container_t *get_copy_of_container(container_t *c, uint8_t *typecode, + bool copy_on_write) { + if (copy_on_write) { + shared_container_t *shared_container; + if (*typecode == SHARED_CONTAINER_TYPE) { + shared_container = CAST_shared(c); + croaring_refcount_inc(&shared_container->counter); + return shared_container; + } + assert(*typecode != SHARED_CONTAINER_TYPE); + + if ((shared_container = (shared_container_t *)roaring_malloc( + sizeof(shared_container_t))) == NULL) { + return NULL; + } + + shared_container->container = c; + shared_container->typecode = *typecode; + // At this point, we are creating new shared container + // so there should be no other references, and setting + // the counter to 2 - even non-atomically - is safe as + // long as the value is set before the return statement. + shared_container->counter = 2; + *typecode = SHARED_CONTAINER_TYPE; + + return shared_container; + } // copy_on_write + // otherwise, no copy on write... + const container_t *actual_container = container_unwrap_shared(c, typecode); + assert(*typecode != SHARED_CONTAINER_TYPE); + return container_clone(actual_container, *typecode); +} + +/** + * Copies a container, requires a typecode. This allocates new memory, caller + * is responsible for deallocation. + */ +container_t *container_clone(const container_t *c, uint8_t typecode) { + // We do not want to allow cloning of shared containers. + // c = container_unwrap_shared(c, &typecode); + switch (typecode) { + case BITSET_CONTAINER_TYPE: + return bitset_container_clone(const_CAST_bitset(c)); + case ARRAY_CONTAINER_TYPE: + return array_container_clone(const_CAST_array(c)); + case RUN_CONTAINER_TYPE: + return run_container_clone(const_CAST_run(c)); + case SHARED_CONTAINER_TYPE: + // Shared containers are not cloneable. Are you mixing COW and + // non-COW bitmaps? + return NULL; + default: + assert(false); + roaring_unreachable; + return NULL; + } +} + +container_t *shared_container_extract_copy(shared_container_t *sc, + uint8_t *typecode) { + assert(sc->typecode != SHARED_CONTAINER_TYPE); + *typecode = sc->typecode; + container_t *answer; + if (croaring_refcount_dec(&sc->counter)) { + answer = sc->container; + sc->container = NULL; // paranoid + roaring_free(sc); + } else { + answer = container_clone(sc->container, *typecode); + } + assert(*typecode != SHARED_CONTAINER_TYPE); + return answer; +} + +void shared_container_free(shared_container_t *container) { + if (croaring_refcount_dec(&container->counter)) { + assert(container->typecode != SHARED_CONTAINER_TYPE); + container_free(container->container, container->typecode); + container->container = NULL; // paranoid + roaring_free(container); + } +} + +extern inline container_t *container_not(const container_t *c1, uint8_t type1, + uint8_t *result_type); + +extern inline container_t *container_not_range(const container_t *c1, + uint8_t type1, + uint32_t range_start, + uint32_t range_end, + uint8_t *result_type); + +extern inline container_t *container_inot(container_t *c1, uint8_t type1, + uint8_t *result_type); + +extern inline container_t *container_inot_range(container_t *c1, uint8_t type1, + uint32_t range_start, + uint32_t range_end, + uint8_t *result_type); + +extern inline container_t *container_range_of_ones(uint32_t range_start, + uint32_t range_end, + uint8_t *result_type); + +// where are the correponding things for union and intersection?? +extern inline container_t *container_lazy_xor(const container_t *c1, + uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type); + +extern inline container_t *container_lazy_ixor(container_t *c1, uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type); + +extern inline container_t *container_andnot(const container_t *c1, + uint8_t type1, + const container_t *c2, + uint8_t type2, + uint8_t *result_type); + +roaring_container_iterator_t container_init_iterator(const container_t *c, + uint8_t typecode, + uint16_t *value) { + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t wordindex = 0; + uint64_t word; + while ((word = bc->words[wordindex]) == 0) { + wordindex++; + } + // word is non-zero + int32_t index = wordindex * 64 + roaring_trailing_zeroes(word); + *value = index; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{ + .index = index, + }; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + *value = ac->array[0]; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{ + .index = 0, + }; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + *value = rc->runs[0].value; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{ + .index = 0, + }; + } + default: + assert(false); + roaring_unreachable; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{0}; + } +} + +roaring_container_iterator_t container_init_iterator_last(const container_t *c, + uint8_t typecode, + uint16_t *value) { + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t wordindex = BITSET_CONTAINER_SIZE_IN_WORDS - 1; + uint64_t word; + while ((word = bc->words[wordindex]) == 0) { + wordindex--; + } + // word is non-zero + int32_t index = + wordindex * 64 + (63 - roaring_leading_zeroes(word)); + *value = index; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{ + .index = index, + }; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + int32_t index = ac->cardinality - 1; + *value = ac->array[index]; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{ + .index = index, + }; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + int32_t run_index = rc->n_runs - 1; + const rle16_t *last_run = &rc->runs[run_index]; + *value = last_run->value + last_run->length; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{ + .index = run_index, + }; + } + default: + assert(false); + roaring_unreachable; + return ROARING_INIT_ROARING_CONTAINER_ITERATOR_T{0}; + } +} + +bool container_iterator_next(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value) { + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + it->index++; + + uint32_t wordindex = it->index / 64; + if (wordindex >= BITSET_CONTAINER_SIZE_IN_WORDS) { + return false; + } + + uint64_t word = + bc->words[wordindex] & (UINT64_MAX << (it->index % 64)); + // next part could be optimized/simplified + while (word == 0 && + (wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS)) { + wordindex++; + word = bc->words[wordindex]; + } + if (word != 0) { + it->index = wordindex * 64 + roaring_trailing_zeroes(word); + *value = it->index; + return true; + } + return false; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + it->index++; + if (it->index < ac->cardinality) { + *value = ac->array[it->index]; + return true; + } + return false; + } + case RUN_CONTAINER_TYPE: { + if (*value == UINT16_MAX) { // Avoid overflow to zero + return false; + } + + const run_container_t *rc = const_CAST_run(c); + uint32_t limit = + rc->runs[it->index].value + rc->runs[it->index].length; + if (*value < limit) { + (*value)++; + return true; + } + + it->index++; + if (it->index < rc->n_runs) { + *value = rc->runs[it->index].value; + return true; + } + return false; + } + default: + assert(false); + roaring_unreachable; + return false; + } +} + +bool container_iterator_prev(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value) { + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + if (--it->index < 0) { + return false; + } + + const bitset_container_t *bc = const_CAST_bitset(c); + int32_t wordindex = it->index / 64; + uint64_t word = + bc->words[wordindex] & (UINT64_MAX >> (63 - (it->index % 64))); + + while (word == 0 && --wordindex >= 0) { + word = bc->words[wordindex]; + } + if (word == 0) { + return false; + } + + it->index = (wordindex * 64) + (63 - roaring_leading_zeroes(word)); + *value = it->index; + return true; + } + case ARRAY_CONTAINER_TYPE: { + if (--it->index < 0) { + return false; + } + const array_container_t *ac = const_CAST_array(c); + *value = ac->array[it->index]; + return true; + } + case RUN_CONTAINER_TYPE: { + if (*value == 0) { + return false; + } + + const run_container_t *rc = const_CAST_run(c); + (*value)--; + if (*value >= rc->runs[it->index].value) { + return true; + } + + if (--it->index < 0) { + return false; + } + + *value = rc->runs[it->index].value + rc->runs[it->index].length; + return true; + } + default: + assert(false); + roaring_unreachable; + return false; + } +} + +bool container_iterator_lower_bound(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint16_t *value_out, uint16_t val) { + if (val > container_maximum(c, typecode)) { + return false; + } + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + it->index = bitset_container_index_equalorlarger(bc, val); + *value_out = it->index; + return true; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + it->index = array_container_index_equalorlarger(ac, val); + *value_out = ac->array[it->index]; + return true; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + it->index = run_container_index_equalorlarger(rc, val); + if (rc->runs[it->index].value <= val) { + *value_out = val; + } else { + *value_out = rc->runs[it->index].value; + } + return true; + } + default: + assert(false); + roaring_unreachable; + return false; + } +} + +bool container_iterator_read_into_uint32(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint32_t high16, uint32_t *buf, + uint32_t count, uint32_t *consumed, + uint16_t *value_out) { + *consumed = 0; + if (count == 0) { + return false; + } + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t wordindex = it->index / 64; + uint64_t word = + bc->words[wordindex] & (UINT64_MAX << (it->index % 64)); + do { + // Read set bits. + while (word != 0 && *consumed < count) { + *buf = high16 | + (wordindex * 64 + roaring_trailing_zeroes(word)); + word = word & (word - 1); + buf++; + (*consumed)++; + } + // Skip unset bits. + while (word == 0 && + wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS) { + wordindex++; + word = bc->words[wordindex]; + } + } while (word != 0 && *consumed < count); + + if (word != 0) { + it->index = wordindex * 64 + roaring_trailing_zeroes(word); + *value_out = it->index; + return true; + } + return false; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint32_t num_values = + minimum_uint32(ac->cardinality - it->index, count); + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high16 | ac->array[it->index + i]; + } + *consumed += num_values; + it->index += num_values; + if (it->index < ac->cardinality) { + *value_out = ac->array[it->index]; + return true; + } + return false; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + do { + uint32_t largest_run_value = + rc->runs[it->index].value + rc->runs[it->index].length; + uint32_t num_values = minimum_uint32( + largest_run_value - *value_out + 1, count - *consumed); + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high16 | (*value_out + i); + } + *value_out += num_values; + buf += num_values; + *consumed += num_values; + + // We check for `value == 0` because `it->value += num_values` + // can overflow when `value == UINT16_MAX`, and `count > + // length`. In this case `value` will overflow to 0. + if (*value_out > largest_run_value || *value_out == 0) { + it->index++; + if (it->index < rc->n_runs) { + *value_out = rc->runs[it->index].value; + } else { + return false; + } + } + } while (*consumed < count); + return true; + } + default: + assert(false); + roaring_unreachable; + return 0; + } +} + +bool container_iterator_read_into_uint64(const container_t *c, uint8_t typecode, + roaring_container_iterator_t *it, + uint64_t high48, uint64_t *buf, + uint32_t count, uint32_t *consumed, + uint16_t *value_out) { + *consumed = 0; + if (count == 0) { + return false; + } + switch (typecode) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = const_CAST_bitset(c); + uint32_t wordindex = it->index / 64; + uint64_t word = + bc->words[wordindex] & (UINT64_MAX << (it->index % 64)); + do { + // Read set bits. + while (word != 0 && *consumed < count) { + *buf = high48 | + (wordindex * 64 + roaring_trailing_zeroes(word)); + word = word & (word - 1); + buf++; + (*consumed)++; + } + // Skip unset bits. + while (word == 0 && + wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS) { + wordindex++; + word = bc->words[wordindex]; + } + } while (word != 0 && *consumed < count); + + if (word != 0) { + it->index = wordindex * 64 + roaring_trailing_zeroes(word); + *value_out = it->index; + return true; + } + return false; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = const_CAST_array(c); + uint32_t num_values = + minimum_uint32(ac->cardinality - it->index, count); + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high48 | ac->array[it->index + i]; + } + *consumed += num_values; + it->index += num_values; + if (it->index < ac->cardinality) { + *value_out = ac->array[it->index]; + return true; + } + return false; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(c); + do { + uint32_t largest_run_value = + rc->runs[it->index].value + rc->runs[it->index].length; + uint32_t num_values = minimum_uint32( + largest_run_value - *value_out + 1, count - *consumed); + for (uint32_t i = 0; i < num_values; i++) { + buf[i] = high48 | (*value_out + i); + } + *value_out += num_values; + buf += num_values; + *consumed += num_values; + + // We check for `value == 0` because `it->value += num_values` + // can overflow when `value == UINT16_MAX`, and `count > + // length`. In this case `value` will overflow to 0. + if (*value_out > largest_run_value || *value_out == 0) { + it->index++; + if (it->index < rc->n_runs) { + *value_out = rc->runs[it->index].value; + } else { + return false; + } + } + } while (*consumed < count); + return true; + } + default: + assert(false); + roaring_unreachable; + return 0; + } +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif + +#undef ROARING_INIT_ROARING_CONTAINER_ITERATOR_T diff --git a/contrib/libs/croaring/src/containers/convert.c b/contrib/libs/croaring/src/containers/convert.c new file mode 100644 index 00000000000..6cd31d34c33 --- /dev/null +++ b/contrib/libs/croaring/src/containers/convert.c @@ -0,0 +1,354 @@ +#include <stdio.h> + +#include <roaring/bitset_util.h> +#include <roaring/containers/containers.h> +#include <roaring/containers/convert.h> +#include <roaring/containers/perfparameters.h> + +#if CROARING_IS_X64 +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +// file contains grubby stuff that must know impl. details of all container +// types. +bitset_container_t *bitset_container_from_array(const array_container_t *ac) { + bitset_container_t *ans = bitset_container_create(); + int limit = array_container_cardinality(ac); + for (int i = 0; i < limit; ++i) bitset_container_set(ans, ac->array[i]); + return ans; +} + +bitset_container_t *bitset_container_from_run(const run_container_t *arr) { + int card = run_container_cardinality(arr); + bitset_container_t *answer = bitset_container_create(); + for (int rlepos = 0; rlepos < arr->n_runs; ++rlepos) { + rle16_t vl = arr->runs[rlepos]; + bitset_set_lenrange(answer->words, vl.value, vl.length); + } + answer->cardinality = card; + return answer; +} + +array_container_t *array_container_from_run(const run_container_t *arr) { + array_container_t *answer = + array_container_create_given_capacity(run_container_cardinality(arr)); + answer->cardinality = 0; + for (int rlepos = 0; rlepos < arr->n_runs; ++rlepos) { + int run_start = arr->runs[rlepos].value; + int run_end = run_start + arr->runs[rlepos].length; + + for (int run_value = run_start; run_value <= run_end; ++run_value) { + answer->array[answer->cardinality++] = (uint16_t)run_value; + } + } + return answer; +} + +array_container_t *array_container_from_bitset(const bitset_container_t *bits) { + array_container_t *result = + array_container_create_given_capacity(bits->cardinality); + result->cardinality = bits->cardinality; +#if CROARING_IS_X64 +#if CROARING_COMPILER_SUPPORTS_AVX512 + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX512) { + bitset_extract_setbits_avx512_uint16( + bits->words, BITSET_CONTAINER_SIZE_IN_WORDS, result->array, + bits->cardinality, 0); + } else +#endif + { + // sse version ends up being slower here + // (bitset_extract_setbits_sse_uint16) + // because of the sparsity of the data + bitset_extract_setbits_uint16( + bits->words, BITSET_CONTAINER_SIZE_IN_WORDS, result->array, 0); + } +#else + // If the system is not x64, then we have no accelerated function. + bitset_extract_setbits_uint16(bits->words, BITSET_CONTAINER_SIZE_IN_WORDS, + result->array, 0); +#endif + + return result; +} + +/* assumes that container has adequate space. Run from [s,e] (inclusive) */ +static void add_run(run_container_t *rc, int s, int e) { + rc->runs[rc->n_runs].value = s; + rc->runs[rc->n_runs].length = e - s; + rc->n_runs++; +} + +run_container_t *run_container_from_array(const array_container_t *c) { + int32_t n_runs = array_container_number_of_runs(c); + run_container_t *answer = run_container_create_given_capacity(n_runs); + int prev = -2; + int run_start = -1; + int32_t card = c->cardinality; + if (card == 0) return answer; + for (int i = 0; i < card; ++i) { + const uint16_t cur_val = c->array[i]; + if (cur_val != prev + 1) { + // new run starts; flush old one, if any + if (run_start != -1) add_run(answer, run_start, prev); + run_start = cur_val; + } + prev = c->array[i]; + } + // now prev is the last seen value + add_run(answer, run_start, prev); + // assert(run_container_cardinality(answer) == c->cardinality); + return answer; +} + +/** + * Convert the runcontainer to either a Bitmap or an Array Container, depending + * on the cardinality. Frees the container. + * Allocates and returns new container, which caller is responsible for freeing. + * It does not free the run container. + */ +container_t *convert_to_bitset_or_array_container(run_container_t *rc, + int32_t card, + uint8_t *resulttype) { + if (card <= DEFAULT_MAX_SIZE) { + array_container_t *answer = array_container_create_given_capacity(card); + answer->cardinality = 0; + for (int rlepos = 0; rlepos < rc->n_runs; ++rlepos) { + uint16_t run_start = rc->runs[rlepos].value; + uint16_t run_end = run_start + rc->runs[rlepos].length; + for (uint16_t run_value = run_start; run_value < run_end; + ++run_value) { + answer->array[answer->cardinality++] = run_value; + } + answer->array[answer->cardinality++] = run_end; + } + assert(card == answer->cardinality); + *resulttype = ARRAY_CONTAINER_TYPE; + // run_container_free(r); + return answer; + } + bitset_container_t *answer = bitset_container_create(); + for (int rlepos = 0; rlepos < rc->n_runs; ++rlepos) { + uint16_t run_start = rc->runs[rlepos].value; + bitset_set_lenrange(answer->words, run_start, rc->runs[rlepos].length); + } + answer->cardinality = card; + *resulttype = BITSET_CONTAINER_TYPE; + // run_container_free(r); + return answer; +} + +/* Converts a run container to either an array or a bitset, IF it saves space. + */ +/* If a conversion occurs, the caller is responsible to free the original + * container and + * he becomes responsible to free the new one. */ +container_t *convert_run_to_efficient_container(run_container_t *c, + uint8_t *typecode_after) { + int32_t size_as_run_container = + run_container_serialized_size_in_bytes(c->n_runs); + + int32_t size_as_bitset_container = + bitset_container_serialized_size_in_bytes(); + int32_t card = run_container_cardinality(c); + int32_t size_as_array_container = + array_container_serialized_size_in_bytes(card); + + int32_t min_size_non_run = + size_as_bitset_container < size_as_array_container + ? size_as_bitset_container + : size_as_array_container; + if (size_as_run_container <= min_size_non_run) { // no conversion + *typecode_after = RUN_CONTAINER_TYPE; + return c; + } + if (card <= DEFAULT_MAX_SIZE) { + // to array + array_container_t *answer = array_container_create_given_capacity(card); + answer->cardinality = 0; + for (int rlepos = 0; rlepos < c->n_runs; ++rlepos) { + int run_start = c->runs[rlepos].value; + int run_end = run_start + c->runs[rlepos].length; + + for (int run_value = run_start; run_value <= run_end; ++run_value) { + answer->array[answer->cardinality++] = (uint16_t)run_value; + } + } + *typecode_after = ARRAY_CONTAINER_TYPE; + return answer; + } + + // else to bitset + bitset_container_t *answer = bitset_container_create(); + + for (int rlepos = 0; rlepos < c->n_runs; ++rlepos) { + int start = c->runs[rlepos].value; + int end = start + c->runs[rlepos].length; + bitset_set_range(answer->words, start, end + 1); + } + answer->cardinality = card; + *typecode_after = BITSET_CONTAINER_TYPE; + return answer; +} + +// like convert_run_to_efficient_container but frees the old result if needed +container_t *convert_run_to_efficient_container_and_free( + run_container_t *c, uint8_t *typecode_after) { + container_t *answer = convert_run_to_efficient_container(c, typecode_after); + if (answer != c) run_container_free(c); + return answer; +} + +/* once converted, the original container is disposed here, rather than + in roaring_array +*/ + +// TODO: split into run- array- and bitset- subfunctions for sanity; +// a few function calls won't really matter. + +container_t *convert_run_optimize(container_t *c, uint8_t typecode_original, + uint8_t *typecode_after) { + if (typecode_original == RUN_CONTAINER_TYPE) { + container_t *newc = + convert_run_to_efficient_container(CAST_run(c), typecode_after); + if (newc != c) { + container_free(c, typecode_original); + } + return newc; + } else if (typecode_original == ARRAY_CONTAINER_TYPE) { + // it might need to be converted to a run container. + array_container_t *c_qua_array = CAST_array(c); + int32_t n_runs = array_container_number_of_runs(c_qua_array); + int32_t size_as_run_container = + run_container_serialized_size_in_bytes(n_runs); + int32_t card = array_container_cardinality(c_qua_array); + int32_t size_as_array_container = + array_container_serialized_size_in_bytes(card); + + if (size_as_run_container >= size_as_array_container) { + *typecode_after = ARRAY_CONTAINER_TYPE; + return c; + } + // else convert array to run container + run_container_t *answer = run_container_create_given_capacity(n_runs); + int prev = -2; + int run_start = -1; + + assert(card > 0); + for (int i = 0; i < card; ++i) { + uint16_t cur_val = c_qua_array->array[i]; + if (cur_val != prev + 1) { + // new run starts; flush old one, if any + if (run_start != -1) add_run(answer, run_start, prev); + run_start = cur_val; + } + prev = c_qua_array->array[i]; + } + assert(run_start >= 0); + // now prev is the last seen value + add_run(answer, run_start, prev); + *typecode_after = RUN_CONTAINER_TYPE; + array_container_free(c_qua_array); + return answer; + } else if (typecode_original == + BITSET_CONTAINER_TYPE) { // run conversions on bitset + // does bitset need conversion to run? + bitset_container_t *c_qua_bitset = CAST_bitset(c); + int32_t n_runs = bitset_container_number_of_runs(c_qua_bitset); + int32_t size_as_run_container = + run_container_serialized_size_in_bytes(n_runs); + int32_t size_as_bitset_container = + bitset_container_serialized_size_in_bytes(); + + if (size_as_bitset_container <= size_as_run_container) { + // no conversion needed. + *typecode_after = BITSET_CONTAINER_TYPE; + return c; + } + // bitset to runcontainer (ported from Java RunContainer( + // BitmapContainer bc, int nbrRuns)) + assert(n_runs > 0); // no empty bitmaps + run_container_t *answer = run_container_create_given_capacity(n_runs); + + int long_ctr = 0; + uint64_t cur_word = c_qua_bitset->words[0]; + while (true) { + while (cur_word == UINT64_C(0) && + long_ctr < BITSET_CONTAINER_SIZE_IN_WORDS - 1) + cur_word = c_qua_bitset->words[++long_ctr]; + + if (cur_word == UINT64_C(0)) { + bitset_container_free(c_qua_bitset); + *typecode_after = RUN_CONTAINER_TYPE; + return answer; + } + + int local_run_start = roaring_trailing_zeroes(cur_word); + int run_start = local_run_start + 64 * long_ctr; + uint64_t cur_word_with_1s = cur_word | (cur_word - 1); + + int run_end = 0; + while (cur_word_with_1s == UINT64_C(0xFFFFFFFFFFFFFFFF) && + long_ctr < BITSET_CONTAINER_SIZE_IN_WORDS - 1) + cur_word_with_1s = c_qua_bitset->words[++long_ctr]; + + if (cur_word_with_1s == UINT64_C(0xFFFFFFFFFFFFFFFF)) { + run_end = 64 + long_ctr * 64; // exclusive, I guess + add_run(answer, run_start, run_end - 1); + bitset_container_free(c_qua_bitset); + *typecode_after = RUN_CONTAINER_TYPE; + return answer; + } + int local_run_end = roaring_trailing_zeroes(~cur_word_with_1s); + run_end = local_run_end + long_ctr * 64; + add_run(answer, run_start, run_end - 1); + cur_word = cur_word_with_1s & (cur_word_with_1s + 1); + } + return answer; + } else { + assert(false); + roaring_unreachable; + return NULL; + } +} + +container_t *container_from_run_range(const run_container_t *run, uint32_t min, + uint32_t max, uint8_t *typecode_after) { + // We expect most of the time to end up with a bitset container + bitset_container_t *bitset = bitset_container_create(); + *typecode_after = BITSET_CONTAINER_TYPE; + int32_t union_cardinality = 0; + for (int32_t i = 0; i < run->n_runs; ++i) { + uint32_t rle_min = run->runs[i].value; + uint32_t rle_max = rle_min + run->runs[i].length; + bitset_set_lenrange(bitset->words, rle_min, rle_max - rle_min); + union_cardinality += run->runs[i].length + 1; + } + union_cardinality += max - min + 1; + union_cardinality -= + bitset_lenrange_cardinality(bitset->words, min, max - min); + bitset_set_lenrange(bitset->words, min, max - min); + bitset->cardinality = union_cardinality; + if (bitset->cardinality <= DEFAULT_MAX_SIZE) { + // we need to convert to an array container + array_container_t *array = array_container_from_bitset(bitset); + *typecode_after = ARRAY_CONTAINER_TYPE; + bitset_container_free(bitset); + return array; + } + return bitset; +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif diff --git a/contrib/libs/croaring/src/containers/mixed_andnot.c b/contrib/libs/croaring/src/containers/mixed_andnot.c new file mode 100644 index 00000000000..8744c84c7b5 --- /dev/null +++ b/contrib/libs/croaring/src/containers/mixed_andnot.c @@ -0,0 +1,521 @@ +/* + * mixed_andnot.c. More methods since operation is not symmetric, + * except no "wide" andnot , so no lazy options motivated. + */ + +#include <assert.h> +#include <string.h> + +#include <roaring/array_util.h> +#include <roaring/bitset_util.h> +#include <roaring/containers/containers.h> +#include <roaring/containers/convert.h> +#include <roaring/containers/mixed_andnot.h> +#include <roaring/containers/perfparameters.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst, a valid array container that could be the same as dst.*/ +void array_bitset_container_andnot(const array_container_t *src_1, + const bitset_container_t *src_2, + array_container_t *dst) { + // follows Java implementation as of June 2016 + if (dst->capacity < src_1->cardinality) { + array_container_grow(dst, src_1->cardinality, false); + } + int32_t newcard = 0; + const int32_t origcard = src_1->cardinality; + for (int i = 0; i < origcard; ++i) { + uint16_t key = src_1->array[i]; + dst->array[newcard] = key; + newcard += 1 - bitset_container_contains(src_2, key); + } + dst->cardinality = newcard; +} + +/* Compute the andnot of src_1 and src_2 and write the result to + * src_1 */ + +void array_bitset_container_iandnot(array_container_t *src_1, + const bitset_container_t *src_2) { + array_bitset_container_andnot(src_1, src_2, src_1); +} + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst, which does not initially have a valid container. + * Return true for a bitset result; false for array + */ + +bool bitset_array_container_andnot(const bitset_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { + // Java did this directly, but we have option of asm or avx + bitset_container_t *result = bitset_container_create(); + bitset_container_copy(src_1, result); + result->cardinality = + (int32_t)bitset_clear_list(result->words, (uint64_t)result->cardinality, + src_2->array, (uint64_t)src_2->cardinality); + + // do required type conversions. + if (result->cardinality <= DEFAULT_MAX_SIZE) { + *dst = array_container_from_bitset(result); + bitset_container_free(result); + return false; + } + *dst = result; + return true; +} + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst (which has no container initially). It will modify src_1 + * to be dst if the result is a bitset. Otherwise, it will + * free src_1 and dst will be a new array container. In both + * cases, the caller is responsible for deallocating dst. + * Returns true iff dst is a bitset */ + +bool bitset_array_container_iandnot(bitset_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { + *dst = src_1; + src_1->cardinality = + (int32_t)bitset_clear_list(src_1->words, (uint64_t)src_1->cardinality, + src_2->array, (uint64_t)src_2->cardinality); + + if (src_1->cardinality <= DEFAULT_MAX_SIZE) { + *dst = array_container_from_bitset(src_1); + bitset_container_free(src_1); + return false; // not bitset + } else + return true; +} + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst. Result may be either a bitset or an array container + * (returns "result is bitset"). dst does not initially have + * any container, but becomes either a bitset container (return + * result true) or an array container. + */ + +bool run_bitset_container_andnot(const run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { + // follows the Java implementation as of June 2016 + int card = run_container_cardinality(src_1); + if (card <= DEFAULT_MAX_SIZE) { + // must be an array + array_container_t *answer = array_container_create_given_capacity(card); + answer->cardinality = 0; + for (int32_t rlepos = 0; rlepos < src_1->n_runs; ++rlepos) { + rle16_t rle = src_1->runs[rlepos]; + for (int run_value = rle.value; run_value <= rle.value + rle.length; + ++run_value) { + if (!bitset_container_get(src_2, (uint16_t)run_value)) { + answer->array[answer->cardinality++] = (uint16_t)run_value; + } + } + } + *dst = answer; + return false; + } else { // we guess it will be a bitset, though have to check guess when + // done + bitset_container_t *answer = bitset_container_clone(src_2); + + uint32_t last_pos = 0; + for (int32_t rlepos = 0; rlepos < src_1->n_runs; ++rlepos) { + rle16_t rle = src_1->runs[rlepos]; + + uint32_t start = rle.value; + uint32_t end = start + rle.length + 1; + bitset_reset_range(answer->words, last_pos, start); + bitset_flip_range(answer->words, start, end); + last_pos = end; + } + bitset_reset_range(answer->words, last_pos, (uint32_t)(1 << 16)); + + answer->cardinality = bitset_container_compute_cardinality(answer); + + if (answer->cardinality <= DEFAULT_MAX_SIZE) { + *dst = array_container_from_bitset(answer); + bitset_container_free(answer); + return false; // not bitset + } + *dst = answer; + return true; // bitset + } +} + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst. Result may be either a bitset or an array container + * (returns "result is bitset"). dst does not initially have + * any container, but becomes either a bitset container (return + * result true) or an array container. + */ + +bool run_bitset_container_iandnot(run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { + // dummy implementation + bool ans = run_bitset_container_andnot(src_1, src_2, dst); + run_container_free(src_1); + return ans; +} + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst. Result may be either a bitset or an array container + * (returns "result is bitset"). dst does not initially have + * any container, but becomes either a bitset container (return + * result true) or an array container. + */ + +bool bitset_run_container_andnot(const bitset_container_t *src_1, + const run_container_t *src_2, + container_t **dst) { + // follows Java implementation + bitset_container_t *result = bitset_container_create(); + + bitset_container_copy(src_1, result); + for (int32_t rlepos = 0; rlepos < src_2->n_runs; ++rlepos) { + rle16_t rle = src_2->runs[rlepos]; + bitset_reset_range(result->words, rle.value, + rle.value + rle.length + UINT32_C(1)); + } + result->cardinality = bitset_container_compute_cardinality(result); + + if (result->cardinality <= DEFAULT_MAX_SIZE) { + *dst = array_container_from_bitset(result); + bitset_container_free(result); + return false; // not bitset + } + *dst = result; + return true; // bitset +} + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst (which has no container initially). It will modify src_1 + * to be dst if the result is a bitset. Otherwise, it will + * free src_1 and dst will be a new array container. In both + * cases, the caller is responsible for deallocating dst. + * Returns true iff dst is a bitset */ + +bool bitset_run_container_iandnot(bitset_container_t *src_1, + const run_container_t *src_2, + container_t **dst) { + *dst = src_1; + + for (int32_t rlepos = 0; rlepos < src_2->n_runs; ++rlepos) { + rle16_t rle = src_2->runs[rlepos]; + bitset_reset_range(src_1->words, rle.value, + rle.value + rle.length + UINT32_C(1)); + } + src_1->cardinality = bitset_container_compute_cardinality(src_1); + + if (src_1->cardinality <= DEFAULT_MAX_SIZE) { + *dst = array_container_from_bitset(src_1); + bitset_container_free(src_1); + return false; // not bitset + } else + return true; +} + +/* helper. a_out must be a valid array container with adequate capacity. + * Returns the cardinality of the output container. Partly Based on Java + * implementation Util.unsignedDifference. + * + * TODO: Util.unsignedDifference does not use advanceUntil. Is it cheaper + * to avoid advanceUntil? + */ + +static int run_array_array_subtract(const run_container_t *rc, + const array_container_t *a_in, + array_container_t *a_out) { + int out_card = 0; + int32_t in_array_pos = + -1; // since advanceUntil always assumes we start the search AFTER this + + for (int rlepos = 0; rlepos < rc->n_runs; rlepos++) { + int32_t start = rc->runs[rlepos].value; + int32_t end = start + rc->runs[rlepos].length + 1; + + in_array_pos = advanceUntil(a_in->array, in_array_pos, + a_in->cardinality, (uint16_t)start); + + if (in_array_pos >= a_in->cardinality) { // run has no items subtracted + for (int32_t i = start; i < end; ++i) + a_out->array[out_card++] = (uint16_t)i; + } else { + uint16_t next_nonincluded = a_in->array[in_array_pos]; + if (next_nonincluded >= end) { + // another case when run goes unaltered + for (int32_t i = start; i < end; ++i) + a_out->array[out_card++] = (uint16_t)i; + in_array_pos--; // ensure we see this item again if necessary + } else { + for (int32_t i = start; i < end; ++i) + if (i != next_nonincluded) + a_out->array[out_card++] = (uint16_t)i; + else // 0 should ensure we don't match + next_nonincluded = + (in_array_pos + 1 >= a_in->cardinality) + ? 0 + : a_in->array[++in_array_pos]; + in_array_pos--; // see again + } + } + } + return out_card; +} + +/* dst does not indicate a valid container initially. Eventually it + * can become any type of container. + */ + +int run_array_container_andnot(const run_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { + // follows the Java impl as of June 2016 + + int card = run_container_cardinality(src_1); + const int arbitrary_threshold = 32; + + if (card <= arbitrary_threshold) { + if (src_2->cardinality == 0) { + *dst = run_container_clone(src_1); + return RUN_CONTAINER_TYPE; + } + // Java's "lazyandNot.toEfficientContainer" thing + run_container_t *answer = run_container_create_given_capacity( + card + array_container_cardinality(src_2)); + + int rlepos = 0; + int xrlepos = 0; // "x" is src_2 + rle16_t rle = src_1->runs[rlepos]; + int32_t start = rle.value; + int32_t end = start + rle.length + 1; + int32_t xstart = src_2->array[xrlepos]; + + while ((rlepos < src_1->n_runs) && (xrlepos < src_2->cardinality)) { + if (end <= xstart) { + // output the first run + answer->runs[answer->n_runs++] = + MAKE_RLE16(start, end - start - 1); + rlepos++; + if (rlepos < src_1->n_runs) { + start = src_1->runs[rlepos].value; + end = start + src_1->runs[rlepos].length + 1; + } + } else if (xstart + 1 <= start) { + // exit the second run + xrlepos++; + if (xrlepos < src_2->cardinality) { + xstart = src_2->array[xrlepos]; + } + } else { + if (start < xstart) { + answer->runs[answer->n_runs++] = + MAKE_RLE16(start, xstart - start - 1); + } + if (xstart + 1 < end) { + start = xstart + 1; + } else { + rlepos++; + if (rlepos < src_1->n_runs) { + start = src_1->runs[rlepos].value; + end = start + src_1->runs[rlepos].length + 1; + } + } + } + } + if (rlepos < src_1->n_runs) { + answer->runs[answer->n_runs++] = MAKE_RLE16(start, end - start - 1); + rlepos++; + if (rlepos < src_1->n_runs) { + memcpy(answer->runs + answer->n_runs, src_1->runs + rlepos, + (src_1->n_runs - rlepos) * sizeof(rle16_t)); + answer->n_runs += (src_1->n_runs - rlepos); + } + } + uint8_t return_type; + *dst = convert_run_to_efficient_container(answer, &return_type); + if (answer != *dst) run_container_free(answer); + return return_type; + } + // else it's a bitmap or array + + if (card <= DEFAULT_MAX_SIZE) { + array_container_t *ac = array_container_create_given_capacity(card); + // nb Java code used a generic iterator-based merge to compute + // difference + ac->cardinality = run_array_array_subtract(src_1, src_2, ac); + *dst = ac; + return ARRAY_CONTAINER_TYPE; + } + bitset_container_t *ans = bitset_container_from_run(src_1); + bool result_is_bitset = bitset_array_container_iandnot(ans, src_2, dst); + return (result_is_bitset ? BITSET_CONTAINER_TYPE : ARRAY_CONTAINER_TYPE); +} + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst (which has no container initially). It will modify src_1 + * to be dst if the result is a bitset. Otherwise, it will + * free src_1 and dst will be a new array container. In both + * cases, the caller is responsible for deallocating dst. + * Returns true iff dst is a bitset */ + +int run_array_container_iandnot(run_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { + // dummy implementation same as June 2016 Java + int ans = run_array_container_andnot(src_1, src_2, dst); + run_container_free(src_1); + return ans; +} + +/* dst must be a valid array container, allowed to be src_1 */ + +void array_run_container_andnot(const array_container_t *src_1, + const run_container_t *src_2, + array_container_t *dst) { + // basically following Java impl as of June 2016 + if (src_1->cardinality > dst->capacity) { + array_container_grow(dst, src_1->cardinality, false); + } + + if (src_2->n_runs == 0) { + memmove(dst->array, src_1->array, + sizeof(uint16_t) * src_1->cardinality); + dst->cardinality = src_1->cardinality; + return; + } + int32_t run_start = src_2->runs[0].value; + int32_t run_end = run_start + src_2->runs[0].length; + int which_run = 0; + + uint16_t val = 0; + int dest_card = 0; + for (int i = 0; i < src_1->cardinality; ++i) { + val = src_1->array[i]; + if (val < run_start) + dst->array[dest_card++] = val; + else if (val <= run_end) { + ; // omitted item + } else { + do { + if (which_run + 1 < src_2->n_runs) { + ++which_run; + run_start = src_2->runs[which_run].value; + run_end = run_start + src_2->runs[which_run].length; + + } else + run_start = run_end = (1 << 16) + 1; + } while (val > run_end); + --i; + } + } + dst->cardinality = dest_card; +} + +/* dst does not indicate a valid container initially. Eventually it + * can become any kind of container. + */ + +void array_run_container_iandnot(array_container_t *src_1, + const run_container_t *src_2) { + array_run_container_andnot(src_1, src_2, src_1); +} + +/* dst does not indicate a valid container initially. Eventually it + * can become any kind of container. + */ + +int run_run_container_andnot(const run_container_t *src_1, + const run_container_t *src_2, container_t **dst) { + run_container_t *ans = run_container_create(); + run_container_andnot(src_1, src_2, ans); + uint8_t typecode_after; + *dst = convert_run_to_efficient_container_and_free(ans, &typecode_after); + return typecode_after; +} + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst (which has no container initially). It will modify src_1 + * to be dst if the result is a bitset. Otherwise, it will + * free src_1 and dst will be a new array container. In both + * cases, the caller is responsible for deallocating dst. + * Returns true iff dst is a bitset */ + +int run_run_container_iandnot(run_container_t *src_1, + const run_container_t *src_2, container_t **dst) { + // following Java impl as of June 2016 (dummy) + int ans = run_run_container_andnot(src_1, src_2, dst); + run_container_free(src_1); + return ans; +} + +/* + * dst is a valid array container and may be the same as src_1 + */ + +void array_array_container_andnot(const array_container_t *src_1, + const array_container_t *src_2, + array_container_t *dst) { + array_container_andnot(src_1, src_2, dst); +} + +/* inplace array-array andnot will always be able to reuse the space of + * src_1 */ +void array_array_container_iandnot(array_container_t *src_1, + const array_container_t *src_2) { + array_container_andnot(src_1, src_2, src_1); +} + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst (which has no container initially). Return value is + * "dst is a bitset" + */ + +bool bitset_bitset_container_andnot(const bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { + bitset_container_t *ans = bitset_container_create(); + int card = bitset_container_andnot(src_1, src_2, ans); + if (card <= DEFAULT_MAX_SIZE) { + *dst = array_container_from_bitset(ans); + bitset_container_free(ans); + return false; // not bitset + } else { + *dst = ans; + return true; + } +} + +/* Compute the andnot of src_1 and src_2 and write the result to + * dst (which has no container initially). It will modify src_1 + * to be dst if the result is a bitset. Otherwise, it will + * free src_1 and dst will be a new array container. In both + * cases, the caller is responsible for deallocating dst. + * Returns true iff dst is a bitset */ + +bool bitset_bitset_container_iandnot(bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { + int card = bitset_container_andnot(src_1, src_2, src_1); + if (card <= DEFAULT_MAX_SIZE) { + *dst = array_container_from_bitset(src_1); + bitset_container_free(src_1); + return false; // not bitset + } else { + *dst = src_1; + return true; + } +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif diff --git a/contrib/libs/croaring/src/containers/mixed_equal.c b/contrib/libs/croaring/src/containers/mixed_equal.c new file mode 100644 index 00000000000..761431e9014 --- /dev/null +++ b/contrib/libs/croaring/src/containers/mixed_equal.c @@ -0,0 +1,88 @@ +#include <roaring/containers/mixed_equal.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +bool array_container_equal_bitset(const array_container_t* container1, + const bitset_container_t* container2) { + if (container2->cardinality != BITSET_UNKNOWN_CARDINALITY) { + if (container2->cardinality != container1->cardinality) { + return false; + } + } + int32_t pos = 0; + for (int32_t i = 0; i < BITSET_CONTAINER_SIZE_IN_WORDS; ++i) { + uint64_t w = container2->words[i]; + while (w != 0) { + uint64_t t = w & (~w + 1); + uint16_t r = i * 64 + roaring_trailing_zeroes(w); + if (pos >= container1->cardinality) { + return false; + } + if (container1->array[pos] != r) { + return false; + } + ++pos; + w ^= t; + } + } + return (pos == container1->cardinality); +} + +bool run_container_equals_array(const run_container_t* container1, + const array_container_t* container2) { + if (run_container_cardinality(container1) != container2->cardinality) + return false; + int32_t pos = 0; + for (int i = 0; i < container1->n_runs; ++i) { + const uint32_t run_start = container1->runs[i].value; + const uint32_t le = container1->runs[i].length; + + if (container2->array[pos] != run_start) { + return false; + } + + if (container2->array[pos + le] != run_start + le) { + return false; + } + + pos += le + 1; + } + return true; +} + +bool run_container_equals_bitset(const run_container_t* container1, + const bitset_container_t* container2) { + int run_card = run_container_cardinality(container1); + int bitset_card = (container2->cardinality != BITSET_UNKNOWN_CARDINALITY) + ? container2->cardinality + : bitset_container_compute_cardinality(container2); + if (bitset_card != run_card) { + return false; + } + + for (int32_t i = 0; i < container1->n_runs; i++) { + uint32_t begin = container1->runs[i].value; + if (container1->runs[i].length) { + uint32_t end = begin + container1->runs[i].length + 1; + if (!bitset_container_contains_range(container2, begin, end)) { + return false; + } + } else { + if (!bitset_container_contains(container2, begin)) { + return false; + } + } + } + + return true; +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif diff --git a/contrib/libs/croaring/src/containers/mixed_intersection.c b/contrib/libs/croaring/src/containers/mixed_intersection.c new file mode 100644 index 00000000000..56efe210b84 --- /dev/null +++ b/contrib/libs/croaring/src/containers/mixed_intersection.c @@ -0,0 +1,351 @@ +/* + * mixed_intersection.c + * + */ + +#include <roaring/array_util.h> +#include <roaring/bitset_util.h> +#include <roaring/containers/convert.h> +#include <roaring/containers/mixed_intersection.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/* Compute the intersection of src_1 and src_2 and write the result to + * dst. */ +void array_bitset_container_intersection(const array_container_t *src_1, + const bitset_container_t *src_2, + array_container_t *dst) { + if (dst->capacity < src_1->cardinality) { + array_container_grow(dst, src_1->cardinality, false); + } + int32_t newcard = 0; // dst could be src_1 + const int32_t origcard = src_1->cardinality; + for (int i = 0; i < origcard; ++i) { + uint16_t key = src_1->array[i]; + // this branchless approach is much faster... + dst->array[newcard] = key; + newcard += bitset_container_contains(src_2, key); + /** + * we could do it this way instead... + * if (bitset_container_contains(src_2, key)) { + * dst->array[newcard++] = key; + * } + * but if the result is unpredictible, the processor generates + * many mispredicted branches. + * Difference can be huge (from 3 cycles when predictible all the way + * to 16 cycles when unpredictible. + * See + * https://github.com/lemire/Code-used-on-Daniel-Lemire-s-blog/blob/master/extra/bitset/c/arraybitsetintersection.c + */ + } + dst->cardinality = newcard; +} + +/* Compute the size of the intersection of src_1 and src_2. */ +int array_bitset_container_intersection_cardinality( + const array_container_t *src_1, const bitset_container_t *src_2) { + int32_t newcard = 0; + const int32_t origcard = src_1->cardinality; + for (int i = 0; i < origcard; ++i) { + uint16_t key = src_1->array[i]; + newcard += bitset_container_contains(src_2, key); + } + return newcard; +} + +bool array_bitset_container_intersect(const array_container_t *src_1, + const bitset_container_t *src_2) { + const int32_t origcard = src_1->cardinality; + for (int i = 0; i < origcard; ++i) { + uint16_t key = src_1->array[i]; + if (bitset_container_contains(src_2, key)) return true; + } + return false; +} + +/* Compute the intersection of src_1 and src_2 and write the result to + * dst. It is allowed for dst to be equal to src_1. We assume that dst is a + * valid container. */ +void array_run_container_intersection(const array_container_t *src_1, + const run_container_t *src_2, + array_container_t *dst) { + if (run_container_is_full(src_2)) { + if (dst != src_1) array_container_copy(src_1, dst); + return; + } + if (dst->capacity < src_1->cardinality) { + array_container_grow(dst, src_1->cardinality, false); + } + if (src_2->n_runs == 0) { + return; + } + int32_t rlepos = 0; + int32_t arraypos = 0; + rle16_t rle = src_2->runs[rlepos]; + int32_t newcard = 0; + while (arraypos < src_1->cardinality) { + const uint16_t arrayval = src_1->array[arraypos]; + while (rle.value + rle.length < + arrayval) { // this will frequently be false + ++rlepos; + if (rlepos == src_2->n_runs) { + dst->cardinality = newcard; + return; // we are done + } + rle = src_2->runs[rlepos]; + } + if (rle.value > arrayval) { + arraypos = advanceUntil(src_1->array, arraypos, src_1->cardinality, + rle.value); + } else { + dst->array[newcard] = arrayval; + newcard++; + arraypos++; + } + } + dst->cardinality = newcard; +} + +/* Compute the intersection of src_1 and src_2 and write the result to + * *dst. If the result is true then the result is a bitset_container_t + * otherwise is a array_container_t. If *dst == src_2, an in-place processing + * is attempted.*/ +bool run_bitset_container_intersection(const run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { + if (run_container_is_full(src_1)) { + if (*dst != src_2) *dst = bitset_container_clone(src_2); + return true; + } + int32_t card = run_container_cardinality(src_1); + if (card <= DEFAULT_MAX_SIZE) { + // result can only be an array (assuming that we never make a + // RunContainer) + if (card > src_2->cardinality) { + card = src_2->cardinality; + } + array_container_t *answer = array_container_create_given_capacity(card); + *dst = answer; + if (*dst == NULL) { + return false; + } + for (int32_t rlepos = 0; rlepos < src_1->n_runs; ++rlepos) { + rle16_t rle = src_1->runs[rlepos]; + uint32_t endofrun = (uint32_t)rle.value + rle.length; + for (uint32_t runValue = rle.value; runValue <= endofrun; + ++runValue) { + answer->array[answer->cardinality] = (uint16_t)runValue; + answer->cardinality += + bitset_container_contains(src_2, runValue); + } + } + return false; + } + if (*dst == src_2) { // we attempt in-place + bitset_container_t *answer = CAST_bitset(*dst); + uint32_t start = 0; + for (int32_t rlepos = 0; rlepos < src_1->n_runs; ++rlepos) { + const rle16_t rle = src_1->runs[rlepos]; + uint32_t end = rle.value; + bitset_reset_range(src_2->words, start, end); + + start = end + rle.length + 1; + } + bitset_reset_range(src_2->words, start, UINT32_C(1) << 16); + answer->cardinality = bitset_container_compute_cardinality(answer); + if (src_2->cardinality > DEFAULT_MAX_SIZE) { + return true; + } else { + array_container_t *newanswer = array_container_from_bitset(src_2); + if (newanswer == NULL) { + *dst = NULL; + return false; + } + *dst = newanswer; + return false; + } + } else { // no inplace + // we expect the answer to be a bitmap (if we are lucky) + bitset_container_t *answer = bitset_container_clone(src_2); + + *dst = answer; + if (answer == NULL) { + return true; + } + uint32_t start = 0; + for (int32_t rlepos = 0; rlepos < src_1->n_runs; ++rlepos) { + const rle16_t rle = src_1->runs[rlepos]; + uint32_t end = rle.value; + bitset_reset_range(answer->words, start, end); + start = end + rle.length + 1; + } + bitset_reset_range(answer->words, start, UINT32_C(1) << 16); + answer->cardinality = bitset_container_compute_cardinality(answer); + + if (answer->cardinality > DEFAULT_MAX_SIZE) { + return true; + } else { + array_container_t *newanswer = array_container_from_bitset(answer); + bitset_container_free(CAST_bitset(*dst)); + if (newanswer == NULL) { + *dst = NULL; + return false; + } + *dst = newanswer; + return false; + } + } +} + +/* Compute the size of the intersection between src_1 and src_2 . */ +int array_run_container_intersection_cardinality(const array_container_t *src_1, + const run_container_t *src_2) { + if (run_container_is_full(src_2)) { + return src_1->cardinality; + } + if (src_2->n_runs == 0) { + return 0; + } + int32_t rlepos = 0; + int32_t arraypos = 0; + rle16_t rle = src_2->runs[rlepos]; + int32_t newcard = 0; + while (arraypos < src_1->cardinality) { + const uint16_t arrayval = src_1->array[arraypos]; + while (rle.value + rle.length < + arrayval) { // this will frequently be false + ++rlepos; + if (rlepos == src_2->n_runs) { + return newcard; // we are done + } + rle = src_2->runs[rlepos]; + } + if (rle.value > arrayval) { + arraypos = advanceUntil(src_1->array, arraypos, src_1->cardinality, + rle.value); + } else { + newcard++; + arraypos++; + } + } + return newcard; +} + +/* Compute the intersection between src_1 and src_2 + **/ +int run_bitset_container_intersection_cardinality( + const run_container_t *src_1, const bitset_container_t *src_2) { + if (run_container_is_full(src_1)) { + return bitset_container_cardinality(src_2); + } + int answer = 0; + for (int32_t rlepos = 0; rlepos < src_1->n_runs; ++rlepos) { + rle16_t rle = src_1->runs[rlepos]; + answer += + bitset_lenrange_cardinality(src_2->words, rle.value, rle.length); + } + return answer; +} + +bool array_run_container_intersect(const array_container_t *src_1, + const run_container_t *src_2) { + if (run_container_is_full(src_2)) { + return !array_container_empty(src_1); + } + if (src_2->n_runs == 0) { + return false; + } + int32_t rlepos = 0; + int32_t arraypos = 0; + rle16_t rle = src_2->runs[rlepos]; + while (arraypos < src_1->cardinality) { + const uint16_t arrayval = src_1->array[arraypos]; + while (rle.value + rle.length < + arrayval) { // this will frequently be false + ++rlepos; + if (rlepos == src_2->n_runs) { + return false; // we are done + } + rle = src_2->runs[rlepos]; + } + if (rle.value > arrayval) { + arraypos = advanceUntil(src_1->array, arraypos, src_1->cardinality, + rle.value); + } else { + return true; + } + } + return false; +} + +/* Compute the intersection between src_1 and src_2 + **/ +bool run_bitset_container_intersect(const run_container_t *src_1, + const bitset_container_t *src_2) { + if (run_container_is_full(src_1)) { + return !bitset_container_empty(src_2); + } + for (int32_t rlepos = 0; rlepos < src_1->n_runs; ++rlepos) { + rle16_t rle = src_1->runs[rlepos]; + if (!bitset_lenrange_empty(src_2->words, rle.value, rle.length)) + return true; + } + return false; +} + +/* + * Compute the intersection between src_1 and src_2 and write the result + * to *dst. If the return function is true, the result is a bitset_container_t + * otherwise is a array_container_t. + */ +bool bitset_bitset_container_intersection(const bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { + const int newCardinality = bitset_container_and_justcard(src_1, src_2); + if (newCardinality > DEFAULT_MAX_SIZE) { + *dst = bitset_container_create(); + if (*dst != NULL) { + bitset_container_and_nocard(src_1, src_2, CAST_bitset(*dst)); + CAST_bitset(*dst)->cardinality = newCardinality; + } + return true; // it is a bitset + } + *dst = array_container_create_given_capacity(newCardinality); + if (*dst != NULL) { + CAST_array(*dst)->cardinality = newCardinality; + bitset_extract_intersection_setbits_uint16( + src_1->words, src_2->words, BITSET_CONTAINER_SIZE_IN_WORDS, + CAST_array(*dst)->array, 0); + } + return false; // not a bitset +} + +bool bitset_bitset_container_intersection_inplace( + bitset_container_t *src_1, const bitset_container_t *src_2, + container_t **dst) { + const int newCardinality = bitset_container_and_justcard(src_1, src_2); + if (newCardinality > DEFAULT_MAX_SIZE) { + *dst = src_1; + bitset_container_and_nocard(src_1, src_2, src_1); + CAST_bitset(*dst)->cardinality = newCardinality; + return true; // it is a bitset + } + *dst = array_container_create_given_capacity(newCardinality); + if (*dst != NULL) { + CAST_array(*dst)->cardinality = newCardinality; + bitset_extract_intersection_setbits_uint16( + src_1->words, src_2->words, BITSET_CONTAINER_SIZE_IN_WORDS, + CAST_array(*dst)->array, 0); + } + return false; // not a bitset +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif diff --git a/contrib/libs/croaring/src/containers/mixed_negation.c b/contrib/libs/croaring/src/containers/mixed_negation.c new file mode 100644 index 00000000000..93c6f2d3741 --- /dev/null +++ b/contrib/libs/croaring/src/containers/mixed_negation.c @@ -0,0 +1,354 @@ +/* + * mixed_negation.c + * + */ + +#include <assert.h> +#include <string.h> + +#include <roaring/array_util.h> +#include <roaring/bitset_util.h> +#include <roaring/containers/containers.h> +#include <roaring/containers/convert.h> +#include <roaring/containers/mixed_negation.h> +#include <roaring/containers/run.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +// TODO: make simplified and optimized negation code across +// the full range. + +/* Negation across the entire range of the container. + * Compute the negation of src and write the result + * to *dst. The complement of a + * sufficiently sparse set will always be dense and a hence a bitmap +' * We assume that dst is pre-allocated and a valid bitset container + * There can be no in-place version. + */ +void array_container_negation(const array_container_t *src, + bitset_container_t *dst) { + uint64_t card = UINT64_C(1 << 16); + bitset_container_set_all(dst); + + if (src->cardinality == 0) { + return; + } + + dst->cardinality = (int32_t)bitset_clear_list(dst->words, card, src->array, + (uint64_t)src->cardinality); +} + +/* Negation across the entire range of the container + * Compute the negation of src and write the result + * to *dst. A true return value indicates a bitset result, + * otherwise the result is an array container. + * We assume that dst is not pre-allocated. In + * case of failure, *dst will be NULL. + */ +bool bitset_container_negation(const bitset_container_t *src, + container_t **dst) { + return bitset_container_negation_range(src, 0, (1 << 16), dst); +} + +/* inplace version */ +/* + * Same as bitset_container_negation except that if the output is to + * be a + * bitset_container_t, then src is modified and no allocation is made. + * If the output is to be an array_container_t, then caller is responsible + * to free the container. + * In all cases, the result is in *dst. + */ +bool bitset_container_negation_inplace(bitset_container_t *src, + container_t **dst) { + return bitset_container_negation_range_inplace(src, 0, (1 << 16), dst); +} + +/* Negation across the entire range of container + * Compute the negation of src and write the result + * to *dst. Return values are the *_TYPECODES as defined * in containers.h + * We assume that dst is not pre-allocated. In + * case of failure, *dst will be NULL. + */ +int run_container_negation(const run_container_t *src, container_t **dst) { + return run_container_negation_range(src, 0, (1 << 16), dst); +} + +/* + * Same as run_container_negation except that if the output is to + * be a + * run_container_t, and has the capacity to hold the result, + * then src is modified and no allocation is made. + * In all cases, the result is in *dst. + */ +int run_container_negation_inplace(run_container_t *src, container_t **dst) { + return run_container_negation_range_inplace(src, 0, (1 << 16), dst); +} + +/* Negation across a range of the container. + * Compute the negation of src and write the result + * to *dst. Returns true if the result is a bitset container + * and false for an array container. *dst is not preallocated. + */ +bool array_container_negation_range(const array_container_t *src, + const int range_start, const int range_end, + container_t **dst) { + /* close port of the Java implementation */ + if (range_start >= range_end) { + *dst = array_container_clone(src); + return false; + } + + int32_t start_index = + binarySearch(src->array, src->cardinality, (uint16_t)range_start); + if (start_index < 0) start_index = -start_index - 1; + + int32_t last_index = + binarySearch(src->array, src->cardinality, (uint16_t)(range_end - 1)); + if (last_index < 0) last_index = -last_index - 2; + + const int32_t current_values_in_range = last_index - start_index + 1; + const int32_t span_to_be_flipped = range_end - range_start; + const int32_t new_values_in_range = + span_to_be_flipped - current_values_in_range; + const int32_t cardinality_change = + new_values_in_range - current_values_in_range; + const int32_t new_cardinality = src->cardinality + cardinality_change; + + if (new_cardinality > DEFAULT_MAX_SIZE) { + bitset_container_t *temp = bitset_container_from_array(src); + bitset_flip_range(temp->words, (uint32_t)range_start, + (uint32_t)range_end); + temp->cardinality = new_cardinality; + *dst = temp; + return true; + } + + array_container_t *arr = + array_container_create_given_capacity(new_cardinality); + *dst = (container_t *)arr; + if (new_cardinality == 0) { + arr->cardinality = new_cardinality; + return false; // we are done. + } + // copy stuff before the active area + memcpy(arr->array, src->array, start_index * sizeof(uint16_t)); + + // work on the range + int32_t out_pos = start_index, in_pos = start_index; + int32_t val_in_range = range_start; + for (; val_in_range < range_end && in_pos <= last_index; ++val_in_range) { + if ((uint16_t)val_in_range != src->array[in_pos]) { + arr->array[out_pos++] = (uint16_t)val_in_range; + } else { + ++in_pos; + } + } + for (; val_in_range < range_end; ++val_in_range) + arr->array[out_pos++] = (uint16_t)val_in_range; + + // content after the active range + memcpy(arr->array + out_pos, src->array + (last_index + 1), + (src->cardinality - (last_index + 1)) * sizeof(uint16_t)); + arr->cardinality = new_cardinality; + return false; +} + +/* Even when the result would fit, it is unclear how to make an + * inplace version without inefficient copying. + */ + +bool array_container_negation_range_inplace(array_container_t *src, + const int range_start, + const int range_end, + container_t **dst) { + bool ans = array_container_negation_range(src, range_start, range_end, dst); + // TODO : try a real inplace version + array_container_free(src); + return ans; +} + +/* Negation across a range of the container + * Compute the negation of src and write the result + * to *dst. A true return value indicates a bitset result, + * otherwise the result is an array container. + * We assume that dst is not pre-allocated. In + * case of failure, *dst will be NULL. + */ +bool bitset_container_negation_range(const bitset_container_t *src, + const int range_start, const int range_end, + container_t **dst) { + // TODO maybe consider density-based estimate + // and sometimes build result directly as array, with + // conversion back to bitset if wrong. Or determine + // actual result cardinality, then go directly for the known final cont. + + // keep computation using bitsets as long as possible. + bitset_container_t *t = bitset_container_clone(src); + bitset_flip_range(t->words, (uint32_t)range_start, (uint32_t)range_end); + t->cardinality = bitset_container_compute_cardinality(t); + + if (t->cardinality > DEFAULT_MAX_SIZE) { + *dst = t; + return true; + } else { + *dst = array_container_from_bitset(t); + bitset_container_free(t); + return false; + } +} + +/* inplace version */ +/* + * Same as bitset_container_negation except that if the output is to + * be a + * bitset_container_t, then src is modified and no allocation is made. + * If the output is to be an array_container_t, then caller is responsible + * to free the container. + * In all cases, the result is in *dst. + */ +bool bitset_container_negation_range_inplace(bitset_container_t *src, + const int range_start, + const int range_end, + container_t **dst) { + bitset_flip_range(src->words, (uint32_t)range_start, (uint32_t)range_end); + src->cardinality = bitset_container_compute_cardinality(src); + if (src->cardinality > DEFAULT_MAX_SIZE) { + *dst = src; + return true; + } + *dst = array_container_from_bitset(src); + bitset_container_free(src); + return false; +} + +/* Negation across a range of container + * Compute the negation of src and write the result + * to *dst. Return values are the *_TYPECODES as defined * in containers.h + * We assume that dst is not pre-allocated. In + * case of failure, *dst will be NULL. + */ +int run_container_negation_range(const run_container_t *src, + const int range_start, const int range_end, + container_t **dst) { + uint8_t return_typecode; + + // follows the Java implementation + if (range_end <= range_start) { + *dst = run_container_clone(src); + return RUN_CONTAINER_TYPE; + } + + run_container_t *ans = run_container_create_given_capacity( + src->n_runs + 1); // src->n_runs + 1); + int k = 0; + for (; k < src->n_runs && src->runs[k].value < range_start; ++k) { + ans->runs[k] = src->runs[k]; + ans->n_runs++; + } + + run_container_smart_append_exclusive( + ans, (uint16_t)range_start, (uint16_t)(range_end - range_start - 1)); + + for (; k < src->n_runs; ++k) { + run_container_smart_append_exclusive(ans, src->runs[k].value, + src->runs[k].length); + } + + *dst = convert_run_to_efficient_container(ans, &return_typecode); + if (return_typecode != RUN_CONTAINER_TYPE) run_container_free(ans); + + return return_typecode; +} + +/* + * Same as run_container_negation except that if the output is to + * be a + * run_container_t, and has the capacity to hold the result, + * then src is modified and no allocation is made. + * In all cases, the result is in *dst. + */ +int run_container_negation_range_inplace(run_container_t *src, + const int range_start, + const int range_end, + container_t **dst) { + uint8_t return_typecode; + + if (range_end <= range_start) { + *dst = src; + return RUN_CONTAINER_TYPE; + } + + // TODO: efficient special case when range is 0 to 65535 inclusive + + if (src->capacity == src->n_runs) { + // no excess room. More checking to see if result can fit + bool last_val_before_range = false; + bool first_val_in_range = false; + bool last_val_in_range = false; + bool first_val_past_range = false; + + if (range_start > 0) + last_val_before_range = + run_container_contains(src, (uint16_t)(range_start - 1)); + first_val_in_range = run_container_contains(src, (uint16_t)range_start); + + if (last_val_before_range == first_val_in_range) { + last_val_in_range = + run_container_contains(src, (uint16_t)(range_end - 1)); + if (range_end != 0x10000) + first_val_past_range = + run_container_contains(src, (uint16_t)range_end); + + if (last_val_in_range == + first_val_past_range) { // no space for inplace + int ans = run_container_negation_range(src, range_start, + range_end, dst); + run_container_free(src); + return ans; + } + } + } + // all other cases: result will fit + + run_container_t *ans = src; + int my_nbr_runs = src->n_runs; + + ans->n_runs = 0; + int k = 0; + for (; (k < my_nbr_runs) && (src->runs[k].value < range_start); ++k) { + // ans->runs[k] = src->runs[k]; (would be self-copy) + ans->n_runs++; + } + + // as with Java implementation, use locals to give self a buffer of depth 1 + rle16_t buffered = MAKE_RLE16(0, 0); + rle16_t next = buffered; + if (k < my_nbr_runs) buffered = src->runs[k]; + + run_container_smart_append_exclusive( + ans, (uint16_t)range_start, (uint16_t)(range_end - range_start - 1)); + + for (; k < my_nbr_runs; ++k) { + if (k + 1 < my_nbr_runs) next = src->runs[k + 1]; + + run_container_smart_append_exclusive(ans, buffered.value, + buffered.length); + buffered = next; + } + + *dst = convert_run_to_efficient_container(ans, &return_typecode); + if (return_typecode != RUN_CONTAINER_TYPE) run_container_free(ans); + + return return_typecode; +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif diff --git a/contrib/libs/croaring/src/containers/mixed_subset.c b/contrib/libs/croaring/src/containers/mixed_subset.c new file mode 100644 index 00000000000..1fcf2f063de --- /dev/null +++ b/contrib/libs/croaring/src/containers/mixed_subset.c @@ -0,0 +1,145 @@ +#include <roaring/array_util.h> +#include <roaring/containers/mixed_subset.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +bool array_container_is_subset_bitset(const array_container_t* container1, + const bitset_container_t* container2) { + if (container2->cardinality != BITSET_UNKNOWN_CARDINALITY) { + if (container2->cardinality < container1->cardinality) { + return false; + } + } + for (int i = 0; i < container1->cardinality; ++i) { + if (!bitset_container_contains(container2, container1->array[i])) { + return false; + } + } + return true; +} + +bool run_container_is_subset_array(const run_container_t* container1, + const array_container_t* container2) { + if (run_container_cardinality(container1) > container2->cardinality) + return false; + int32_t start_pos = -1, stop_pos = -1; + for (int i = 0; i < container1->n_runs; ++i) { + int32_t start = container1->runs[i].value; + int32_t stop = start + container1->runs[i].length; + start_pos = advanceUntil(container2->array, stop_pos, + container2->cardinality, start); + stop_pos = advanceUntil(container2->array, stop_pos, + container2->cardinality, stop); + if (stop_pos == container2->cardinality) { + return false; + } else if (stop_pos - start_pos != stop - start || + container2->array[start_pos] != start || + container2->array[stop_pos] != stop) { + return false; + } + } + return true; +} + +bool array_container_is_subset_run(const array_container_t* container1, + const run_container_t* container2) { + if (container1->cardinality > run_container_cardinality(container2)) + return false; + int i_array = 0, i_run = 0; + while (i_array < container1->cardinality && i_run < container2->n_runs) { + uint32_t start = container2->runs[i_run].value; + uint32_t stop = start + container2->runs[i_run].length; + if (container1->array[i_array] < start) { + return false; + } else if (container1->array[i_array] > stop) { + i_run++; + } else { // the value of the array is in the run + i_array++; + } + } + if (i_array == container1->cardinality) { + return true; + } else { + return false; + } +} + +bool run_container_is_subset_bitset(const run_container_t* container1, + const bitset_container_t* container2) { + // todo: this code could be much faster + if (container2->cardinality != BITSET_UNKNOWN_CARDINALITY) { + if (container2->cardinality < run_container_cardinality(container1)) { + return false; + } + } else { + int32_t card = bitset_container_compute_cardinality( + container2); // modify container2? + if (card < run_container_cardinality(container1)) { + return false; + } + } + for (int i = 0; i < container1->n_runs; ++i) { + uint32_t run_start = container1->runs[i].value; + uint32_t le = container1->runs[i].length; + for (uint32_t j = run_start; j <= run_start + le; ++j) { + if (!bitset_container_contains(container2, j)) { + return false; + } + } + } + return true; +} + +bool bitset_container_is_subset_run(const bitset_container_t* container1, + const run_container_t* container2) { + // todo: this code could be much faster + if (container1->cardinality != BITSET_UNKNOWN_CARDINALITY) { + if (container1->cardinality > run_container_cardinality(container2)) { + return false; + } + } + int32_t i_bitset = 0, i_run = 0; + while (i_bitset < BITSET_CONTAINER_SIZE_IN_WORDS && + i_run < container2->n_runs) { + uint64_t w = container1->words[i_bitset]; + while (w != 0 && i_run < container2->n_runs) { + uint32_t start = container2->runs[i_run].value; + uint32_t stop = start + container2->runs[i_run].length; + uint64_t t = w & (~w + 1); + uint16_t r = i_bitset * 64 + roaring_trailing_zeroes(w); + if (r < start) { + return false; + } else if (r > stop) { + i_run++; + continue; + } else { + w ^= t; + } + } + if (w == 0) { + i_bitset++; + } else { + return false; + } + } + if (i_bitset < BITSET_CONTAINER_SIZE_IN_WORDS) { + // terminated iterating on the run containers, check that rest of bitset + // is empty + for (; i_bitset < BITSET_CONTAINER_SIZE_IN_WORDS; i_bitset++) { + if (container1->words[i_bitset] != 0) { + return false; + } + } + } + return true; +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif diff --git a/contrib/libs/croaring/src/containers/mixed_union.c b/contrib/libs/croaring/src/containers/mixed_union.c new file mode 100644 index 00000000000..9fe315395ae --- /dev/null +++ b/contrib/libs/croaring/src/containers/mixed_union.c @@ -0,0 +1,372 @@ +/* + * mixed_union.c + * + */ + +#include <assert.h> +#include <string.h> + +#include <roaring/bitset_util.h> +#include <roaring/containers/convert.h> +#include <roaring/containers/mixed_union.h> +#include <roaring/containers/perfparameters.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/* Compute the union of src_1 and src_2 and write the result to + * dst. */ +void array_bitset_container_union(const array_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst) { + if (src_2 != dst) bitset_container_copy(src_2, dst); + dst->cardinality = (int32_t)bitset_set_list_withcard( + dst->words, dst->cardinality, src_1->array, src_1->cardinality); +} + +/* Compute the union of src_1 and src_2 and write the result to + * dst. It is allowed for src_2 to be dst. This version does not + * update the cardinality of dst (it is set to BITSET_UNKNOWN_CARDINALITY). */ +void array_bitset_container_lazy_union(const array_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst) { + if (src_2 != dst) bitset_container_copy(src_2, dst); + bitset_set_list(dst->words, src_1->array, src_1->cardinality); + dst->cardinality = BITSET_UNKNOWN_CARDINALITY; +} + +void run_bitset_container_union(const run_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst) { + assert(!run_container_is_full(src_1)); // catch this case upstream + if (src_2 != dst) bitset_container_copy(src_2, dst); + for (int32_t rlepos = 0; rlepos < src_1->n_runs; ++rlepos) { + rle16_t rle = src_1->runs[rlepos]; + bitset_set_lenrange(dst->words, rle.value, rle.length); + } + dst->cardinality = bitset_container_compute_cardinality(dst); +} + +void run_bitset_container_lazy_union(const run_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst) { + assert(!run_container_is_full(src_1)); // catch this case upstream + if (src_2 != dst) bitset_container_copy(src_2, dst); + for (int32_t rlepos = 0; rlepos < src_1->n_runs; ++rlepos) { + rle16_t rle = src_1->runs[rlepos]; + bitset_set_lenrange(dst->words, rle.value, rle.length); + } + dst->cardinality = BITSET_UNKNOWN_CARDINALITY; +} + +// why do we leave the result as a run container?? +void array_run_container_union(const array_container_t *src_1, + const run_container_t *src_2, + run_container_t *dst) { + if (run_container_is_full(src_2)) { + run_container_copy(src_2, dst); + return; + } + // TODO: see whether the "2*" is spurious + run_container_grow(dst, 2 * (src_1->cardinality + src_2->n_runs), false); + int32_t rlepos = 0; + int32_t arraypos = 0; + rle16_t previousrle; + if (src_2->runs[rlepos].value <= src_1->array[arraypos]) { + previousrle = run_container_append_first(dst, src_2->runs[rlepos]); + rlepos++; + } else { + previousrle = + run_container_append_value_first(dst, src_1->array[arraypos]); + arraypos++; + } + while ((rlepos < src_2->n_runs) && (arraypos < src_1->cardinality)) { + if (src_2->runs[rlepos].value <= src_1->array[arraypos]) { + run_container_append(dst, src_2->runs[rlepos], &previousrle); + rlepos++; + } else { + run_container_append_value(dst, src_1->array[arraypos], + &previousrle); + arraypos++; + } + } + if (arraypos < src_1->cardinality) { + while (arraypos < src_1->cardinality) { + run_container_append_value(dst, src_1->array[arraypos], + &previousrle); + arraypos++; + } + } else { + while (rlepos < src_2->n_runs) { + run_container_append(dst, src_2->runs[rlepos], &previousrle); + rlepos++; + } + } +} + +void array_run_container_inplace_union(const array_container_t *src_1, + run_container_t *src_2) { + if (run_container_is_full(src_2)) { + return; + } + const int32_t maxoutput = src_1->cardinality + src_2->n_runs; + const int32_t neededcapacity = maxoutput + src_2->n_runs; + if (src_2->capacity < neededcapacity) + run_container_grow(src_2, neededcapacity, true); + memmove(src_2->runs + maxoutput, src_2->runs, + src_2->n_runs * sizeof(rle16_t)); + rle16_t *inputsrc2 = src_2->runs + maxoutput; + int32_t rlepos = 0; + int32_t arraypos = 0; + int src2nruns = src_2->n_runs; + src_2->n_runs = 0; + + rle16_t previousrle; + + if (inputsrc2[rlepos].value <= src_1->array[arraypos]) { + previousrle = run_container_append_first(src_2, inputsrc2[rlepos]); + rlepos++; + } else { + previousrle = + run_container_append_value_first(src_2, src_1->array[arraypos]); + arraypos++; + } + + while ((rlepos < src2nruns) && (arraypos < src_1->cardinality)) { + if (inputsrc2[rlepos].value <= src_1->array[arraypos]) { + run_container_append(src_2, inputsrc2[rlepos], &previousrle); + rlepos++; + } else { + run_container_append_value(src_2, src_1->array[arraypos], + &previousrle); + arraypos++; + } + } + if (arraypos < src_1->cardinality) { + while (arraypos < src_1->cardinality) { + run_container_append_value(src_2, src_1->array[arraypos], + &previousrle); + arraypos++; + } + } else { + while (rlepos < src2nruns) { + run_container_append(src_2, inputsrc2[rlepos], &previousrle); + rlepos++; + } + } +} + +bool array_array_container_union(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { + int totalCardinality = src_1->cardinality + src_2->cardinality; + if (totalCardinality <= DEFAULT_MAX_SIZE) { + *dst = array_container_create_given_capacity(totalCardinality); + if (*dst != NULL) { + array_container_union(src_1, src_2, CAST_array(*dst)); + } else { + return true; // otherwise failure won't be caught + } + return false; // not a bitset + } + *dst = bitset_container_create(); + bool returnval = true; // expect a bitset + if (*dst != NULL) { + bitset_container_t *ourbitset = CAST_bitset(*dst); + bitset_set_list(ourbitset->words, src_1->array, src_1->cardinality); + ourbitset->cardinality = (int32_t)bitset_set_list_withcard( + ourbitset->words, src_1->cardinality, src_2->array, + src_2->cardinality); + if (ourbitset->cardinality <= DEFAULT_MAX_SIZE) { + // need to convert! + *dst = array_container_from_bitset(ourbitset); + bitset_container_free(ourbitset); + returnval = false; // not going to be a bitset + } + } + return returnval; +} + +bool array_array_container_inplace_union(array_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { + int totalCardinality = src_1->cardinality + src_2->cardinality; + *dst = NULL; + if (totalCardinality <= DEFAULT_MAX_SIZE) { + if (src_1->capacity < totalCardinality) { + *dst = array_container_create_given_capacity( + 2 * totalCardinality); // be purposefully generous + if (*dst != NULL) { + array_container_union(src_1, src_2, CAST_array(*dst)); + } else { + return true; // otherwise failure won't be caught + } + return false; // not a bitset + } else { + memmove(src_1->array + src_2->cardinality, src_1->array, + src_1->cardinality * sizeof(uint16_t)); + // In theory, we could use fast_union_uint16, but it is unsafe. It + // fails with Intel compilers in particular. + // https://github.com/RoaringBitmap/CRoaring/pull/452 + // See report https://github.com/RoaringBitmap/CRoaring/issues/476 + src_1->cardinality = (int32_t)union_uint16( + src_1->array + src_2->cardinality, src_1->cardinality, + src_2->array, src_2->cardinality, src_1->array); + return false; // not a bitset + } + } + *dst = bitset_container_create(); + bool returnval = true; // expect a bitset + if (*dst != NULL) { + bitset_container_t *ourbitset = CAST_bitset(*dst); + bitset_set_list(ourbitset->words, src_1->array, src_1->cardinality); + ourbitset->cardinality = (int32_t)bitset_set_list_withcard( + ourbitset->words, src_1->cardinality, src_2->array, + src_2->cardinality); + if (ourbitset->cardinality <= DEFAULT_MAX_SIZE) { + // need to convert! + if (src_1->capacity < ourbitset->cardinality) { + array_container_grow(src_1, ourbitset->cardinality, false); + } + + bitset_extract_setbits_uint16(ourbitset->words, + BITSET_CONTAINER_SIZE_IN_WORDS, + src_1->array, 0); + src_1->cardinality = ourbitset->cardinality; + *dst = src_1; + bitset_container_free(ourbitset); + returnval = false; // not going to be a bitset + } + } + return returnval; +} + +bool array_array_container_lazy_union(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { + int totalCardinality = src_1->cardinality + src_2->cardinality; + // + // We assume that operations involving bitset containers will be faster than + // operations involving solely array containers, except maybe when array + // containers are small. Indeed, for example, it is cheap to compute the + // union between an array and a bitset container, generally more so than + // between a large array and another array. So it is advantageous to favour + // bitset containers during the computation. Of course, if we convert array + // containers eagerly to bitset containers, we may later need to revert the + // bitset containers to array containerr to satisfy the Roaring format + // requirements, but such one-time conversions at the end may not be overly + // expensive. We arrived to this design based on extensive benchmarking. + // + if (totalCardinality <= ARRAY_LAZY_LOWERBOUND) { + *dst = array_container_create_given_capacity(totalCardinality); + if (*dst != NULL) { + array_container_union(src_1, src_2, CAST_array(*dst)); + } else { + return true; // otherwise failure won't be caught + } + return false; // not a bitset + } + *dst = bitset_container_create(); + bool returnval = true; // expect a bitset + if (*dst != NULL) { + bitset_container_t *ourbitset = CAST_bitset(*dst); + bitset_set_list(ourbitset->words, src_1->array, src_1->cardinality); + bitset_set_list(ourbitset->words, src_2->array, src_2->cardinality); + ourbitset->cardinality = BITSET_UNKNOWN_CARDINALITY; + } + return returnval; +} + +bool array_array_container_lazy_inplace_union(array_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { + int totalCardinality = src_1->cardinality + src_2->cardinality; + *dst = NULL; + // + // We assume that operations involving bitset containers will be faster than + // operations involving solely array containers, except maybe when array + // containers are small. Indeed, for example, it is cheap to compute the + // union between an array and a bitset container, generally more so than + // between a large array and another array. So it is advantageous to favour + // bitset containers during the computation. Of course, if we convert array + // containers eagerly to bitset containers, we may later need to revert the + // bitset containers to array containerr to satisfy the Roaring format + // requirements, but such one-time conversions at the end may not be overly + // expensive. We arrived to this design based on extensive benchmarking. + // + if (totalCardinality <= ARRAY_LAZY_LOWERBOUND) { + if (src_1->capacity < totalCardinality) { + *dst = array_container_create_given_capacity( + 2 * totalCardinality); // be purposefully generous + if (*dst != NULL) { + array_container_union(src_1, src_2, CAST_array(*dst)); + } else { + return true; // otherwise failure won't be caught + } + return false; // not a bitset + } else { + memmove(src_1->array + src_2->cardinality, src_1->array, + src_1->cardinality * sizeof(uint16_t)); + /* + Next line is safe: + + We just need to focus on the reading and writing performed on + array1. In `union_vector16`, both vectorized and scalar code still + obey the basic rule: read from two inputs, do the union, and then + write the output. + + Let's say the length(cardinality) of input2 is L2: + ``` + |<- L2 ->| + array1: [output--- |input 1---|---] + array2: [input 2---] + ``` + Let's define 3 __m128i pointers, `pos1` starts from `input1`, + `pos2` starts from `input2`, these 2 point at the next byte to + read, `out` starts from `output`, pointing at the next byte to + overwrite. + ``` + array1: [output--- |input 1---|---] + ^ ^ + out pos1 + array2: [input 2---] + ^ + pos2 + ``` + The union output always contains less or equal number of elements + than all inputs added, so we have: + ``` + out <= pos1 + pos2 + ``` + therefore: + ``` + out <= pos1 + L2 + ``` + which means you will not overwrite data beyond pos1, so the data + haven't read is safe, and we don't care the data already read. + */ + src_1->cardinality = (int32_t)fast_union_uint16( + src_1->array + src_2->cardinality, src_1->cardinality, + src_2->array, src_2->cardinality, src_1->array); + return false; // not a bitset + } + } + *dst = bitset_container_create(); + bool returnval = true; // expect a bitset + if (*dst != NULL) { + bitset_container_t *ourbitset = CAST_bitset(*dst); + bitset_set_list(ourbitset->words, src_1->array, src_1->cardinality); + bitset_set_list(ourbitset->words, src_2->array, src_2->cardinality); + ourbitset->cardinality = BITSET_UNKNOWN_CARDINALITY; + } + return returnval; +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif diff --git a/contrib/libs/croaring/src/containers/mixed_xor.c b/contrib/libs/croaring/src/containers/mixed_xor.c new file mode 100644 index 00000000000..ad23aa33664 --- /dev/null +++ b/contrib/libs/croaring/src/containers/mixed_xor.c @@ -0,0 +1,385 @@ +/* + * mixed_xor.c + */ + +#include <assert.h> +#include <string.h> + +#include <roaring/bitset_util.h> +#include <roaring/containers/containers.h> +#include <roaring/containers/convert.h> +#include <roaring/containers/mixed_xor.h> +#include <roaring/containers/perfparameters.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +/* Compute the xor of src_1 and src_2 and write the result to + * dst (which has no container initially). + * Result is true iff dst is a bitset */ +bool array_bitset_container_xor(const array_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { + bitset_container_t *result = bitset_container_create(); + bitset_container_copy(src_2, result); + result->cardinality = (int32_t)bitset_flip_list_withcard( + result->words, result->cardinality, src_1->array, src_1->cardinality); + + // do required type conversions. + if (result->cardinality <= DEFAULT_MAX_SIZE) { + *dst = array_container_from_bitset(result); + bitset_container_free(result); + return false; // not bitset + } + *dst = result; + return true; // bitset +} + +/* Compute the xor of src_1 and src_2 and write the result to + * dst. It is allowed for src_2 to be dst. This version does not + * update the cardinality of dst (it is set to BITSET_UNKNOWN_CARDINALITY). + */ + +void array_bitset_container_lazy_xor(const array_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst) { + if (src_2 != dst) bitset_container_copy(src_2, dst); + bitset_flip_list(dst->words, src_1->array, src_1->cardinality); + dst->cardinality = BITSET_UNKNOWN_CARDINALITY; +} + +/* Compute the xor of src_1 and src_2 and write the result to + * dst. Result may be either a bitset or an array container + * (returns "result is bitset"). dst does not initially have + * any container, but becomes either a bitset container (return + * result true) or an array container. + */ + +bool run_bitset_container_xor(const run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { + bitset_container_t *result = bitset_container_create(); + + bitset_container_copy(src_2, result); + for (int32_t rlepos = 0; rlepos < src_1->n_runs; ++rlepos) { + rle16_t rle = src_1->runs[rlepos]; + bitset_flip_range(result->words, rle.value, + rle.value + rle.length + UINT32_C(1)); + } + result->cardinality = bitset_container_compute_cardinality(result); + + if (result->cardinality <= DEFAULT_MAX_SIZE) { + *dst = array_container_from_bitset(result); + bitset_container_free(result); + return false; // not bitset + } + *dst = result; + return true; // bitset +} + +/* lazy xor. Dst is initialized and may be equal to src_2. + * Result is left as a bitset container, even if actual + * cardinality would dictate an array container. + */ + +void run_bitset_container_lazy_xor(const run_container_t *src_1, + const bitset_container_t *src_2, + bitset_container_t *dst) { + if (src_2 != dst) bitset_container_copy(src_2, dst); + for (int32_t rlepos = 0; rlepos < src_1->n_runs; ++rlepos) { + rle16_t rle = src_1->runs[rlepos]; + bitset_flip_range(dst->words, rle.value, + rle.value + rle.length + UINT32_C(1)); + } + dst->cardinality = BITSET_UNKNOWN_CARDINALITY; +} + +/* dst does not indicate a valid container initially. Eventually it + * can become any kind of container. + */ + +int array_run_container_xor(const array_container_t *src_1, + const run_container_t *src_2, container_t **dst) { + // semi following Java XOR implementation as of May 2016 + // the C OR implementation works quite differently and can return a run + // container + // TODO could optimize for full run containers. + + // use of lazy following Java impl. + const int arbitrary_threshold = 32; + if (src_1->cardinality < arbitrary_threshold) { + run_container_t *ans = run_container_create(); + array_run_container_lazy_xor(src_1, src_2, ans); // keeps runs. + uint8_t typecode_after; + *dst = + convert_run_to_efficient_container_and_free(ans, &typecode_after); + return typecode_after; + } + + int card = run_container_cardinality(src_2); + if (card <= DEFAULT_MAX_SIZE) { + // Java implementation works with the array, xoring the run elements via + // iterator + array_container_t *temp = array_container_from_run(src_2); + bool ret_is_bitset = array_array_container_xor(temp, src_1, dst); + array_container_free(temp); + return ret_is_bitset ? BITSET_CONTAINER_TYPE : ARRAY_CONTAINER_TYPE; + + } else { // guess that it will end up as a bitset + bitset_container_t *result = bitset_container_from_run(src_2); + bool is_bitset = bitset_array_container_ixor(result, src_1, dst); + // any necessary type conversion has been done by the ixor + int retval = (is_bitset ? BITSET_CONTAINER_TYPE : ARRAY_CONTAINER_TYPE); + return retval; + } +} + +/* Dst is a valid run container. (Can it be src_2? Let's say not.) + * Leaves result as run container, even if other options are + * smaller. + */ + +void array_run_container_lazy_xor(const array_container_t *src_1, + const run_container_t *src_2, + run_container_t *dst) { + run_container_grow(dst, src_1->cardinality + src_2->n_runs, false); + int32_t rlepos = 0; + int32_t arraypos = 0; + dst->n_runs = 0; + + while ((rlepos < src_2->n_runs) && (arraypos < src_1->cardinality)) { + if (src_2->runs[rlepos].value <= src_1->array[arraypos]) { + run_container_smart_append_exclusive(dst, src_2->runs[rlepos].value, + src_2->runs[rlepos].length); + rlepos++; + } else { + run_container_smart_append_exclusive(dst, src_1->array[arraypos], + 0); + arraypos++; + } + } + while (arraypos < src_1->cardinality) { + run_container_smart_append_exclusive(dst, src_1->array[arraypos], 0); + arraypos++; + } + while (rlepos < src_2->n_runs) { + run_container_smart_append_exclusive(dst, src_2->runs[rlepos].value, + src_2->runs[rlepos].length); + rlepos++; + } +} + +/* dst does not indicate a valid container initially. Eventually it + * can become any kind of container. + */ + +int run_run_container_xor(const run_container_t *src_1, + const run_container_t *src_2, container_t **dst) { + run_container_t *ans = run_container_create(); + run_container_xor(src_1, src_2, ans); + uint8_t typecode_after; + *dst = convert_run_to_efficient_container_and_free(ans, &typecode_after); + return typecode_after; +} + +/* + * Java implementation (as of May 2016) for array_run, run_run + * and bitset_run don't do anything different for inplace. + * Could adopt the mixed_union.c approach instead (ie, using + * smart_append_exclusive) + * + */ + +bool array_array_container_xor(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { + int totalCardinality = + src_1->cardinality + src_2->cardinality; // upper bound + if (totalCardinality <= DEFAULT_MAX_SIZE) { + *dst = array_container_create_given_capacity(totalCardinality); + array_container_xor(src_1, src_2, CAST_array(*dst)); + return false; // not a bitset + } + *dst = bitset_container_from_array(src_1); + bool returnval = true; // expect a bitset + bitset_container_t *ourbitset = CAST_bitset(*dst); + ourbitset->cardinality = (uint32_t)bitset_flip_list_withcard( + ourbitset->words, src_1->cardinality, src_2->array, src_2->cardinality); + if (ourbitset->cardinality <= DEFAULT_MAX_SIZE) { + // need to convert! + *dst = array_container_from_bitset(ourbitset); + bitset_container_free(ourbitset); + returnval = false; // not going to be a bitset + } + + return returnval; +} + +bool array_array_container_lazy_xor(const array_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { + int totalCardinality = src_1->cardinality + src_2->cardinality; + // + // We assume that operations involving bitset containers will be faster than + // operations involving solely array containers, except maybe when array + // containers are small. Indeed, for example, it is cheap to compute the + // exclusive union between an array and a bitset container, generally more + // so than between a large array and another array. So it is advantageous to + // favour bitset containers during the computation. Of course, if we convert + // array containers eagerly to bitset containers, we may later need to + // revert the bitset containers to array containerr to satisfy the Roaring + // format requirements, but such one-time conversions at the end may not be + // overly expensive. We arrived to this design based on extensive + // benchmarking on unions. For XOR/exclusive union, we simply followed the + // heuristic used by the unions (see mixed_union.c). Further tuning is + // possible. + // + if (totalCardinality <= ARRAY_LAZY_LOWERBOUND) { + *dst = array_container_create_given_capacity(totalCardinality); + if (*dst != NULL) array_container_xor(src_1, src_2, CAST_array(*dst)); + return false; // not a bitset + } + *dst = bitset_container_from_array(src_1); + bool returnval = true; // expect a bitset (maybe, for XOR??) + if (*dst != NULL) { + bitset_container_t *ourbitset = CAST_bitset(*dst); + bitset_flip_list(ourbitset->words, src_2->array, src_2->cardinality); + ourbitset->cardinality = BITSET_UNKNOWN_CARDINALITY; + } + return returnval; +} + +/* Compute the xor of src_1 and src_2 and write the result to + * dst (which has no container initially). Return value is + * "dst is a bitset" + */ + +bool bitset_bitset_container_xor(const bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { + bitset_container_t *ans = bitset_container_create(); + int card = bitset_container_xor(src_1, src_2, ans); + if (card <= DEFAULT_MAX_SIZE) { + *dst = array_container_from_bitset(ans); + bitset_container_free(ans); + return false; // not bitset + } else { + *dst = ans; + return true; + } +} + +/* Compute the xor of src_1 and src_2 and write the result to + * dst (which has no container initially). It will modify src_1 + * to be dst if the result is a bitset. Otherwise, it will + * free src_1 and dst will be a new array container. In both + * cases, the caller is responsible for deallocating dst. + * Returns true iff dst is a bitset */ + +bool bitset_array_container_ixor(bitset_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { + *dst = src_1; + src_1->cardinality = (uint32_t)bitset_flip_list_withcard( + src_1->words, src_1->cardinality, src_2->array, src_2->cardinality); + + if (src_1->cardinality <= DEFAULT_MAX_SIZE) { + *dst = array_container_from_bitset(src_1); + bitset_container_free(src_1); + return false; // not bitset + } else + return true; +} + +/* a bunch of in-place, some of which may not *really* be inplace. + * TODO: write actual inplace routine if efficiency warrants it + * Anything inplace with a bitset is a good candidate + */ + +bool bitset_bitset_container_ixor(bitset_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { + int card = bitset_container_xor(src_1, src_2, src_1); + if (card <= DEFAULT_MAX_SIZE) { + *dst = array_container_from_bitset(src_1); + bitset_container_free(src_1); + return false; // not bitset + } else { + *dst = src_1; + return true; + } +} + +bool array_bitset_container_ixor(array_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { + bool ans = array_bitset_container_xor(src_1, src_2, dst); + array_container_free(src_1); + return ans; +} + +/* Compute the xor of src_1 and src_2 and write the result to + * dst. Result may be either a bitset or an array container + * (returns "result is bitset"). dst does not initially have + * any container, but becomes either a bitset container (return + * result true) or an array container. + */ + +bool run_bitset_container_ixor(run_container_t *src_1, + const bitset_container_t *src_2, + container_t **dst) { + bool ans = run_bitset_container_xor(src_1, src_2, dst); + run_container_free(src_1); + return ans; +} + +bool bitset_run_container_ixor(bitset_container_t *src_1, + const run_container_t *src_2, + container_t **dst) { + bool ans = run_bitset_container_xor(src_2, src_1, dst); + bitset_container_free(src_1); + return ans; +} + +/* dst does not indicate a valid container initially. Eventually it + * can become any kind of container. + */ + +int array_run_container_ixor(array_container_t *src_1, + const run_container_t *src_2, container_t **dst) { + int ans = array_run_container_xor(src_1, src_2, dst); + array_container_free(src_1); + return ans; +} + +int run_array_container_ixor(run_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { + int ans = array_run_container_xor(src_2, src_1, dst); + run_container_free(src_1); + return ans; +} + +bool array_array_container_ixor(array_container_t *src_1, + const array_container_t *src_2, + container_t **dst) { + bool ans = array_array_container_xor(src_1, src_2, dst); + array_container_free(src_1); + return ans; +} + +int run_run_container_ixor(run_container_t *src_1, const run_container_t *src_2, + container_t **dst) { + int ans = run_run_container_xor(src_1, src_2, dst); + run_container_free(src_1); + return ans; +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif diff --git a/contrib/libs/croaring/src/containers/run.c b/contrib/libs/croaring/src/containers/run.c new file mode 100644 index 00000000000..2a2787b2453 --- /dev/null +++ b/contrib/libs/croaring/src/containers/run.c @@ -0,0 +1,1079 @@ +#include <stdio.h> +#include <stdlib.h> + +#include <roaring/containers/run.h> +#include <roaring/memory.h> +#include <roaring/portability.h> + +#if CROARING_IS_X64 +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuninitialized" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +extern inline uint16_t run_container_minimum(const run_container_t *run); +extern inline uint16_t run_container_maximum(const run_container_t *run); +extern inline int32_t interleavedBinarySearch(const rle16_t *array, + int32_t lenarray, uint16_t ikey); +extern inline bool run_container_contains(const run_container_t *run, + uint16_t pos); +extern inline int run_container_index_equalorlarger(const run_container_t *arr, + uint16_t x); +extern inline bool run_container_is_full(const run_container_t *run); +extern inline bool run_container_nonzero_cardinality(const run_container_t *rc); +extern inline int32_t run_container_serialized_size_in_bytes(int32_t num_runs); +extern inline run_container_t *run_container_create_range(uint32_t start, + uint32_t stop); +extern inline int run_container_cardinality(const run_container_t *run); + +bool run_container_add(run_container_t *run, uint16_t pos) { + int32_t index = interleavedBinarySearch(run->runs, run->n_runs, pos); + if (index >= 0) return false; // already there + index = -index - 2; // points to preceding value, possibly -1 + if (index >= 0) { // possible match + int32_t offset = pos - run->runs[index].value; + int32_t le = run->runs[index].length; + if (offset <= le) return false; // already there + if (offset == le + 1) { + // we may need to fuse + if (index + 1 < run->n_runs) { + if (run->runs[index + 1].value == pos + 1) { + // indeed fusion is needed + run->runs[index].length = run->runs[index + 1].value + + run->runs[index + 1].length - + run->runs[index].value; + recoverRoomAtIndex(run, (uint16_t)(index + 1)); + return true; + } + } + run->runs[index].length++; + return true; + } + if (index + 1 < run->n_runs) { + // we may need to fuse + if (run->runs[index + 1].value == pos + 1) { + // indeed fusion is needed + run->runs[index + 1].value = pos; + run->runs[index + 1].length = run->runs[index + 1].length + 1; + return true; + } + } + } + if (index == -1) { + // we may need to extend the first run + if (0 < run->n_runs) { + if (run->runs[0].value == pos + 1) { + run->runs[0].length++; + run->runs[0].value--; + return true; + } + } + } + makeRoomAtIndex(run, (uint16_t)(index + 1)); + run->runs[index + 1].value = pos; + run->runs[index + 1].length = 0; + return true; +} + +/* Create a new run container. Return NULL in case of failure. */ +run_container_t *run_container_create_given_capacity(int32_t size) { + run_container_t *run; + /* Allocate the run container itself. */ + if ((run = (run_container_t *)roaring_malloc(sizeof(run_container_t))) == + NULL) { + return NULL; + } + if (size <= 0) { // we don't want to rely on malloc(0) + run->runs = NULL; + } else if ((run->runs = (rle16_t *)roaring_malloc(sizeof(rle16_t) * + size)) == NULL) { + roaring_free(run); + return NULL; + } + run->capacity = size; + run->n_runs = 0; + return run; +} + +int run_container_shrink_to_fit(run_container_t *src) { + if (src->n_runs == src->capacity) return 0; // nothing to do + int savings = src->capacity - src->n_runs; + src->capacity = src->n_runs; + rle16_t *oldruns = src->runs; + src->runs = + (rle16_t *)roaring_realloc(oldruns, src->capacity * sizeof(rle16_t)); + if (src->runs == NULL) roaring_free(oldruns); // should never happen? + return savings; +} +/* Create a new run container. Return NULL in case of failure. */ +run_container_t *run_container_create(void) { + return run_container_create_given_capacity(RUN_DEFAULT_INIT_SIZE); +} + +ALLOW_UNALIGNED +run_container_t *run_container_clone(const run_container_t *src) { + run_container_t *run = run_container_create_given_capacity(src->capacity); + if (run == NULL) return NULL; + run->capacity = src->capacity; + run->n_runs = src->n_runs; + memcpy(run->runs, src->runs, src->n_runs * sizeof(rle16_t)); + return run; +} + +void run_container_offset(const run_container_t *c, container_t **loc, + container_t **hic, uint16_t offset) { + run_container_t *lo = NULL, *hi = NULL; + + bool split; + int lo_cap, hi_cap; + int top, pivot; + + top = (1 << 16) - offset; + pivot = run_container_index_equalorlarger(c, top); + + if (pivot == -1) { + split = false; + lo_cap = c->n_runs; + hi_cap = 0; + } else { + split = c->runs[pivot].value < top; + lo_cap = pivot + (split ? 1 : 0); + hi_cap = c->n_runs - pivot; + } + + if (loc && lo_cap) { + lo = run_container_create_given_capacity(lo_cap); + memcpy(lo->runs, c->runs, lo_cap * sizeof(rle16_t)); + lo->n_runs = lo_cap; + for (int i = 0; i < lo_cap; ++i) { + lo->runs[i].value += offset; + } + *loc = (container_t *)lo; + } + + if (hic && hi_cap) { + hi = run_container_create_given_capacity(hi_cap); + memcpy(hi->runs, c->runs + pivot, hi_cap * sizeof(rle16_t)); + hi->n_runs = hi_cap; + for (int i = 0; i < hi_cap; ++i) { + hi->runs[i].value += offset; + } + *hic = (container_t *)hi; + } + + // Fix the split. + if (split) { + if (lo != NULL) { + // Add the missing run to 'lo', exhausting length. + lo->runs[lo->n_runs - 1].length = + (1 << 16) - lo->runs[lo->n_runs - 1].value - 1; + } + + if (hi != NULL) { + // Fix the first run in 'hi'. + hi->runs[0].length -= UINT16_MAX - hi->runs[0].value + 1; + hi->runs[0].value = 0; + } + } +} + +/* Free memory. */ +void run_container_free(run_container_t *run) { + if (run->runs != + NULL) { // Jon Strabala reports that some tools complain otherwise + roaring_free(run->runs); + run->runs = NULL; // pedantic + } + roaring_free(run); +} + +void run_container_grow(run_container_t *run, int32_t min, bool copy) { + int32_t newCapacity = (run->capacity == 0) ? RUN_DEFAULT_INIT_SIZE + : run->capacity < 64 ? run->capacity * 2 + : run->capacity < 1024 ? run->capacity * 3 / 2 + : run->capacity * 5 / 4; + if (newCapacity < min) newCapacity = min; + run->capacity = newCapacity; + assert(run->capacity >= min); + if (copy) { + rle16_t *oldruns = run->runs; + run->runs = (rle16_t *)roaring_realloc(oldruns, + run->capacity * sizeof(rle16_t)); + if (run->runs == NULL) roaring_free(oldruns); + } else { + // Jon Strabala reports that some tools complain otherwise + if (run->runs != NULL) { + roaring_free(run->runs); + } + run->runs = (rle16_t *)roaring_malloc(run->capacity * sizeof(rle16_t)); + } + // We may have run->runs == NULL. +} + +/* copy one container into another */ +void run_container_copy(const run_container_t *src, run_container_t *dst) { + const int32_t n_runs = src->n_runs; + if (src->n_runs > dst->capacity) { + run_container_grow(dst, n_runs, false); + } + dst->n_runs = n_runs; + memcpy(dst->runs, src->runs, sizeof(rle16_t) * n_runs); +} + +/* Compute the union of `src_1' and `src_2' and write the result to `dst' + * It is assumed that `dst' is distinct from both `src_1' and `src_2'. */ +void run_container_union(const run_container_t *src_1, + const run_container_t *src_2, run_container_t *dst) { + // TODO: this could be a lot more efficient + + // we start out with inexpensive checks + const bool if1 = run_container_is_full(src_1); + const bool if2 = run_container_is_full(src_2); + if (if1 || if2) { + if (if1) { + run_container_copy(src_1, dst); + return; + } + if (if2) { + run_container_copy(src_2, dst); + return; + } + } + const int32_t neededcapacity = src_1->n_runs + src_2->n_runs; + if (dst->capacity < neededcapacity) + run_container_grow(dst, neededcapacity, false); + dst->n_runs = 0; + int32_t rlepos = 0; + int32_t xrlepos = 0; + + rle16_t previousrle; + if (src_1->runs[rlepos].value <= src_2->runs[xrlepos].value) { + previousrle = run_container_append_first(dst, src_1->runs[rlepos]); + rlepos++; + } else { + previousrle = run_container_append_first(dst, src_2->runs[xrlepos]); + xrlepos++; + } + + while ((xrlepos < src_2->n_runs) && (rlepos < src_1->n_runs)) { + rle16_t newrl; + if (src_1->runs[rlepos].value <= src_2->runs[xrlepos].value) { + newrl = src_1->runs[rlepos]; + rlepos++; + } else { + newrl = src_2->runs[xrlepos]; + xrlepos++; + } + run_container_append(dst, newrl, &previousrle); + } + while (xrlepos < src_2->n_runs) { + run_container_append(dst, src_2->runs[xrlepos], &previousrle); + xrlepos++; + } + while (rlepos < src_1->n_runs) { + run_container_append(dst, src_1->runs[rlepos], &previousrle); + rlepos++; + } +} + +/* Compute the union of `src_1' and `src_2' and write the result to `src_1' + */ +void run_container_union_inplace(run_container_t *src_1, + const run_container_t *src_2) { + // TODO: this could be a lot more efficient + + // we start out with inexpensive checks + const bool if1 = run_container_is_full(src_1); + const bool if2 = run_container_is_full(src_2); + if (if1 || if2) { + if (if1) { + return; + } + if (if2) { + run_container_copy(src_2, src_1); + return; + } + } + // we move the data to the end of the current array + const int32_t maxoutput = src_1->n_runs + src_2->n_runs; + const int32_t neededcapacity = maxoutput + src_1->n_runs; + if (src_1->capacity < neededcapacity) + run_container_grow(src_1, neededcapacity, true); + memmove(src_1->runs + maxoutput, src_1->runs, + src_1->n_runs * sizeof(rle16_t)); + rle16_t *inputsrc1 = src_1->runs + maxoutput; + const int32_t input1nruns = src_1->n_runs; + src_1->n_runs = 0; + int32_t rlepos = 0; + int32_t xrlepos = 0; + + rle16_t previousrle; + if (inputsrc1[rlepos].value <= src_2->runs[xrlepos].value) { + previousrle = run_container_append_first(src_1, inputsrc1[rlepos]); + rlepos++; + } else { + previousrle = run_container_append_first(src_1, src_2->runs[xrlepos]); + xrlepos++; + } + while ((xrlepos < src_2->n_runs) && (rlepos < input1nruns)) { + rle16_t newrl; + if (inputsrc1[rlepos].value <= src_2->runs[xrlepos].value) { + newrl = inputsrc1[rlepos]; + rlepos++; + } else { + newrl = src_2->runs[xrlepos]; + xrlepos++; + } + run_container_append(src_1, newrl, &previousrle); + } + while (xrlepos < src_2->n_runs) { + run_container_append(src_1, src_2->runs[xrlepos], &previousrle); + xrlepos++; + } + while (rlepos < input1nruns) { + run_container_append(src_1, inputsrc1[rlepos], &previousrle); + rlepos++; + } +} + +/* Compute the symmetric difference of `src_1' and `src_2' and write the result + * to `dst' + * It is assumed that `dst' is distinct from both `src_1' and `src_2'. */ +void run_container_xor(const run_container_t *src_1, + const run_container_t *src_2, run_container_t *dst) { + // don't bother to convert xor with full range into negation + // since negation is implemented similarly + + const int32_t neededcapacity = src_1->n_runs + src_2->n_runs; + if (dst->capacity < neededcapacity) + run_container_grow(dst, neededcapacity, false); + + int32_t pos1 = 0; + int32_t pos2 = 0; + dst->n_runs = 0; + + while ((pos1 < src_1->n_runs) && (pos2 < src_2->n_runs)) { + if (src_1->runs[pos1].value <= src_2->runs[pos2].value) { + run_container_smart_append_exclusive(dst, src_1->runs[pos1].value, + src_1->runs[pos1].length); + pos1++; + } else { + run_container_smart_append_exclusive(dst, src_2->runs[pos2].value, + src_2->runs[pos2].length); + pos2++; + } + } + while (pos1 < src_1->n_runs) { + run_container_smart_append_exclusive(dst, src_1->runs[pos1].value, + src_1->runs[pos1].length); + pos1++; + } + + while (pos2 < src_2->n_runs) { + run_container_smart_append_exclusive(dst, src_2->runs[pos2].value, + src_2->runs[pos2].length); + pos2++; + } +} + +/* Compute the intersection of src_1 and src_2 and write the result to + * dst. It is assumed that dst is distinct from both src_1 and src_2. */ +void run_container_intersection(const run_container_t *src_1, + const run_container_t *src_2, + run_container_t *dst) { + const bool if1 = run_container_is_full(src_1); + const bool if2 = run_container_is_full(src_2); + if (if1 || if2) { + if (if1) { + run_container_copy(src_2, dst); + return; + } + if (if2) { + run_container_copy(src_1, dst); + return; + } + } + // TODO: this could be a lot more efficient, could use SIMD optimizations + const int32_t neededcapacity = src_1->n_runs + src_2->n_runs; + if (dst->capacity < neededcapacity) + run_container_grow(dst, neededcapacity, false); + dst->n_runs = 0; + int32_t rlepos = 0; + int32_t xrlepos = 0; + int32_t start = src_1->runs[rlepos].value; + int32_t end = start + src_1->runs[rlepos].length + 1; + int32_t xstart = src_2->runs[xrlepos].value; + int32_t xend = xstart + src_2->runs[xrlepos].length + 1; + while ((rlepos < src_1->n_runs) && (xrlepos < src_2->n_runs)) { + if (end <= xstart) { + ++rlepos; + if (rlepos < src_1->n_runs) { + start = src_1->runs[rlepos].value; + end = start + src_1->runs[rlepos].length + 1; + } + } else if (xend <= start) { + ++xrlepos; + if (xrlepos < src_2->n_runs) { + xstart = src_2->runs[xrlepos].value; + xend = xstart + src_2->runs[xrlepos].length + 1; + } + } else { // they overlap + const int32_t lateststart = start > xstart ? start : xstart; + int32_t earliestend; + if (end == xend) { // improbable + earliestend = end; + rlepos++; + xrlepos++; + if (rlepos < src_1->n_runs) { + start = src_1->runs[rlepos].value; + end = start + src_1->runs[rlepos].length + 1; + } + if (xrlepos < src_2->n_runs) { + xstart = src_2->runs[xrlepos].value; + xend = xstart + src_2->runs[xrlepos].length + 1; + } + } else if (end < xend) { + earliestend = end; + rlepos++; + if (rlepos < src_1->n_runs) { + start = src_1->runs[rlepos].value; + end = start + src_1->runs[rlepos].length + 1; + } + + } else { // end > xend + earliestend = xend; + xrlepos++; + if (xrlepos < src_2->n_runs) { + xstart = src_2->runs[xrlepos].value; + xend = xstart + src_2->runs[xrlepos].length + 1; + } + } + dst->runs[dst->n_runs].value = (uint16_t)lateststart; + dst->runs[dst->n_runs].length = + (uint16_t)(earliestend - lateststart - 1); + dst->n_runs++; + } + } +} + +/* Compute the size of the intersection of src_1 and src_2 . */ +int run_container_intersection_cardinality(const run_container_t *src_1, + const run_container_t *src_2) { + const bool if1 = run_container_is_full(src_1); + const bool if2 = run_container_is_full(src_2); + if (if1 || if2) { + if (if1) { + return run_container_cardinality(src_2); + } + if (if2) { + return run_container_cardinality(src_1); + } + } + int answer = 0; + int32_t rlepos = 0; + int32_t xrlepos = 0; + int32_t start = src_1->runs[rlepos].value; + int32_t end = start + src_1->runs[rlepos].length + 1; + int32_t xstart = src_2->runs[xrlepos].value; + int32_t xend = xstart + src_2->runs[xrlepos].length + 1; + while ((rlepos < src_1->n_runs) && (xrlepos < src_2->n_runs)) { + if (end <= xstart) { + ++rlepos; + if (rlepos < src_1->n_runs) { + start = src_1->runs[rlepos].value; + end = start + src_1->runs[rlepos].length + 1; + } + } else if (xend <= start) { + ++xrlepos; + if (xrlepos < src_2->n_runs) { + xstart = src_2->runs[xrlepos].value; + xend = xstart + src_2->runs[xrlepos].length + 1; + } + } else { // they overlap + const int32_t lateststart = start > xstart ? start : xstart; + int32_t earliestend; + if (end == xend) { // improbable + earliestend = end; + rlepos++; + xrlepos++; + if (rlepos < src_1->n_runs) { + start = src_1->runs[rlepos].value; + end = start + src_1->runs[rlepos].length + 1; + } + if (xrlepos < src_2->n_runs) { + xstart = src_2->runs[xrlepos].value; + xend = xstart + src_2->runs[xrlepos].length + 1; + } + } else if (end < xend) { + earliestend = end; + rlepos++; + if (rlepos < src_1->n_runs) { + start = src_1->runs[rlepos].value; + end = start + src_1->runs[rlepos].length + 1; + } + + } else { // end > xend + earliestend = xend; + xrlepos++; + if (xrlepos < src_2->n_runs) { + xstart = src_2->runs[xrlepos].value; + xend = xstart + src_2->runs[xrlepos].length + 1; + } + } + answer += earliestend - lateststart; + } + } + return answer; +} + +bool run_container_intersect(const run_container_t *src_1, + const run_container_t *src_2) { + const bool if1 = run_container_is_full(src_1); + const bool if2 = run_container_is_full(src_2); + if (if1 || if2) { + if (if1) { + return !run_container_empty(src_2); + } + if (if2) { + return !run_container_empty(src_1); + } + } + int32_t rlepos = 0; + int32_t xrlepos = 0; + int32_t start = src_1->runs[rlepos].value; + int32_t end = start + src_1->runs[rlepos].length + 1; + int32_t xstart = src_2->runs[xrlepos].value; + int32_t xend = xstart + src_2->runs[xrlepos].length + 1; + while ((rlepos < src_1->n_runs) && (xrlepos < src_2->n_runs)) { + if (end <= xstart) { + ++rlepos; + if (rlepos < src_1->n_runs) { + start = src_1->runs[rlepos].value; + end = start + src_1->runs[rlepos].length + 1; + } + } else if (xend <= start) { + ++xrlepos; + if (xrlepos < src_2->n_runs) { + xstart = src_2->runs[xrlepos].value; + xend = xstart + src_2->runs[xrlepos].length + 1; + } + } else { // they overlap + return true; + } + } + return false; +} + +/* Compute the difference of src_1 and src_2 and write the result to + * dst. It is assumed that dst is distinct from both src_1 and src_2. */ +void run_container_andnot(const run_container_t *src_1, + const run_container_t *src_2, run_container_t *dst) { + // following Java implementation as of June 2016 + + if (dst->capacity < src_1->n_runs + src_2->n_runs) + run_container_grow(dst, src_1->n_runs + src_2->n_runs, false); + + dst->n_runs = 0; + + int rlepos1 = 0; + int rlepos2 = 0; + int32_t start = src_1->runs[rlepos1].value; + int32_t end = start + src_1->runs[rlepos1].length + 1; + int32_t start2 = src_2->runs[rlepos2].value; + int32_t end2 = start2 + src_2->runs[rlepos2].length + 1; + + while ((rlepos1 < src_1->n_runs) && (rlepos2 < src_2->n_runs)) { + if (end <= start2) { + // output the first run + dst->runs[dst->n_runs++] = MAKE_RLE16(start, end - start - 1); + rlepos1++; + if (rlepos1 < src_1->n_runs) { + start = src_1->runs[rlepos1].value; + end = start + src_1->runs[rlepos1].length + 1; + } + } else if (end2 <= start) { + // exit the second run + rlepos2++; + if (rlepos2 < src_2->n_runs) { + start2 = src_2->runs[rlepos2].value; + end2 = start2 + src_2->runs[rlepos2].length + 1; + } + } else { + if (start < start2) { + dst->runs[dst->n_runs++] = + MAKE_RLE16(start, start2 - start - 1); + } + if (end2 < end) { + start = end2; + } else { + rlepos1++; + if (rlepos1 < src_1->n_runs) { + start = src_1->runs[rlepos1].value; + end = start + src_1->runs[rlepos1].length + 1; + } + } + } + } + if (rlepos1 < src_1->n_runs) { + dst->runs[dst->n_runs++] = MAKE_RLE16(start, end - start - 1); + rlepos1++; + if (rlepos1 < src_1->n_runs) { + memcpy(dst->runs + dst->n_runs, src_1->runs + rlepos1, + sizeof(rle16_t) * (src_1->n_runs - rlepos1)); + dst->n_runs += src_1->n_runs - rlepos1; + } + } +} + +ALLOW_UNALIGNED +int run_container_to_uint32_array(void *vout, const run_container_t *cont, + uint32_t base) { + int outpos = 0; + uint32_t *out = (uint32_t *)vout; + for (int i = 0; i < cont->n_runs; ++i) { + uint32_t run_start = base + cont->runs[i].value; + uint16_t le = cont->runs[i].length; + for (int j = 0; j <= le; ++j) { + uint32_t val = run_start + j; + memcpy(out + outpos, &val, + sizeof(uint32_t)); // should be compiled as a MOV on x64 + outpos++; + } + } + return outpos; +} + +/* + * Print this container using printf (useful for debugging). + */ +void run_container_printf(const run_container_t *cont) { + for (int i = 0; i < cont->n_runs; ++i) { + uint16_t run_start = cont->runs[i].value; + uint16_t le = cont->runs[i].length; + printf("[%d,%d]", run_start, run_start + le); + } +} + +/* + * Print this container using printf as a comma-separated list of 32-bit + * integers starting at base. + */ +void run_container_printf_as_uint32_array(const run_container_t *cont, + uint32_t base) { + if (cont->n_runs == 0) return; + { + uint32_t run_start = base + cont->runs[0].value; + uint16_t le = cont->runs[0].length; + printf("%u", run_start); + for (uint32_t j = 1; j <= le; ++j) printf(",%u", run_start + j); + } + for (int32_t i = 1; i < cont->n_runs; ++i) { + uint32_t run_start = base + cont->runs[i].value; + uint16_t le = cont->runs[i].length; + for (uint32_t j = 0; j <= le; ++j) printf(",%u", run_start + j); + } +} + +/* + * Validate the container. Returns true if valid. + */ +bool run_container_validate(const run_container_t *run, const char **reason) { + if (run->n_runs < 0) { + *reason = "negative run count"; + return false; + } + if (run->capacity < 0) { + *reason = "negative run capacity"; + return false; + } + if (run->capacity < run->n_runs) { + *reason = "capacity less than run count"; + return false; + } + + if (run->n_runs == 0) { + *reason = "zero run count"; + return false; + } + if (run->runs == NULL) { + *reason = "NULL runs"; + return false; + } + + // Use uint32_t to avoid overflow issues on ranges that contain UINT16_MAX. + uint32_t last_end = 0; + for (int i = 0; i < run->n_runs; ++i) { + uint32_t start = run->runs[i].value; + uint32_t end = start + run->runs[i].length + 1; + if (end <= start) { + *reason = "run start + length overflow"; + return false; + } + if (end > (1 << 16)) { + *reason = "run start + length too large"; + return false; + } + if (start < last_end) { + *reason = "run start less than last end"; + return false; + } + if (start == last_end && last_end != 0) { + *reason = "run start equal to last end, should have combined"; + return false; + } + last_end = end; + } + return true; +} + +int32_t run_container_write(const run_container_t *container, char *buf) { + uint16_t cast_16 = container->n_runs; + memcpy(buf, &cast_16, sizeof(uint16_t)); + memcpy(buf + sizeof(uint16_t), container->runs, + container->n_runs * sizeof(rle16_t)); + return run_container_size_in_bytes(container); +} + +int32_t run_container_read(int32_t cardinality, run_container_t *container, + const char *buf) { + (void)cardinality; + uint16_t cast_16; + memcpy(&cast_16, buf, sizeof(uint16_t)); + container->n_runs = cast_16; + if (container->n_runs > container->capacity) + run_container_grow(container, container->n_runs, false); + if (container->n_runs > 0) { + memcpy(container->runs, buf + sizeof(uint16_t), + container->n_runs * sizeof(rle16_t)); + } + return run_container_size_in_bytes(container); +} + +bool run_container_iterate(const run_container_t *cont, uint32_t base, + roaring_iterator iterator, void *ptr) { + for (int i = 0; i < cont->n_runs; ++i) { + uint32_t run_start = base + cont->runs[i].value; + uint16_t le = cont->runs[i].length; + + for (int j = 0; j <= le; ++j) + if (!iterator(run_start + j, ptr)) return false; + } + return true; +} + +bool run_container_iterate64(const run_container_t *cont, uint32_t base, + roaring_iterator64 iterator, uint64_t high_bits, + void *ptr) { + for (int i = 0; i < cont->n_runs; ++i) { + uint32_t run_start = base + cont->runs[i].value; + uint16_t le = cont->runs[i].length; + + for (int j = 0; j <= le; ++j) + if (!iterator(high_bits | (uint64_t)(run_start + j), ptr)) + return false; + } + return true; +} + +bool run_container_is_subset(const run_container_t *container1, + const run_container_t *container2) { + int i1 = 0, i2 = 0; + while (i1 < container1->n_runs && i2 < container2->n_runs) { + int start1 = container1->runs[i1].value; + int stop1 = start1 + container1->runs[i1].length; + int start2 = container2->runs[i2].value; + int stop2 = start2 + container2->runs[i2].length; + if (start1 < start2) { + return false; + } else { // start1 >= start2 + if (stop1 < stop2) { + i1++; + } else if (stop1 == stop2) { + i1++; + i2++; + } else { // stop1 > stop2 + i2++; + } + } + } + if (i1 == container1->n_runs) { + return true; + } else { + return false; + } +} + +// TODO: write smart_append_exclusive version to match the overloaded 1 param +// Java version (or is it even used?) + +// follows the Java implementation closely +// length is the rle-value. Ie, run [10,12) uses a length value 1. +void run_container_smart_append_exclusive(run_container_t *src, + const uint16_t start, + const uint16_t length) { + int old_end; + rle16_t *last_run = src->n_runs ? src->runs + (src->n_runs - 1) : NULL; + rle16_t *appended_last_run = src->runs + src->n_runs; + + if (!src->n_runs || + (start > (old_end = last_run->value + last_run->length + 1))) { + *appended_last_run = MAKE_RLE16(start, length); + src->n_runs++; + return; + } + if (old_end == start) { + // we merge + last_run->length += (length + 1); + return; + } + int new_end = start + length + 1; + + if (start == last_run->value) { + // wipe out previous + if (new_end < old_end) { + *last_run = MAKE_RLE16(new_end, old_end - new_end - 1); + return; + } else if (new_end > old_end) { + *last_run = MAKE_RLE16(old_end, new_end - old_end - 1); + return; + } else { + src->n_runs--; + return; + } + } + last_run->length = start - last_run->value - 1; + if (new_end < old_end) { + *appended_last_run = MAKE_RLE16(new_end, old_end - new_end - 1); + src->n_runs++; + } else if (new_end > old_end) { + *appended_last_run = MAKE_RLE16(old_end, new_end - old_end - 1); + src->n_runs++; + } +} + +bool run_container_select(const run_container_t *container, + uint32_t *start_rank, uint32_t rank, + uint32_t *element) { + for (int i = 0; i < container->n_runs; i++) { + uint16_t length = container->runs[i].length; + if (rank <= *start_rank + length) { + uint16_t value = container->runs[i].value; + *element = value + rank - (*start_rank); + return true; + } else + *start_rank += length + 1; + } + return false; +} + +int run_container_rank(const run_container_t *container, uint16_t x) { + int sum = 0; + uint32_t x32 = x; + for (int i = 0; i < container->n_runs; i++) { + uint32_t startpoint = container->runs[i].value; + uint32_t length = container->runs[i].length; + uint32_t endpoint = length + startpoint; + if (x <= endpoint) { + if (x < startpoint) break; + return sum + (x32 - startpoint) + 1; + } else { + sum += length + 1; + } + } + return sum; +} +uint32_t run_container_rank_many(const run_container_t *container, + uint64_t start_rank, const uint32_t *begin, + const uint32_t *end, uint64_t *ans) { + const uint16_t high = (uint16_t)((*begin) >> 16); + const uint32_t *iter = begin; + int sum = 0; + int i = 0; + for (; iter != end; iter++) { + uint32_t x = *iter; + uint16_t xhigh = (uint16_t)(x >> 16); + if (xhigh != high) return iter - begin; // stop at next container + + uint32_t x32 = x & 0xFFFF; + while (i < container->n_runs) { + uint32_t startpoint = container->runs[i].value; + uint32_t length = container->runs[i].length; + uint32_t endpoint = length + startpoint; + if (x32 <= endpoint) { + if (x32 < startpoint) { + *(ans++) = start_rank + sum; + } else { + *(ans++) = start_rank + sum + (x32 - startpoint) + 1; + } + break; + } else { + sum += length + 1; + i++; + } + } + if (i >= container->n_runs) *(ans++) = start_rank + sum; + } + + return iter - begin; +} + +int run_container_get_index(const run_container_t *container, uint16_t x) { + if (run_container_contains(container, x)) { + int sum = 0; + uint32_t x32 = x; + for (int i = 0; i < container->n_runs; i++) { + uint32_t startpoint = container->runs[i].value; + uint32_t length = container->runs[i].length; + uint32_t endpoint = length + startpoint; + if (x <= endpoint) { + if (x < startpoint) break; + return sum + (x32 - startpoint); + } else { + sum += length + 1; + } + } + return sum - 1; + } else { + return -1; + } +} + +#if defined(CROARING_IS_X64) && CROARING_COMPILER_SUPPORTS_AVX512 + +CROARING_TARGET_AVX512 +ALLOW_UNALIGNED +/* Get the cardinality of `run'. Requires an actual computation. */ +static inline int _avx512_run_container_cardinality( + const run_container_t *run) { + const int32_t n_runs = run->n_runs; + const rle16_t *runs = run->runs; + + /* by initializing with n_runs, we omit counting the +1 for each pair. */ + int sum = n_runs; + int32_t k = 0; + const int32_t step = sizeof(__m512i) / sizeof(rle16_t); + if (n_runs > step) { + __m512i total = _mm512_setzero_si512(); + for (; k + step <= n_runs; k += step) { + __m512i ymm1 = _mm512_loadu_si512((const __m512i *)(runs + k)); + __m512i justlengths = _mm512_srli_epi32(ymm1, 16); + total = _mm512_add_epi32(total, justlengths); + } + + __m256i lo = _mm512_extracti32x8_epi32(total, 0); + __m256i hi = _mm512_extracti32x8_epi32(total, 1); + + // a store might be faster than extract? + uint32_t buffer[sizeof(__m256i) / sizeof(rle16_t)]; + _mm256_storeu_si256((__m256i *)buffer, lo); + sum += (buffer[0] + buffer[1]) + (buffer[2] + buffer[3]) + + (buffer[4] + buffer[5]) + (buffer[6] + buffer[7]); + + _mm256_storeu_si256((__m256i *)buffer, hi); + sum += (buffer[0] + buffer[1]) + (buffer[2] + buffer[3]) + + (buffer[4] + buffer[5]) + (buffer[6] + buffer[7]); + } + for (; k < n_runs; ++k) { + sum += runs[k].length; + } + + return sum; +} + +CROARING_UNTARGET_AVX512 + +CROARING_TARGET_AVX2 +ALLOW_UNALIGNED +/* Get the cardinality of `run'. Requires an actual computation. */ +static inline int _avx2_run_container_cardinality(const run_container_t *run) { + const int32_t n_runs = run->n_runs; + const rle16_t *runs = run->runs; + + /* by initializing with n_runs, we omit counting the +1 for each pair. */ + int sum = n_runs; + int32_t k = 0; + const int32_t step = sizeof(__m256i) / sizeof(rle16_t); + if (n_runs > step) { + __m256i total = _mm256_setzero_si256(); + for (; k + step <= n_runs; k += step) { + __m256i ymm1 = _mm256_lddqu_si256((const __m256i *)(runs + k)); + __m256i justlengths = _mm256_srli_epi32(ymm1, 16); + total = _mm256_add_epi32(total, justlengths); + } + // a store might be faster than extract? + uint32_t buffer[sizeof(__m256i) / sizeof(rle16_t)]; + _mm256_storeu_si256((__m256i *)buffer, total); + sum += (buffer[0] + buffer[1]) + (buffer[2] + buffer[3]) + + (buffer[4] + buffer[5]) + (buffer[6] + buffer[7]); + } + for (; k < n_runs; ++k) { + sum += runs[k].length; + } + + return sum; +} + +CROARING_UNTARGET_AVX2 + +/* Get the cardinality of `run'. Requires an actual computation. */ +static inline int _scalar_run_container_cardinality( + const run_container_t *run) { + const int32_t n_runs = run->n_runs; + const rle16_t *runs = run->runs; + + /* by initializing with n_runs, we omit counting the +1 for each pair. */ + int sum = n_runs; + for (int k = 0; k < n_runs; ++k) { + sum += runs[k].length; + } + + return sum; +} + +int run_container_cardinality(const run_container_t *run) { +#if CROARING_COMPILER_SUPPORTS_AVX512 + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX512) { + return _avx512_run_container_cardinality(run); + } else +#endif + if (croaring_hardware_support() & ROARING_SUPPORTS_AVX2) { + return _avx2_run_container_cardinality(run); + } else { + return _scalar_run_container_cardinality(run); + } +} +#else + +/* Get the cardinality of `run'. Requires an actual computation. */ +int run_container_cardinality(const run_container_t *run) { + const int32_t n_runs = run->n_runs; + const rle16_t *runs = run->runs; + + /* by initializing with n_runs, we omit counting the +1 for each pair. */ + int sum = n_runs; + for (int k = 0; k < n_runs; ++k) { + sum += runs[k].length; + } + + return sum; +} +#endif + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif
\ No newline at end of file diff --git a/contrib/libs/croaring/src/isadetection.c b/contrib/libs/croaring/src/isadetection.c new file mode 100644 index 00000000000..d7904149ba9 --- /dev/null +++ b/contrib/libs/croaring/src/isadetection.c @@ -0,0 +1,339 @@ + +/* From +https://github.com/endorno/pytorch/blob/master/torch/lib/TH/generic/simd/simd.h +Highly modified. + +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, +Iain Melvin, Jason Weston) Copyright (c) 2006 Idiap Research Institute +(Samy Bengio) Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, +Samy Bengio, Johnny Mariethoz) + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories +America and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. +*/ + +#include <stdbool.h> +#include <stdint.h> +#include <stdlib.h> + +// Binaries produced by Visual Studio with solely AVX2 routines +// can compile to AVX-512 thus causing crashes on non-AVX-512 systems. +// This appears to affect VS 17.8 and 17.9. We disable AVX-512 and AVX2 +// on these systems. It seems that ClangCL is not affected. +// https://github.com/RoaringBitmap/CRoaring/pull/603 +#ifndef __clang__ +#if _MSC_VER >= 1938 +#define ROARING_DISABLE_AVX 1 +#endif // _MSC_VER >= 1938 +#endif // __clang__ + +// We need portability.h to be included first, see +// https://github.com/RoaringBitmap/CRoaring/issues/394 +#include <roaring/portability.h> +#if CROARING_REGULAR_VISUAL_STUDIO +#include <intrin.h> +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#include <cpuid.h> +#endif // CROARING_REGULAR_VISUAL_STUDIO +#include <roaring/isadetection.h> + +#if CROARING_IS_X64 +#ifndef CROARING_COMPILER_SUPPORTS_AVX512 +#error "CROARING_COMPILER_SUPPORTS_AVX512 needs to be defined." +#endif // CROARING_COMPILER_SUPPORTS_AVX512 +#endif + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif +enum croaring_instruction_set { + CROARING_DEFAULT = 0x0, + CROARING_NEON = 0x1, + CROARING_AVX2 = 0x4, + CROARING_SSE42 = 0x8, + CROARING_PCLMULQDQ = 0x10, + CROARING_BMI1 = 0x20, + CROARING_BMI2 = 0x40, + CROARING_ALTIVEC = 0x80, + CROARING_AVX512F = 0x100, + CROARING_AVX512DQ = 0x200, + CROARING_AVX512BW = 0x400, + CROARING_AVX512VBMI2 = 0x800, + CROARING_AVX512BITALG = 0x1000, + CROARING_AVX512VPOPCNTDQ = 0x2000, + CROARING_UNINITIALIZED = 0x8000 +}; + +#if CROARING_COMPILER_SUPPORTS_AVX512 +unsigned int CROARING_AVX512_REQUIRED = + (CROARING_AVX512F | CROARING_AVX512DQ | CROARING_AVX512BW | + CROARING_AVX512VBMI2 | CROARING_AVX512BITALG | CROARING_AVX512VPOPCNTDQ); +#endif + +#if defined(__x86_64__) || defined(_M_AMD64) // x64 + +static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, + uint32_t *edx) { +#if CROARING_REGULAR_VISUAL_STUDIO + int cpu_info[4]; + __cpuidex(cpu_info, *eax, *ecx); + *eax = cpu_info[0]; + *ebx = cpu_info[1]; + *ecx = cpu_info[2]; + *edx = cpu_info[3]; +#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) + uint32_t level = *eax; + __get_cpuid(level, eax, ebx, ecx, edx); +#else + uint32_t a = *eax, b, c = *ecx, d; + __asm__("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); + *eax = a; + *ebx = b; + *ecx = c; + *edx = d; +#endif +} + +static inline uint64_t xgetbv(void) { +#if defined(_MSC_VER) + return _xgetbv(0); +#else + uint32_t xcr0_lo, xcr0_hi; + __asm__("xgetbv\n\t" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); + return xcr0_lo | ((uint64_t)xcr0_hi << 32); +#endif +} + +/** + * This is a relatively expensive function but it will get called at most + * *once* per compilation units. Normally, the CRoaring library is built + * as one compilation unit. + */ +static inline uint32_t dynamic_croaring_detect_supported_architectures(void) { + uint32_t eax, ebx, ecx, edx; + uint32_t host_isa = 0x0; + // Can be found on Intel ISA Reference for CPUID + static uint32_t cpuid_avx2_bit = + 1 << 5; ///< @private Bit 5 of EBX for EAX=0x7 + static uint32_t cpuid_bmi1_bit = + 1 << 3; ///< @private bit 3 of EBX for EAX=0x7 + static uint32_t cpuid_bmi2_bit = + 1 << 8; ///< @private bit 8 of EBX for EAX=0x7 + static uint32_t cpuid_avx512f_bit = + 1 << 16; ///< @private bit 16 of EBX for EAX=0x7 + static uint32_t cpuid_avx512dq_bit = + 1 << 17; ///< @private bit 17 of EBX for EAX=0x7 + static uint32_t cpuid_avx512bw_bit = + 1 << 30; ///< @private bit 30 of EBX for EAX=0x7 + static uint32_t cpuid_avx512vbmi2_bit = + 1 << 6; ///< @private bit 6 of ECX for EAX=0x7 + static uint32_t cpuid_avx512bitalg_bit = + 1 << 12; ///< @private bit 12 of ECX for EAX=0x7 + static uint32_t cpuid_avx512vpopcntdq_bit = + 1 << 14; ///< @private bit 14 of ECX for EAX=0x7 + static uint64_t cpuid_avx256_saved = 1 << 2; ///< @private bit 2 = AVX + static uint64_t cpuid_avx512_saved = + 7 << 5; ///< @private bits 5,6,7 = opmask, ZMM_hi256, hi16_ZMM + static uint32_t cpuid_sse42_bit = + 1 << 20; ///< @private bit 20 of ECX for EAX=0x1 + static uint32_t cpuid_osxsave = + (1 << 26) | (1 << 27); ///< @private bits 26+27 of ECX for EAX=0x1 + static uint32_t cpuid_pclmulqdq_bit = + 1 << 1; ///< @private bit 1 of ECX for EAX=0x1 + + // EBX for EAX=0x1 + eax = 0x1; + ecx = 0x0; + cpuid(&eax, &ebx, &ecx, &edx); + + if (ecx & cpuid_sse42_bit) { + host_isa |= CROARING_SSE42; + } else { + return host_isa; // everything after is redundant + } + + if (ecx & cpuid_pclmulqdq_bit) { + host_isa |= CROARING_PCLMULQDQ; + } + + if ((ecx & cpuid_osxsave) != cpuid_osxsave) { + return host_isa; + } + + // xgetbv for checking if the OS saves registers + uint64_t xcr0 = xgetbv(); + + if ((xcr0 & cpuid_avx256_saved) == 0) { + return host_isa; + } + + // ECX for EAX=0x7 + eax = 0x7; + ecx = 0x0; + cpuid(&eax, &ebx, &ecx, &edx); + if (ebx & cpuid_avx2_bit) { + host_isa |= CROARING_AVX2; + } + if (ebx & cpuid_bmi1_bit) { + host_isa |= CROARING_BMI1; + } + + if (ebx & cpuid_bmi2_bit) { + host_isa |= CROARING_BMI2; + } + + if (!((xcr0 & cpuid_avx512_saved) == cpuid_avx512_saved)) { + return host_isa; + } + + if (ebx & cpuid_avx512f_bit) { + host_isa |= CROARING_AVX512F; + } + + if (ebx & cpuid_avx512bw_bit) { + host_isa |= CROARING_AVX512BW; + } + + if (ebx & cpuid_avx512dq_bit) { + host_isa |= CROARING_AVX512DQ; + } + + if (ecx & cpuid_avx512vbmi2_bit) { + host_isa |= CROARING_AVX512VBMI2; + } + + if (ecx & cpuid_avx512bitalg_bit) { + host_isa |= CROARING_AVX512BITALG; + } + + if (ecx & cpuid_avx512vpopcntdq_bit) { + host_isa |= CROARING_AVX512VPOPCNTDQ; + } + + return host_isa; +} + +#endif // end SIMD extension detection code + +#if defined(__x86_64__) || defined(_M_AMD64) // x64 + +#if CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_CPP +static inline uint32_t croaring_detect_supported_architectures(void) { + // thread-safe as per the C++11 standard. + static uint32_t buffer = dynamic_croaring_detect_supported_architectures(); + return buffer; +} +#elif CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_C +static uint32_t croaring_detect_supported_architectures(void) { + // we use an atomic for thread safety + static _Atomic uint32_t buffer = CROARING_UNINITIALIZED; + if (buffer == CROARING_UNINITIALIZED) { + // atomicity is sufficient + buffer = dynamic_croaring_detect_supported_architectures(); + } + return buffer; +} +#else +// If we do not have atomics, we do the best we can. +static inline uint32_t croaring_detect_supported_architectures(void) { + static uint32_t buffer = CROARING_UNINITIALIZED; + if (buffer == CROARING_UNINITIALIZED) { + buffer = dynamic_croaring_detect_supported_architectures(); + } + return buffer; +} +#endif // CROARING_C_ATOMIC + +#ifdef ROARING_DISABLE_AVX + +int croaring_hardware_support(void) { return 0; } + +#elif defined(__AVX512F__) && defined(__AVX512DQ__) && \ + defined(__AVX512BW__) && defined(__AVX512VBMI2__) && \ + defined(__AVX512BITALG__) && defined(__AVX512VPOPCNTDQ__) +int croaring_hardware_support(void) { + return ROARING_SUPPORTS_AVX2 | ROARING_SUPPORTS_AVX512; +} +#elif defined(__AVX2__) + +int croaring_hardware_support(void) { + static +#if CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_C + _Atomic +#endif + int support = 0xFFFFFFF; + if (support == 0xFFFFFFF) { + bool avx512_support = false; +#if CROARING_COMPILER_SUPPORTS_AVX512 + avx512_support = + ((croaring_detect_supported_architectures() & + CROARING_AVX512_REQUIRED) == CROARING_AVX512_REQUIRED); +#endif + support = ROARING_SUPPORTS_AVX2 | + (avx512_support ? ROARING_SUPPORTS_AVX512 : 0); + } + return support; +} +#else + +int croaring_hardware_support(void) { + static +#if CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_C + _Atomic +#endif + int support = 0xFFFFFFF; + if (support == 0xFFFFFFF) { + bool has_avx2 = (croaring_detect_supported_architectures() & + CROARING_AVX2) == CROARING_AVX2; + bool has_avx512 = false; +#if CROARING_COMPILER_SUPPORTS_AVX512 + has_avx512 = (croaring_detect_supported_architectures() & + CROARING_AVX512_REQUIRED) == CROARING_AVX512_REQUIRED; +#endif // CROARING_COMPILER_SUPPORTS_AVX512 + support = (has_avx2 ? ROARING_SUPPORTS_AVX2 : 0) | + (has_avx512 ? ROARING_SUPPORTS_AVX512 : 0); + } + return support; +} +#endif + +#endif // defined(__x86_64__) || defined(_M_AMD64) // x64 +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif diff --git a/contrib/libs/croaring/src/memory.c b/contrib/libs/croaring/src/memory.c new file mode 100644 index 00000000000..1b474ed992b --- /dev/null +++ b/contrib/libs/croaring/src/memory.c @@ -0,0 +1,64 @@ +#include <stdlib.h> + +#include <roaring/memory.h> + +// without the following, we get lots of warnings about posix_memalign +#ifndef __cplusplus +extern int posix_memalign(void** __memptr, size_t __alignment, size_t __size); +#endif //__cplusplus // C++ does not have a well defined signature + +// portable version of posix_memalign +static void* roaring_bitmap_aligned_malloc(size_t alignment, size_t size) { + void* p; +#ifdef _MSC_VER + p = _aligned_malloc(size, alignment); +#elif defined(__MINGW32__) || defined(__MINGW64__) + p = __mingw_aligned_malloc(size, alignment); +#else + // somehow, if this is used before including "x86intrin.h", it creates an + // implicit defined warning. + if (posix_memalign(&p, alignment, size) != 0) return NULL; +#endif + return p; +} + +static void roaring_bitmap_aligned_free(void* memblock) { +#ifdef _MSC_VER + _aligned_free(memblock); +#elif defined(__MINGW32__) || defined(__MINGW64__) + __mingw_aligned_free(memblock); +#else + free(memblock); +#endif +} + +static roaring_memory_t global_memory_hook = { + .malloc = malloc, + .realloc = realloc, + .calloc = calloc, + .free = free, + .aligned_malloc = roaring_bitmap_aligned_malloc, + .aligned_free = roaring_bitmap_aligned_free, +}; + +void roaring_init_memory_hook(roaring_memory_t memory_hook) { + global_memory_hook = memory_hook; +} + +void* roaring_malloc(size_t n) { return global_memory_hook.malloc(n); } + +void* roaring_realloc(void* p, size_t new_sz) { + return global_memory_hook.realloc(p, new_sz); +} + +void* roaring_calloc(size_t n_elements, size_t element_size) { + return global_memory_hook.calloc(n_elements, element_size); +} + +void roaring_free(void* p) { global_memory_hook.free(p); } + +void* roaring_aligned_malloc(size_t alignment, size_t size) { + return global_memory_hook.aligned_malloc(alignment, size); +} + +void roaring_aligned_free(void* p) { global_memory_hook.aligned_free(p); } diff --git a/contrib/libs/croaring/src/roaring.c b/contrib/libs/croaring/src/roaring.c new file mode 100644 index 00000000000..f644b8f071a --- /dev/null +++ b/contrib/libs/croaring/src/roaring.c @@ -0,0 +1,3386 @@ +#include <assert.h> +#include <inttypes.h> +#include <stdarg.h> +#include <stdint.h> +#include <stdio.h> +#include <string.h> + +#include <roaring/roaring.h> + +// Include after roaring.h +#include <roaring/array_util.h> +#include <roaring/bitset_util.h> +#include <roaring/containers/containers.h> +#include <roaring/roaring_array.h> + +#ifdef __cplusplus +using namespace ::roaring::internal; + +extern "C" { +namespace roaring { +namespace api { +#endif + +#define CROARING_SERIALIZATION_ARRAY_UINT32 1 +#define CROARING_SERIALIZATION_CONTAINER 2 +extern inline int roaring_trailing_zeroes(unsigned long long input_num); +extern inline int roaring_leading_zeroes(unsigned long long input_num); +extern inline void roaring_bitmap_init_cleared(roaring_bitmap_t *r); +extern inline bool roaring_bitmap_get_copy_on_write(const roaring_bitmap_t *r); +extern inline void roaring_bitmap_set_copy_on_write(roaring_bitmap_t *r, + bool cow); +extern inline roaring_bitmap_t *roaring_bitmap_create(void); +extern inline void roaring_bitmap_add_range(roaring_bitmap_t *r, uint64_t min, + uint64_t max); +extern inline void roaring_bitmap_remove_range(roaring_bitmap_t *r, + uint64_t min, uint64_t max); + +static inline bool is_cow(const roaring_bitmap_t *r) { + return r->high_low_container.flags & ROARING_FLAG_COW; +} +static inline bool is_frozen(const roaring_bitmap_t *r) { + return r->high_low_container.flags & ROARING_FLAG_FROZEN; +} + +// this is like roaring_bitmap_add, but it populates pointer arguments in such a +// way +// that we can recover the container touched, which, in turn can be used to +// accelerate some functions (when you repeatedly need to add to the same +// container) +static inline container_t *containerptr_roaring_bitmap_add(roaring_bitmap_t *r, + uint32_t val, + uint8_t *type, + int *index) { + roaring_array_t *ra = &r->high_low_container; + + uint16_t hb = val >> 16; + const int i = ra_get_index(ra, hb); + if (i >= 0) { + ra_unshare_container_at_index(ra, (uint16_t)i); + container_t *c = ra_get_container_at_index(ra, (uint16_t)i, type); + uint8_t new_type = *type; + container_t *c2 = container_add(c, val & 0xFFFF, *type, &new_type); + *index = i; + if (c2 != c) { + container_free(c, *type); + ra_set_container_at_index(ra, i, c2, new_type); + *type = new_type; + return c2; + } else { + return c; + } + } else { + array_container_t *new_ac = array_container_create(); + container_t *c = + container_add(new_ac, val & 0xFFFF, ARRAY_CONTAINER_TYPE, type); + // we could just assume that it stays an array container + ra_insert_new_key_value_at(ra, -i - 1, hb, c, *type); + *index = -i - 1; + return c; + } +} + +roaring_bitmap_t *roaring_bitmap_create_with_capacity(uint32_t cap) { + roaring_bitmap_t *ans = + (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); + if (!ans) { + return NULL; + } + bool is_ok = ra_init_with_capacity(&ans->high_low_container, cap); + if (!is_ok) { + roaring_free(ans); + return NULL; + } + return ans; +} + +bool roaring_bitmap_init_with_capacity(roaring_bitmap_t *r, uint32_t cap) { + return ra_init_with_capacity(&r->high_low_container, cap); +} + +static inline void add_bulk_impl(roaring_bitmap_t *r, + roaring_bulk_context_t *context, + uint32_t val) { + uint16_t key = val >> 16; + if (context->container == NULL || context->key != key) { + uint8_t typecode; + int idx; + context->container = + containerptr_roaring_bitmap_add(r, val, &typecode, &idx); + context->typecode = typecode; + context->idx = idx; + context->key = key; + } else { + // no need to seek the container, it is at hand + // because we already have the container at hand, we can do the + // insertion directly, bypassing the roaring_bitmap_add call + uint8_t new_typecode; + container_t *container2 = container_add( + context->container, val & 0xFFFF, context->typecode, &new_typecode); + if (container2 != context->container) { + // rare instance when we need to change the container type + container_free(context->container, context->typecode); + ra_set_container_at_index(&r->high_low_container, context->idx, + container2, new_typecode); + context->typecode = new_typecode; + context->container = container2; + } + } +} + +void roaring_bitmap_add_many(roaring_bitmap_t *r, size_t n_args, + const uint32_t *vals) { + uint32_t val; + const uint32_t *start = vals; + const uint32_t *end = vals + n_args; + const uint32_t *current_val = start; + + if (n_args == 0) { + return; + } + + uint8_t typecode; + int idx; + container_t *container; + val = *current_val; + container = containerptr_roaring_bitmap_add(r, val, &typecode, &idx); + roaring_bulk_context_t context = {container, idx, (uint16_t)(val >> 16), + typecode}; + + for (; current_val != end; current_val++) { + memcpy(&val, current_val, sizeof(val)); + add_bulk_impl(r, &context, val); + } +} + +void roaring_bitmap_add_bulk(roaring_bitmap_t *r, + roaring_bulk_context_t *context, uint32_t val) { + add_bulk_impl(r, context, val); +} + +bool roaring_bitmap_contains_bulk(const roaring_bitmap_t *r, + roaring_bulk_context_t *context, + uint32_t val) { + uint16_t key = val >> 16; + if (context->container == NULL || context->key != key) { + int32_t start_idx = -1; + if (context->container != NULL && context->key < key) { + start_idx = context->idx; + } + int idx = ra_advance_until(&r->high_low_container, key, start_idx); + if (idx == ra_get_size(&r->high_low_container)) { + return false; + } + uint8_t typecode; + context->container = ra_get_container_at_index( + &r->high_low_container, (uint16_t)idx, &typecode); + context->typecode = typecode; + context->idx = idx; + context->key = + ra_get_key_at_index(&r->high_low_container, (uint16_t)idx); + // ra_advance_until finds the next key >= the target, we found a later + // container. + if (context->key != key) { + return false; + } + } + // context is now set up + return container_contains(context->container, val & 0xFFFF, + context->typecode); +} + +roaring_bitmap_t *roaring_bitmap_of_ptr(size_t n_args, const uint32_t *vals) { + roaring_bitmap_t *answer = roaring_bitmap_create(); + roaring_bitmap_add_many(answer, n_args, vals); + return answer; +} + +roaring_bitmap_t *roaring_bitmap_of(size_t n_args, ...) { + // todo: could be greatly optimized but we do not expect this call to ever + // include long lists + roaring_bitmap_t *answer = roaring_bitmap_create(); + roaring_bulk_context_t context = {0}; + va_list ap; + va_start(ap, n_args); + for (size_t i = 0; i < n_args; i++) { + uint32_t val = va_arg(ap, uint32_t); + roaring_bitmap_add_bulk(answer, &context, val); + } + va_end(ap); + return answer; +} + +static inline uint64_t minimum_uint64(uint64_t a, uint64_t b) { + return (a < b) ? a : b; +} + +roaring_bitmap_t *roaring_bitmap_from_range(uint64_t min, uint64_t max, + uint32_t step) { + if (max >= UINT64_C(0x100000000)) { + max = UINT64_C(0x100000000); + } + if (step == 0) return NULL; + if (max <= min) return NULL; + roaring_bitmap_t *answer = roaring_bitmap_create(); + if (step >= (1 << 16)) { + for (uint32_t value = (uint32_t)min; value < max; value += step) { + roaring_bitmap_add(answer, value); + } + return answer; + } + uint64_t min_tmp = min; + do { + uint32_t key = (uint32_t)min_tmp >> 16; + uint32_t container_min = min_tmp & 0xFFFF; + uint32_t container_max = + (uint32_t)minimum_uint64(max - (key << 16), 1 << 16); + uint8_t type; + container_t *container = container_from_range( + &type, container_min, container_max, (uint16_t)step); + ra_append(&answer->high_low_container, (uint16_t)key, container, type); + uint32_t gap = container_max - container_min + step - 1; + min_tmp += gap - (gap % step); + } while (min_tmp < max); + // cardinality of bitmap will be ((uint64_t) max - min + step - 1 ) / step + return answer; +} + +void roaring_bitmap_add_range_closed(roaring_bitmap_t *r, uint32_t min, + uint32_t max) { + if (min > max) { + return; + } + + roaring_array_t *ra = &r->high_low_container; + + uint32_t min_key = min >> 16; + uint32_t max_key = max >> 16; + + int32_t num_required_containers = max_key - min_key + 1; + int32_t suffix_length = + count_greater(ra->keys, ra->size, (uint16_t)max_key); + int32_t prefix_length = + count_less(ra->keys, ra->size - suffix_length, (uint16_t)min_key); + int32_t common_length = ra->size - prefix_length - suffix_length; + + if (num_required_containers > common_length) { + ra_shift_tail(ra, suffix_length, + num_required_containers - common_length); + } + + int32_t src = prefix_length + common_length - 1; + int32_t dst = ra->size - suffix_length - 1; + for (uint32_t key = max_key; key != min_key - 1; + key--) { // beware of min_key==0 + uint32_t container_min = (min_key == key) ? (min & 0xffff) : 0; + uint32_t container_max = (max_key == key) ? (max & 0xffff) : 0xffff; + container_t *new_container; + uint8_t new_type; + + if (src >= 0 && ra->keys[src] == key) { + ra_unshare_container_at_index(ra, (uint16_t)src); + new_container = + container_add_range(ra->containers[src], ra->typecodes[src], + container_min, container_max, &new_type); + if (new_container != ra->containers[src]) { + container_free(ra->containers[src], ra->typecodes[src]); + } + src--; + } else { + new_container = container_from_range(&new_type, container_min, + container_max + 1, 1); + } + ra_replace_key_and_container_at_index(ra, dst, (uint16_t)key, + new_container, new_type); + dst--; + } +} + +void roaring_bitmap_remove_range_closed(roaring_bitmap_t *r, uint32_t min, + uint32_t max) { + if (min > max) { + return; + } + + roaring_array_t *ra = &r->high_low_container; + + uint32_t min_key = min >> 16; + uint32_t max_key = max >> 16; + + int32_t src = count_less(ra->keys, ra->size, (uint16_t)min_key); + int32_t dst = src; + while (src < ra->size && ra->keys[src] <= max_key) { + uint32_t container_min = + (min_key == ra->keys[src]) ? (min & 0xffff) : 0; + uint32_t container_max = + (max_key == ra->keys[src]) ? (max & 0xffff) : 0xffff; + ra_unshare_container_at_index(ra, (uint16_t)src); + container_t *new_container; + uint8_t new_type; + new_container = + container_remove_range(ra->containers[src], ra->typecodes[src], + container_min, container_max, &new_type); + if (new_container != ra->containers[src]) { + container_free(ra->containers[src], ra->typecodes[src]); + } + if (new_container) { + ra_replace_key_and_container_at_index(ra, dst, ra->keys[src], + new_container, new_type); + dst++; + } + src++; + } + if (src > dst) { + ra_shift_tail(ra, ra->size - src, dst - src); + } +} + +void roaring_bitmap_printf(const roaring_bitmap_t *r) { + const roaring_array_t *ra = &r->high_low_container; + + printf("{"); + for (int i = 0; i < ra->size; ++i) { + container_printf_as_uint32_array(ra->containers[i], ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16); + + if (i + 1 < ra->size) { + printf(","); + } + } + printf("}"); +} + +void roaring_bitmap_printf_describe(const roaring_bitmap_t *r) { + const roaring_array_t *ra = &r->high_low_container; + + printf("{"); + for (int i = 0; i < ra->size; ++i) { + printf("%d: %s (%d)", ra->keys[i], + get_full_container_name(ra->containers[i], ra->typecodes[i]), + container_get_cardinality(ra->containers[i], ra->typecodes[i])); + if (ra->typecodes[i] == SHARED_CONTAINER_TYPE) { + printf("(shared count = %" PRIu32 " )", + croaring_refcount_get( + &(CAST_shared(ra->containers[i])->counter))); + } + + if (i + 1 < ra->size) { + printf(", "); + } + } + printf("}"); +} + +typedef struct min_max_sum_s { + uint32_t min; + uint32_t max; + uint64_t sum; +} min_max_sum_t; + +static bool min_max_sum_fnc(uint32_t value, void *param) { + min_max_sum_t *mms = (min_max_sum_t *)param; + if (value > mms->max) mms->max = value; + if (value < mms->min) mms->min = value; + mms->sum += value; + return true; // we always process all data points +} + +/** + * (For advanced users.) + * Collect statistics about the bitmap + */ +void roaring_bitmap_statistics(const roaring_bitmap_t *r, + roaring_statistics_t *stat) { + const roaring_array_t *ra = &r->high_low_container; + + memset(stat, 0, sizeof(*stat)); + stat->n_containers = ra->size; + stat->cardinality = roaring_bitmap_get_cardinality(r); + min_max_sum_t mms; + mms.min = UINT32_C(0xFFFFFFFF); + mms.max = UINT32_C(0); + mms.sum = 0; + roaring_iterate(r, &min_max_sum_fnc, &mms); + stat->min_value = mms.min; + stat->max_value = mms.max; + stat->sum_value = mms.sum; + + for (int i = 0; i < ra->size; ++i) { + uint8_t truetype = + get_container_type(ra->containers[i], ra->typecodes[i]); + uint32_t card = + container_get_cardinality(ra->containers[i], ra->typecodes[i]); + uint32_t sbytes = + container_size_in_bytes(ra->containers[i], ra->typecodes[i]); + switch (truetype) { + case BITSET_CONTAINER_TYPE: + stat->n_bitset_containers++; + stat->n_values_bitset_containers += card; + stat->n_bytes_bitset_containers += sbytes; + break; + case ARRAY_CONTAINER_TYPE: + stat->n_array_containers++; + stat->n_values_array_containers += card; + stat->n_bytes_array_containers += sbytes; + break; + case RUN_CONTAINER_TYPE: + stat->n_run_containers++; + stat->n_values_run_containers += card; + stat->n_bytes_run_containers += sbytes; + break; + default: + assert(false); + roaring_unreachable; + } + } +} + +/* + * Checks that: + * - Array containers are sorted and contain no duplicates + * - Range containers are sorted and contain no overlapping ranges + * - Roaring containers are sorted by key and there are no duplicate keys + * - The correct container type is use for each container (e.g. bitmaps aren't + * used for small containers) + */ +bool roaring_bitmap_internal_validate(const roaring_bitmap_t *r, + const char **reason) { + const char *reason_local; + if (reason == NULL) { + // Always allow assigning through *reason + reason = &reason_local; + } + *reason = NULL; + const roaring_array_t *ra = &r->high_low_container; + if (ra->size < 0) { + *reason = "negative size"; + return false; + } + if (ra->allocation_size < 0) { + *reason = "negative allocation size"; + return false; + } + if (ra->size > ra->allocation_size) { + *reason = "more containers than allocated space"; + return false; + } + if (ra->flags & ~(ROARING_FLAG_COW | ROARING_FLAG_FROZEN)) { + *reason = "invalid flags"; + return false; + } + if (ra->size == 0) { + return true; + } + + if (ra->keys == NULL) { + *reason = "keys is NULL"; + return false; + } + if (ra->typecodes == NULL) { + *reason = "typecodes is NULL"; + return false; + } + if (ra->containers == NULL) { + *reason = "containers is NULL"; + return false; + } + + uint32_t prev_key = ra->keys[0]; + for (int32_t i = 1; i < ra->size; ++i) { + if (ra->keys[i] <= prev_key) { + *reason = "keys not strictly increasing"; + return false; + } + prev_key = ra->keys[i]; + } + + for (int32_t i = 0; i < ra->size; ++i) { + if (!container_internal_validate(ra->containers[i], ra->typecodes[i], + reason)) { + // reason should already be set + if (*reason == NULL) { + *reason = "container failed to validate but no reason given"; + } + return false; + } + } + + return true; +} + +roaring_bitmap_t *roaring_bitmap_copy(const roaring_bitmap_t *r) { + roaring_bitmap_t *ans = + (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); + if (!ans) { + return NULL; + } + if (!ra_init_with_capacity( // allocation of list of containers can fail + &ans->high_low_container, r->high_low_container.size)) { + roaring_free(ans); + return NULL; + } + if (!ra_overwrite( // memory allocation of individual containers may fail + &r->high_low_container, &ans->high_low_container, is_cow(r))) { + roaring_bitmap_free(ans); // overwrite should leave in freeable state + return NULL; + } + roaring_bitmap_set_copy_on_write(ans, is_cow(r)); + return ans; +} + +bool roaring_bitmap_overwrite(roaring_bitmap_t *dest, + const roaring_bitmap_t *src) { + roaring_bitmap_set_copy_on_write(dest, is_cow(src)); + return ra_overwrite(&src->high_low_container, &dest->high_low_container, + is_cow(src)); +} + +void roaring_bitmap_free(const roaring_bitmap_t *r) { + if (r == NULL) { + return; + } + if (!is_frozen(r)) { + ra_clear((roaring_array_t *)&r->high_low_container); + } + roaring_free((roaring_bitmap_t *)r); +} + +void roaring_bitmap_clear(roaring_bitmap_t *r) { + ra_reset(&r->high_low_container); +} + +void roaring_bitmap_add(roaring_bitmap_t *r, uint32_t val) { + roaring_array_t *ra = &r->high_low_container; + + const uint16_t hb = val >> 16; + const int i = ra_get_index(ra, hb); + uint8_t typecode; + if (i >= 0) { + ra_unshare_container_at_index(ra, (uint16_t)i); + container_t *container = + ra_get_container_at_index(ra, (uint16_t)i, &typecode); + uint8_t newtypecode = typecode; + container_t *container2 = + container_add(container, val & 0xFFFF, typecode, &newtypecode); + if (container2 != container) { + container_free(container, typecode); + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); + } + } else { + array_container_t *newac = array_container_create(); + container_t *container = + container_add(newac, val & 0xFFFF, ARRAY_CONTAINER_TYPE, &typecode); + // we could just assume that it stays an array container + ra_insert_new_key_value_at(&r->high_low_container, -i - 1, hb, + container, typecode); + } +} + +bool roaring_bitmap_add_checked(roaring_bitmap_t *r, uint32_t val) { + const uint16_t hb = val >> 16; + const int i = ra_get_index(&r->high_low_container, hb); + uint8_t typecode; + bool result = false; + if (i >= 0) { + ra_unshare_container_at_index(&r->high_low_container, (uint16_t)i); + container_t *container = ra_get_container_at_index( + &r->high_low_container, (uint16_t)i, &typecode); + + const int oldCardinality = + container_get_cardinality(container, typecode); + + uint8_t newtypecode = typecode; + container_t *container2 = + container_add(container, val & 0xFFFF, typecode, &newtypecode); + if (container2 != container) { + container_free(container, typecode); + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); + result = true; + } else { + const int newCardinality = + container_get_cardinality(container, newtypecode); + + result = oldCardinality != newCardinality; + } + } else { + array_container_t *newac = array_container_create(); + container_t *container = + container_add(newac, val & 0xFFFF, ARRAY_CONTAINER_TYPE, &typecode); + // we could just assume that it stays an array container + ra_insert_new_key_value_at(&r->high_low_container, -i - 1, hb, + container, typecode); + result = true; + } + + return result; +} + +void roaring_bitmap_remove(roaring_bitmap_t *r, uint32_t val) { + const uint16_t hb = val >> 16; + const int i = ra_get_index(&r->high_low_container, hb); + uint8_t typecode; + if (i >= 0) { + ra_unshare_container_at_index(&r->high_low_container, (uint16_t)i); + container_t *container = ra_get_container_at_index( + &r->high_low_container, (uint16_t)i, &typecode); + uint8_t newtypecode = typecode; + container_t *container2 = + container_remove(container, val & 0xFFFF, typecode, &newtypecode); + if (container2 != container) { + container_free(container, typecode); + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); + } + if (container_get_cardinality(container2, newtypecode) != 0) { + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); + } else { + ra_remove_at_index_and_free(&r->high_low_container, i); + } + } +} + +bool roaring_bitmap_remove_checked(roaring_bitmap_t *r, uint32_t val) { + const uint16_t hb = val >> 16; + const int i = ra_get_index(&r->high_low_container, hb); + uint8_t typecode; + bool result = false; + if (i >= 0) { + ra_unshare_container_at_index(&r->high_low_container, (uint16_t)i); + container_t *container = ra_get_container_at_index( + &r->high_low_container, (uint16_t)i, &typecode); + + const int oldCardinality = + container_get_cardinality(container, typecode); + + uint8_t newtypecode = typecode; + container_t *container2 = + container_remove(container, val & 0xFFFF, typecode, &newtypecode); + if (container2 != container) { + container_free(container, typecode); + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); + } + + const int newCardinality = + container_get_cardinality(container2, newtypecode); + + if (newCardinality != 0) { + ra_set_container_at_index(&r->high_low_container, i, container2, + newtypecode); + } else { + ra_remove_at_index_and_free(&r->high_low_container, i); + } + + result = oldCardinality != newCardinality; + } + return result; +} + +void roaring_bitmap_remove_many(roaring_bitmap_t *r, size_t n_args, + const uint32_t *vals) { + if (n_args == 0 || r->high_low_container.size == 0) { + return; + } + int32_t pos = + -1; // position of the container used in the previous iteration + for (size_t i = 0; i < n_args; i++) { + uint16_t key = (uint16_t)(vals[i] >> 16); + if (pos < 0 || key != r->high_low_container.keys[pos]) { + pos = ra_get_index(&r->high_low_container, key); + } + if (pos >= 0) { + uint8_t new_typecode; + container_t *new_container; + new_container = container_remove( + r->high_low_container.containers[pos], vals[i] & 0xffff, + r->high_low_container.typecodes[pos], &new_typecode); + if (new_container != r->high_low_container.containers[pos]) { + container_free(r->high_low_container.containers[pos], + r->high_low_container.typecodes[pos]); + ra_replace_key_and_container_at_index(&r->high_low_container, + pos, key, new_container, + new_typecode); + } + if (!container_nonzero_cardinality(new_container, new_typecode)) { + container_free(new_container, new_typecode); + ra_remove_at_index(&r->high_low_container, pos); + pos = -1; + } + } + } +} + +// there should be some SIMD optimizations possible here +roaring_bitmap_t *roaring_bitmap_and(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + uint32_t neededcap = length1 > length2 ? length2 : length1; + roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(neededcap); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + + int pos1 = 0, pos2 = 0; + + while (pos1 < length1 && pos2 < length2) { + const uint16_t s1 = + ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + const uint16_t s2 = + ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + if (s1 == s2) { + uint8_t type1, type2; + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = container_and(c1, type1, c2, type2, &result_type); + + if (container_nonzero_cardinality(c, result_type)) { + ra_append(&answer->high_low_container, s1, c, result_type); + } else { + container_free(c, result_type); // otherwise: memory leak! + } + ++pos1; + ++pos2; + } else if (s1 < s2) { // s1 < s2 + pos1 = ra_advance_until(&x1->high_low_container, s2, pos1); + } else { // s1 > s2 + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + } + } + return answer; +} + +/** + * Compute the union of 'number' bitmaps. + */ +roaring_bitmap_t *roaring_bitmap_or_many(size_t number, + const roaring_bitmap_t **x) { + if (number == 0) { + return roaring_bitmap_create(); + } + if (number == 1) { + return roaring_bitmap_copy(x[0]); + } + roaring_bitmap_t *answer = + roaring_bitmap_lazy_or(x[0], x[1], LAZY_OR_BITSET_CONVERSION); + for (size_t i = 2; i < number; i++) { + roaring_bitmap_lazy_or_inplace(answer, x[i], LAZY_OR_BITSET_CONVERSION); + } + roaring_bitmap_repair_after_lazy(answer); + return answer; +} + +/** + * Compute the xor of 'number' bitmaps. + */ +roaring_bitmap_t *roaring_bitmap_xor_many(size_t number, + const roaring_bitmap_t **x) { + if (number == 0) { + return roaring_bitmap_create(); + } + if (number == 1) { + return roaring_bitmap_copy(x[0]); + } + roaring_bitmap_t *answer = roaring_bitmap_lazy_xor(x[0], x[1]); + for (size_t i = 2; i < number; i++) { + roaring_bitmap_lazy_xor_inplace(answer, x[i]); + } + roaring_bitmap_repair_after_lazy(answer); + return answer; +} + +// inplace and (modifies its first argument). +void roaring_bitmap_and_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + if (x1 == x2) return; + int pos1 = 0, pos2 = 0, intersection_size = 0; + const int length1 = ra_get_size(&x1->high_low_container); + const int length2 = ra_get_size(&x2->high_low_container); + + // any skipped-over or newly emptied containers in x1 + // have to be freed. + while (pos1 < length1 && pos2 < length2) { + const uint16_t s1 = + ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + const uint16_t s2 = + ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + if (s1 == s2) { + uint8_t type1, type2, result_type; + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + + // We do the computation "in place" only when c1 is not a shared + // container. Rationale: using a shared container safely with in + // place computation would require making a copy and then doing the + // computation in place which is likely less efficient than avoiding + // in place entirely and always generating a new container. + container_t *c = + (type1 == SHARED_CONTAINER_TYPE) + ? container_and(c1, type1, c2, type2, &result_type) + : container_iand(c1, type1, c2, type2, &result_type); + + if (c != c1) { // in this instance a new container was created, and + // we need to free the old one + container_free(c1, type1); + } + if (container_nonzero_cardinality(c, result_type)) { + ra_replace_key_and_container_at_index(&x1->high_low_container, + intersection_size, s1, c, + result_type); + intersection_size++; + } else { + container_free(c, result_type); + } + ++pos1; + ++pos2; + } else if (s1 < s2) { + pos1 = ra_advance_until_freeing(&x1->high_low_container, s2, pos1); + } else { // s1 > s2 + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + } + } + + // if we ended early because x2 ran out, then all remaining in x1 should be + // freed + while (pos1 < length1) { + container_free(x1->high_low_container.containers[pos1], + x1->high_low_container.typecodes[pos1]); + ++pos1; + } + + // all containers after this have either been copied or freed + ra_downsize(&x1->high_low_container, intersection_size); +} + +roaring_bitmap_t *roaring_bitmap_or(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + if (0 == length1) { + return roaring_bitmap_copy(x2); + } + if (0 == length2) { + return roaring_bitmap_copy(x1); + } + roaring_bitmap_t *answer = + roaring_bitmap_create_with_capacity(length1 + length2); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = container_or(c1, type1, c2, type2, &result_type); + + // since we assume that the initial containers are non-empty, the + // result here + // can only be non-empty + ra_append(&answer->high_low_container, s1, c, result_type); + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + // c1 = container_clone(c1, type1); + c1 = get_copy_of_container(c1, &type1, is_cow(x1)); + if (is_cow(x1)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c1, + type1); + } + ra_append(&answer->high_low_container, s1, c1, type1); + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + // c2 = container_clone(c2, type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_append(&answer->high_low_container, s2, c2, type2); + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + } + } + if (pos1 == length1) { + ra_append_copy_range(&answer->high_low_container, + &x2->high_low_container, pos2, length2, + is_cow(x2)); + } else if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); + } + return answer; +} + +// inplace or (modifies its first argument). +void roaring_bitmap_or_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; + + if (0 == length2) return; + + if (0 == length1) { + roaring_bitmap_overwrite(x1, x2); + return; + } + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + if (!container_is_full(c1, type1)) { + container_t *c2 = ra_get_container_at_index( + &x2->high_low_container, (uint16_t)pos2, &type2); + container_t *c = + (type1 == SHARED_CONTAINER_TYPE) + ? container_or(c1, type1, c2, type2, &result_type) + : container_ior(c1, type1, c2, type2, &result_type); + + if (c != c1) { // in this instance a new container was created, + // and we need to free the old one + container_free(c1, type1); + } + ra_set_container_at_index(&x1->high_low_container, pos1, c, + result_type); + } + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + + // container_t *c2_clone = container_clone(c2, type2); + ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, + type2); + pos1++; + length1++; + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + } + } + if (pos1 == length1) { + ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, + pos2, length2, is_cow(x2)); + } +} + +roaring_bitmap_t *roaring_bitmap_xor(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + if (0 == length1) { + return roaring_bitmap_copy(x2); + } + if (0 == length2) { + return roaring_bitmap_copy(x1); + } + roaring_bitmap_t *answer = + roaring_bitmap_create_with_capacity(length1 + length2); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = container_xor(c1, type1, c2, type2, &result_type); + + if (container_nonzero_cardinality(c, result_type)) { + ra_append(&answer->high_low_container, s1, c, result_type); + } else { + container_free(c, result_type); + } + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + c1 = get_copy_of_container(c1, &type1, is_cow(x1)); + if (is_cow(x1)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c1, + type1); + } + ra_append(&answer->high_low_container, s1, c1, type1); + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_append(&answer->high_low_container, s2, c2, type2); + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + } + } + if (pos1 == length1) { + ra_append_copy_range(&answer->high_low_container, + &x2->high_low_container, pos2, length2, + is_cow(x2)); + } else if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); + } + return answer; +} + +// inplace xor (modifies its first argument). + +void roaring_bitmap_xor_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + assert(x1 != x2); + uint8_t result_type = 0; + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; + + if (0 == length2) return; + + if (0 == length1) { + roaring_bitmap_overwrite(x1, x2); + return; + } + + // XOR can have new containers inserted from x2, but can also + // lose containers when x1 and x2 are nonempty and identical. + + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + + // We do the computation "in place" only when c1 is not a shared + // container. Rationale: using a shared container safely with in + // place computation would require making a copy and then doing the + // computation in place which is likely less efficient than avoiding + // in place entirely and always generating a new container. + + container_t *c; + if (type1 == SHARED_CONTAINER_TYPE) { + c = container_xor(c1, type1, c2, type2, &result_type); + shared_container_free(CAST_shared(c1)); // so release + } else { + c = container_ixor(c1, type1, c2, type2, &result_type); + } + + if (container_nonzero_cardinality(c, result_type)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c, + result_type); + ++pos1; + } else { + container_free(c, result_type); + ra_remove_at_index(&x1->high_low_container, pos1); + --length1; + } + + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + + ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, + type2); + pos1++; + length1++; + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + } + } + if (pos1 == length1) { + ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, + pos2, length2, is_cow(x2)); + } +} + +roaring_bitmap_t *roaring_bitmap_andnot(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + if (0 == length1) { + roaring_bitmap_t *empty_bitmap = roaring_bitmap_create(); + roaring_bitmap_set_copy_on_write(empty_bitmap, + is_cow(x1) || is_cow(x2)); + return empty_bitmap; + } + if (0 == length2) { + return roaring_bitmap_copy(x1); + } + roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(length1); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = 0; + uint16_t s2 = 0; + while (true) { + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = + container_andnot(c1, type1, c2, type2, &result_type); + + if (container_nonzero_cardinality(c, result_type)) { + ra_append(&answer->high_low_container, s1, c, result_type); + } else { + container_free(c, result_type); + } + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + } else if (s1 < s2) { // s1 < s2 + const int next_pos1 = + ra_advance_until(&x1->high_low_container, s2, pos1); + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, next_pos1, + is_cow(x1)); + // TODO : perhaps some of the copy_on_write should be based on + // answer rather than x1 (more stringent?). Many similar cases + pos1 = next_pos1; + if (pos1 == length1) break; + } else { // s1 > s2 + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + if (pos2 == length2) break; + } + } + if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); + } + return answer; +} + +// inplace andnot (modifies its first argument). + +void roaring_bitmap_andnot_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + assert(x1 != x2); + + uint8_t result_type = 0; + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; + int intersection_size = 0; + + if (0 == length2) return; + + if (0 == length1) { + roaring_bitmap_clear(x1); + return; + } + + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + + // We do the computation "in place" only when c1 is not a shared + // container. Rationale: using a shared container safely with in + // place computation would require making a copy and then doing the + // computation in place which is likely less efficient than avoiding + // in place entirely and always generating a new container. + + container_t *c; + if (type1 == SHARED_CONTAINER_TYPE) { + c = container_andnot(c1, type1, c2, type2, &result_type); + shared_container_free(CAST_shared(c1)); // release + } else { + c = container_iandnot(c1, type1, c2, type2, &result_type); + } + + if (container_nonzero_cardinality(c, result_type)) { + ra_replace_key_and_container_at_index(&x1->high_low_container, + intersection_size++, s1, + c, result_type); + } else { + container_free(c, result_type); + } + + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + if (pos1 != intersection_size) { + container_t *c1 = ra_get_container_at_index( + &x1->high_low_container, (uint16_t)pos1, &type1); + + ra_replace_key_and_container_at_index( + &x1->high_low_container, intersection_size, s1, c1, type1); + } + intersection_size++; + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + } + } + + if (pos1 < length1) { + // all containers between intersection_size and + // pos1 are junk. However, they have either been moved + // (thus still referenced) or involved in an iandnot + // that will clean up all containers that could not be reused. + // Thus we should not free the junk containers between + // intersection_size and pos1. + if (pos1 > intersection_size) { + // left slide of remaining items + ra_copy_range(&x1->high_low_container, pos1, length1, + intersection_size); + } + // else current placement is fine + intersection_size += (length1 - pos1); + } + ra_downsize(&x1->high_low_container, intersection_size); +} + +uint64_t roaring_bitmap_get_cardinality(const roaring_bitmap_t *r) { + const roaring_array_t *ra = &r->high_low_container; + + uint64_t card = 0; + for (int i = 0; i < ra->size; ++i) + card += container_get_cardinality(ra->containers[i], ra->typecodes[i]); + return card; +} + +uint64_t roaring_bitmap_range_cardinality(const roaring_bitmap_t *r, + uint64_t range_start, + uint64_t range_end) { + const roaring_array_t *ra = &r->high_low_container; + + if (range_end > UINT32_MAX) { + range_end = UINT32_MAX + UINT64_C(1); + } + if (range_start >= range_end) { + return 0; + } + range_end--; // make range_end inclusive + // now we have: 0 <= range_start <= range_end <= UINT32_MAX + + uint16_t minhb = (uint16_t)(range_start >> 16); + uint16_t maxhb = (uint16_t)(range_end >> 16); + + uint64_t card = 0; + + int i = ra_get_index(ra, minhb); + if (i >= 0) { + if (minhb == maxhb) { + card += container_rank(ra->containers[i], ra->typecodes[i], + range_end & 0xffff); + } else { + card += + container_get_cardinality(ra->containers[i], ra->typecodes[i]); + } + if ((range_start & 0xffff) != 0) { + card -= container_rank(ra->containers[i], ra->typecodes[i], + (range_start & 0xffff) - 1); + } + i++; + } else { + i = -i - 1; + } + + for (; i < ra->size; i++) { + uint16_t key = ra->keys[i]; + if (key < maxhb) { + card += + container_get_cardinality(ra->containers[i], ra->typecodes[i]); + } else if (key == maxhb) { + card += container_rank(ra->containers[i], ra->typecodes[i], + range_end & 0xffff); + break; + } else { + break; + } + } + + return card; +} + +bool roaring_bitmap_is_empty(const roaring_bitmap_t *r) { + return r->high_low_container.size == 0; +} + +void roaring_bitmap_to_uint32_array(const roaring_bitmap_t *r, uint32_t *ans) { + ra_to_uint32_array(&r->high_low_container, ans); +} + +bool roaring_bitmap_range_uint32_array(const roaring_bitmap_t *r, size_t offset, + size_t limit, uint32_t *ans) { + return ra_range_uint32_array(&r->high_low_container, offset, limit, ans); +} + +/** convert array and bitmap containers to run containers when it is more + * efficient; + * also convert from run containers when more space efficient. Returns + * true if the result has at least one run container. + */ +bool roaring_bitmap_run_optimize(roaring_bitmap_t *r) { + bool answer = false; + for (int i = 0; i < r->high_low_container.size; i++) { + uint8_t type_original, type_after; + ra_unshare_container_at_index( + &r->high_low_container, + (uint16_t)i); // TODO: this introduces extra cloning! + container_t *c = ra_get_container_at_index(&r->high_low_container, + (uint16_t)i, &type_original); + container_t *c1 = convert_run_optimize(c, type_original, &type_after); + if (type_after == RUN_CONTAINER_TYPE) { + answer = true; + } + ra_set_container_at_index(&r->high_low_container, i, c1, type_after); + } + return answer; +} + +size_t roaring_bitmap_shrink_to_fit(roaring_bitmap_t *r) { + size_t answer = 0; + for (int i = 0; i < r->high_low_container.size; i++) { + uint8_t type_original; + container_t *c = ra_get_container_at_index(&r->high_low_container, + (uint16_t)i, &type_original); + answer += container_shrink_to_fit(c, type_original); + } + answer += ra_shrink_to_fit(&r->high_low_container); + return answer; +} + +/** + * Remove run-length encoding even when it is more space efficient + * return whether a change was applied + */ +bool roaring_bitmap_remove_run_compression(roaring_bitmap_t *r) { + bool answer = false; + for (int i = 0; i < r->high_low_container.size; i++) { + uint8_t type_original, type_after; + container_t *c = ra_get_container_at_index(&r->high_low_container, + (uint16_t)i, &type_original); + if (get_container_type(c, type_original) == RUN_CONTAINER_TYPE) { + answer = true; + if (type_original == SHARED_CONTAINER_TYPE) { + run_container_t *truec = CAST_run(CAST_shared(c)->container); + int32_t card = run_container_cardinality(truec); + container_t *c1 = convert_to_bitset_or_array_container( + truec, card, &type_after); + shared_container_free(CAST_shared(c)); // frees run as needed + ra_set_container_at_index(&r->high_low_container, i, c1, + type_after); + + } else { + int32_t card = run_container_cardinality(CAST_run(c)); + container_t *c1 = convert_to_bitset_or_array_container( + CAST_run(c), card, &type_after); + run_container_free(CAST_run(c)); + ra_set_container_at_index(&r->high_low_container, i, c1, + type_after); + } + } + } + return answer; +} + +size_t roaring_bitmap_serialize(const roaring_bitmap_t *r, char *buf) { + size_t portablesize = roaring_bitmap_portable_size_in_bytes(r); + uint64_t cardinality = roaring_bitmap_get_cardinality(r); + uint64_t sizeasarray = cardinality * sizeof(uint32_t) + sizeof(uint32_t); + if (portablesize < sizeasarray) { + buf[0] = CROARING_SERIALIZATION_CONTAINER; + return roaring_bitmap_portable_serialize(r, buf + 1) + 1; + } else { + buf[0] = CROARING_SERIALIZATION_ARRAY_UINT32; + memcpy(buf + 1, &cardinality, sizeof(uint32_t)); + roaring_bitmap_to_uint32_array( + r, (uint32_t *)(buf + 1 + sizeof(uint32_t))); + return 1 + (size_t)sizeasarray; + } +} + +size_t roaring_bitmap_size_in_bytes(const roaring_bitmap_t *r) { + size_t portablesize = roaring_bitmap_portable_size_in_bytes(r); + uint64_t sizeasarray = + roaring_bitmap_get_cardinality(r) * sizeof(uint32_t) + sizeof(uint32_t); + return portablesize < sizeasarray ? portablesize + 1 + : (size_t)sizeasarray + 1; +} + +size_t roaring_bitmap_portable_size_in_bytes(const roaring_bitmap_t *r) { + return ra_portable_size_in_bytes(&r->high_low_container); +} + +roaring_bitmap_t *roaring_bitmap_portable_deserialize_safe(const char *buf, + size_t maxbytes) { + roaring_bitmap_t *ans = + (roaring_bitmap_t *)roaring_malloc(sizeof(roaring_bitmap_t)); + if (ans == NULL) { + return NULL; + } + size_t bytesread; + bool is_ok = ra_portable_deserialize(&ans->high_low_container, buf, + maxbytes, &bytesread); + if (!is_ok) { + roaring_free(ans); + return NULL; + } + roaring_bitmap_set_copy_on_write(ans, false); + if (!is_ok) { + roaring_free(ans); + return NULL; + } + return ans; +} + +roaring_bitmap_t *roaring_bitmap_portable_deserialize(const char *buf) { + return roaring_bitmap_portable_deserialize_safe(buf, SIZE_MAX); +} + +size_t roaring_bitmap_portable_deserialize_size(const char *buf, + size_t maxbytes) { + return ra_portable_deserialize_size(buf, maxbytes); +} + +size_t roaring_bitmap_portable_serialize(const roaring_bitmap_t *r, char *buf) { + return ra_portable_serialize(&r->high_low_container, buf); +} + +roaring_bitmap_t *roaring_bitmap_deserialize(const void *buf) { + const char *bufaschar = (const char *)buf; + if (bufaschar[0] == CROARING_SERIALIZATION_ARRAY_UINT32) { + /* This looks like a compressed set of uint32_t elements */ + uint32_t card; + + memcpy(&card, bufaschar + 1, sizeof(uint32_t)); + + const uint32_t *elems = + (const uint32_t *)(bufaschar + 1 + sizeof(uint32_t)); + + roaring_bitmap_t *bitmap = roaring_bitmap_create(); + if (bitmap == NULL) { + return NULL; + } + roaring_bulk_context_t context = {0}; + for (uint32_t i = 0; i < card; i++) { + // elems may not be aligned, read with memcpy + uint32_t elem; + memcpy(&elem, elems + i, sizeof(elem)); + roaring_bitmap_add_bulk(bitmap, &context, elem); + } + return bitmap; + + } else if (bufaschar[0] == CROARING_SERIALIZATION_CONTAINER) { + return roaring_bitmap_portable_deserialize(bufaschar + 1); + } else + return (NULL); +} + +roaring_bitmap_t *roaring_bitmap_deserialize_safe(const void *buf, + size_t maxbytes) { + if (maxbytes < 1) { + return NULL; + } + + const char *bufaschar = (const char *)buf; + if (bufaschar[0] == CROARING_SERIALIZATION_ARRAY_UINT32) { + if (maxbytes < 1 + sizeof(uint32_t)) { + return NULL; + } + + /* This looks like a compressed set of uint32_t elements */ + uint32_t card; + memcpy(&card, bufaschar + 1, sizeof(uint32_t)); + + // Check the buffer is big enough to contain card uint32_t elements + if (maxbytes < 1 + sizeof(uint32_t) + card * sizeof(uint32_t)) { + return NULL; + } + + const uint32_t *elems = + (const uint32_t *)(bufaschar + 1 + sizeof(uint32_t)); + + roaring_bitmap_t *bitmap = roaring_bitmap_create(); + if (bitmap == NULL) { + return NULL; + } + roaring_bulk_context_t context = {0}; + for (uint32_t i = 0; i < card; i++) { + // elems may not be aligned, read with memcpy + uint32_t elem; + memcpy(&elem, elems + i, sizeof(elem)); + roaring_bitmap_add_bulk(bitmap, &context, elem); + } + return bitmap; + + } else if (bufaschar[0] == CROARING_SERIALIZATION_CONTAINER) { + return roaring_bitmap_portable_deserialize_safe(bufaschar + 1, + maxbytes - 1); + } else + return (NULL); +} + +bool roaring_iterate(const roaring_bitmap_t *r, roaring_iterator iterator, + void *ptr) { + const roaring_array_t *ra = &r->high_low_container; + + for (int i = 0; i < ra->size; ++i) + if (!container_iterate(ra->containers[i], ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16, iterator, ptr)) { + return false; + } + return true; +} + +bool roaring_iterate64(const roaring_bitmap_t *r, roaring_iterator64 iterator, + uint64_t high_bits, void *ptr) { + const roaring_array_t *ra = &r->high_low_container; + + for (int i = 0; i < ra->size; ++i) + if (!container_iterate64(ra->containers[i], ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16, iterator, + high_bits, ptr)) { + return false; + } + return true; +} + +/**** + * begin roaring_uint32_iterator_t + *****/ + +/** + * Partially initializes the iterator. Leaves it in either state: + * 1. Invalid due to `has_value = false`, or + * 2. At a container, with the high bits set, `has_value = true`. + */ +CROARING_WARN_UNUSED static bool iter_new_container_partial_init( + roaring_uint32_iterator_t *newit) { + newit->current_value = 0; + if (newit->container_index >= newit->parent->high_low_container.size || + newit->container_index < 0) { + newit->current_value = UINT32_MAX; + return (newit->has_value = false); + } + newit->has_value = true; + // we precompute container, typecode and highbits so that successive + // iterators do not have to grab them from odd memory locations + // and have to worry about the (easily predicted) container_unwrap_shared + // call. + newit->container = + newit->parent->high_low_container.containers[newit->container_index]; + newit->typecode = + newit->parent->high_low_container.typecodes[newit->container_index]; + newit->highbits = + ((uint32_t) + newit->parent->high_low_container.keys[newit->container_index]) + << 16; + newit->container = + container_unwrap_shared(newit->container, &(newit->typecode)); + return true; +} + +/** + * Positions the iterator at the first value of the current container that the + * iterator points at, if available. + */ +CROARING_WARN_UNUSED static bool loadfirstvalue( + roaring_uint32_iterator_t *newit) { + if (iter_new_container_partial_init(newit)) { + uint16_t value = 0; + newit->container_it = + container_init_iterator(newit->container, newit->typecode, &value); + newit->current_value = newit->highbits | value; + } + return newit->has_value; +} + +/** + * Positions the iterator at the last value of the current container that the + * iterator points at, if available. + */ +CROARING_WARN_UNUSED static bool loadlastvalue( + roaring_uint32_iterator_t *newit) { + if (iter_new_container_partial_init(newit)) { + uint16_t value = 0; + newit->container_it = container_init_iterator_last( + newit->container, newit->typecode, &value); + newit->current_value = newit->highbits | value; + } + return newit->has_value; +} + +/** + * Positions the iterator at the smallest value that is larger than or equal to + * `val` within the current container that the iterator points at. Assumes such + * a value exists within the current container. + */ +CROARING_WARN_UNUSED static bool loadfirstvalue_largeorequal( + roaring_uint32_iterator_t *newit, uint32_t val) { + bool partial_init = iter_new_container_partial_init(newit); + assert(partial_init); + if (!partial_init) { + return false; + } + uint16_t value = 0; + newit->container_it = + container_init_iterator(newit->container, newit->typecode, &value); + bool found = container_iterator_lower_bound( + newit->container, newit->typecode, &newit->container_it, &value, + val & 0xFFFF); + assert(found); + if (!found) { + return false; + } + newit->current_value = newit->highbits | value; + return true; +} + +void roaring_iterator_init(const roaring_bitmap_t *r, + roaring_uint32_iterator_t *newit) { + newit->parent = r; + newit->container_index = 0; + newit->has_value = loadfirstvalue(newit); +} + +void roaring_iterator_init_last(const roaring_bitmap_t *r, + roaring_uint32_iterator_t *newit) { + newit->parent = r; + newit->container_index = newit->parent->high_low_container.size - 1; + newit->has_value = loadlastvalue(newit); +} + +roaring_uint32_iterator_t *roaring_iterator_create(const roaring_bitmap_t *r) { + roaring_uint32_iterator_t *newit = + (roaring_uint32_iterator_t *)roaring_malloc( + sizeof(roaring_uint32_iterator_t)); + if (newit == NULL) return NULL; + roaring_iterator_init(r, newit); + return newit; +} + +roaring_uint32_iterator_t *roaring_uint32_iterator_copy( + const roaring_uint32_iterator_t *it) { + roaring_uint32_iterator_t *newit = + (roaring_uint32_iterator_t *)roaring_malloc( + sizeof(roaring_uint32_iterator_t)); + memcpy(newit, it, sizeof(roaring_uint32_iterator_t)); + return newit; +} + +bool roaring_uint32_iterator_move_equalorlarger(roaring_uint32_iterator_t *it, + uint32_t val) { + uint16_t hb = val >> 16; + const int i = ra_get_index(&it->parent->high_low_container, hb); + if (i >= 0) { + uint32_t lowvalue = + container_maximum(it->parent->high_low_container.containers[i], + it->parent->high_low_container.typecodes[i]); + uint16_t lb = val & 0xFFFF; + if (lowvalue < lb) { + // will have to load first value of next container + it->container_index = i + 1; + } else { + // the value is necessarily within the range of the container + it->container_index = i; + it->has_value = loadfirstvalue_largeorequal(it, val); + return it->has_value; + } + } else { + // there is no matching, so we are going for the next container + it->container_index = -i - 1; + } + it->has_value = loadfirstvalue(it); + return it->has_value; +} + +bool roaring_uint32_iterator_advance(roaring_uint32_iterator_t *it) { + if (it->container_index >= it->parent->high_low_container.size) { + return (it->has_value = false); + } + if (it->container_index < 0) { + it->container_index = 0; + return (it->has_value = loadfirstvalue(it)); + } + uint16_t low16 = (uint16_t)it->current_value; + if (container_iterator_next(it->container, it->typecode, &it->container_it, + &low16)) { + it->current_value = it->highbits | low16; + return (it->has_value = true); + } + it->container_index++; + return (it->has_value = loadfirstvalue(it)); +} + +bool roaring_uint32_iterator_previous(roaring_uint32_iterator_t *it) { + if (it->container_index < 0) { + return (it->has_value = false); + } + if (it->container_index >= it->parent->high_low_container.size) { + it->container_index = it->parent->high_low_container.size - 1; + return (it->has_value = loadlastvalue(it)); + } + uint16_t low16 = (uint16_t)it->current_value; + if (container_iterator_prev(it->container, it->typecode, &it->container_it, + &low16)) { + it->current_value = it->highbits | low16; + return (it->has_value = true); + } + it->container_index--; + return (it->has_value = loadlastvalue(it)); +} + +uint32_t roaring_uint32_iterator_read(roaring_uint32_iterator_t *it, + uint32_t *buf, uint32_t count) { + uint32_t ret = 0; + while (it->has_value && ret < count) { + uint32_t consumed; + uint16_t low16 = (uint16_t)it->current_value; + bool has_value = container_iterator_read_into_uint32( + it->container, it->typecode, &it->container_it, it->highbits, buf, + count - ret, &consumed, &low16); + ret += consumed; + buf += consumed; + if (has_value) { + it->has_value = true; + it->current_value = it->highbits | low16; + assert(ret == count); + return ret; + } + it->container_index++; + it->has_value = loadfirstvalue(it); + } + return ret; +} + +void roaring_uint32_iterator_free(roaring_uint32_iterator_t *it) { + roaring_free(it); +} + +/**** + * end of roaring_uint32_iterator_t + *****/ + +bool roaring_bitmap_equals(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2) { + const roaring_array_t *ra1 = &r1->high_low_container; + const roaring_array_t *ra2 = &r2->high_low_container; + + if (ra1->size != ra2->size) { + return false; + } + for (int i = 0; i < ra1->size; ++i) { + if (ra1->keys[i] != ra2->keys[i]) { + return false; + } + } + for (int i = 0; i < ra1->size; ++i) { + bool areequal = container_equals(ra1->containers[i], ra1->typecodes[i], + ra2->containers[i], ra2->typecodes[i]); + if (!areequal) { + return false; + } + } + return true; +} + +bool roaring_bitmap_is_subset(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2) { + const roaring_array_t *ra1 = &r1->high_low_container; + const roaring_array_t *ra2 = &r2->high_low_container; + + const int length1 = ra1->size, length2 = ra2->size; + + int pos1 = 0, pos2 = 0; + + while (pos1 < length1 && pos2 < length2) { + const uint16_t s1 = ra_get_key_at_index(ra1, (uint16_t)pos1); + const uint16_t s2 = ra_get_key_at_index(ra2, (uint16_t)pos2); + + if (s1 == s2) { + uint8_t type1, type2; + container_t *c1 = + ra_get_container_at_index(ra1, (uint16_t)pos1, &type1); + container_t *c2 = + ra_get_container_at_index(ra2, (uint16_t)pos2, &type2); + if (!container_is_subset(c1, type1, c2, type2)) return false; + ++pos1; + ++pos2; + } else if (s1 < s2) { // s1 < s2 + return false; + } else { // s1 > s2 + pos2 = ra_advance_until(ra2, s1, pos2); + } + } + if (pos1 == length1) + return true; + else + return false; +} + +static void insert_flipped_container(roaring_array_t *ans_arr, + const roaring_array_t *x1_arr, uint16_t hb, + uint16_t lb_start, uint16_t lb_end) { + const int i = ra_get_index(x1_arr, hb); + const int j = ra_get_index(ans_arr, hb); + uint8_t ctype_in, ctype_out; + container_t *flipped_container = NULL; + if (i >= 0) { + container_t *container_to_flip = + ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); + flipped_container = + container_not_range(container_to_flip, ctype_in, (uint32_t)lb_start, + (uint32_t)(lb_end + 1), &ctype_out); + + if (container_get_cardinality(flipped_container, ctype_out)) + ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, + ctype_out); + else { + container_free(flipped_container, ctype_out); + } + } else { + flipped_container = container_range_of_ones( + (uint32_t)lb_start, (uint32_t)(lb_end + 1), &ctype_out); + ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, + ctype_out); + } +} + +static void inplace_flip_container(roaring_array_t *x1_arr, uint16_t hb, + uint16_t lb_start, uint16_t lb_end) { + const int i = ra_get_index(x1_arr, hb); + uint8_t ctype_in, ctype_out; + container_t *flipped_container = NULL; + if (i >= 0) { + container_t *container_to_flip = + ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); + flipped_container = container_inot_range( + container_to_flip, ctype_in, (uint32_t)lb_start, + (uint32_t)(lb_end + 1), &ctype_out); + // if a new container was created, the old one was already freed + if (container_get_cardinality(flipped_container, ctype_out)) { + ra_set_container_at_index(x1_arr, i, flipped_container, ctype_out); + } else { + container_free(flipped_container, ctype_out); + ra_remove_at_index(x1_arr, i); + } + + } else { + flipped_container = container_range_of_ones( + (uint32_t)lb_start, (uint32_t)(lb_end + 1), &ctype_out); + ra_insert_new_key_value_at(x1_arr, -i - 1, hb, flipped_container, + ctype_out); + } +} + +static void insert_fully_flipped_container(roaring_array_t *ans_arr, + const roaring_array_t *x1_arr, + uint16_t hb) { + const int i = ra_get_index(x1_arr, hb); + const int j = ra_get_index(ans_arr, hb); + uint8_t ctype_in, ctype_out; + container_t *flipped_container = NULL; + if (i >= 0) { + container_t *container_to_flip = + ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); + flipped_container = + container_not(container_to_flip, ctype_in, &ctype_out); + if (container_get_cardinality(flipped_container, ctype_out)) + ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, + ctype_out); + else { + container_free(flipped_container, ctype_out); + } + } else { + flipped_container = container_range_of_ones(0U, 0x10000U, &ctype_out); + ra_insert_new_key_value_at(ans_arr, -j - 1, hb, flipped_container, + ctype_out); + } +} + +static void inplace_fully_flip_container(roaring_array_t *x1_arr, uint16_t hb) { + const int i = ra_get_index(x1_arr, hb); + uint8_t ctype_in, ctype_out; + container_t *flipped_container = NULL; + if (i >= 0) { + container_t *container_to_flip = + ra_get_container_at_index(x1_arr, (uint16_t)i, &ctype_in); + flipped_container = + container_inot(container_to_flip, ctype_in, &ctype_out); + + if (container_get_cardinality(flipped_container, ctype_out)) { + ra_set_container_at_index(x1_arr, i, flipped_container, ctype_out); + } else { + container_free(flipped_container, ctype_out); + ra_remove_at_index(x1_arr, i); + } + + } else { + flipped_container = container_range_of_ones(0U, 0x10000U, &ctype_out); + ra_insert_new_key_value_at(x1_arr, -i - 1, hb, flipped_container, + ctype_out); + } +} + +roaring_bitmap_t *roaring_bitmap_flip(const roaring_bitmap_t *x1, + uint64_t range_start, + uint64_t range_end) { + if (range_start >= range_end) { + return roaring_bitmap_copy(x1); + } + if (range_end >= UINT64_C(0x100000000)) { + range_end = UINT64_C(0x100000000); + } + + roaring_bitmap_t *ans = roaring_bitmap_create(); + roaring_bitmap_set_copy_on_write(ans, is_cow(x1)); + + uint16_t hb_start = (uint16_t)(range_start >> 16); + const uint16_t lb_start = (uint16_t)range_start; // & 0xFFFF; + uint16_t hb_end = (uint16_t)((range_end - 1) >> 16); + const uint16_t lb_end = (uint16_t)(range_end - 1); // & 0xFFFF; + + ra_append_copies_until(&ans->high_low_container, &x1->high_low_container, + hb_start, is_cow(x1)); + if (hb_start == hb_end) { + insert_flipped_container(&ans->high_low_container, + &x1->high_low_container, hb_start, lb_start, + lb_end); + } else { + // start and end containers are distinct + if (lb_start > 0) { + // handle first (partial) container + insert_flipped_container(&ans->high_low_container, + &x1->high_low_container, hb_start, + lb_start, 0xFFFF); + ++hb_start; // for the full containers. Can't wrap. + } + + if (lb_end != 0xFFFF) --hb_end; // later we'll handle the partial block + + for (uint32_t hb = hb_start; hb <= hb_end; ++hb) { + insert_fully_flipped_container(&ans->high_low_container, + &x1->high_low_container, + (uint16_t)hb); + } + + // handle a partial final container + if (lb_end != 0xFFFF) { + insert_flipped_container(&ans->high_low_container, + &x1->high_low_container, hb_end + 1, 0, + lb_end); + ++hb_end; + } + } + ra_append_copies_after(&ans->high_low_container, &x1->high_low_container, + hb_end, is_cow(x1)); + return ans; +} + +void roaring_bitmap_flip_inplace(roaring_bitmap_t *x1, uint64_t range_start, + uint64_t range_end) { + if (range_start >= range_end) { + return; // empty range + } + if (range_end >= UINT64_C(0x100000000)) { + range_end = UINT64_C(0x100000000); + } + + uint16_t hb_start = (uint16_t)(range_start >> 16); + const uint16_t lb_start = (uint16_t)range_start; + uint16_t hb_end = (uint16_t)((range_end - 1) >> 16); + const uint16_t lb_end = (uint16_t)(range_end - 1); + + if (hb_start == hb_end) { + inplace_flip_container(&x1->high_low_container, hb_start, lb_start, + lb_end); + } else { + // start and end containers are distinct + if (lb_start > 0) { + // handle first (partial) container + inplace_flip_container(&x1->high_low_container, hb_start, lb_start, + 0xFFFF); + ++hb_start; // for the full containers. Can't wrap. + } + + if (lb_end != 0xFFFF) --hb_end; + + for (uint32_t hb = hb_start; hb <= hb_end; ++hb) { + inplace_fully_flip_container(&x1->high_low_container, (uint16_t)hb); + } + // handle a partial final container + if (lb_end != 0xFFFF) { + inplace_flip_container(&x1->high_low_container, hb_end + 1, 0, + lb_end); + ++hb_end; + } + } +} + +static void offset_append_with_merge(roaring_array_t *ra, int k, container_t *c, + uint8_t t) { + int size = ra_get_size(ra); + if (size == 0 || ra_get_key_at_index(ra, (uint16_t)(size - 1)) != k) { + // No merge. + ra_append(ra, (uint16_t)k, c, t); + return; + } + + uint8_t last_t, new_t; + container_t *last_c, *new_c; + + // NOTE: we don't need to unwrap here, since we added last_c ourselves + // we have the certainty it's not a shared container. + // The same applies to c, as it's the result of calling container_offset. + last_c = ra_get_container_at_index(ra, (uint16_t)(size - 1), &last_t); + new_c = container_ior(last_c, last_t, c, t, &new_t); + + ra_set_container_at_index(ra, size - 1, new_c, new_t); + + // Comparison of pointers of different origin is UB (or so claim some + // compiler makers), so we compare their bit representation only. + if ((uintptr_t)last_c != (uintptr_t)new_c) { + container_free(last_c, last_t); + } + container_free(c, t); +} + +// roaring_bitmap_add_offset adds the value 'offset' to each and every value in +// a bitmap, generating a new bitmap in the process. If offset + element is +// outside of the range [0,2^32), that the element will be dropped. +// We need "offset" to be 64 bits because we want to support values +// between -0xFFFFFFFF up to +0xFFFFFFFF. +roaring_bitmap_t *roaring_bitmap_add_offset(const roaring_bitmap_t *bm, + int64_t offset) { + roaring_bitmap_t *answer; + roaring_array_t *ans_ra; + int64_t container_offset; + uint16_t in_offset; + + const roaring_array_t *bm_ra = &bm->high_low_container; + int length = bm_ra->size; + + if (offset == 0) { + return roaring_bitmap_copy(bm); + } + + container_offset = offset >> 16; + in_offset = (uint16_t)(offset - container_offset * (1 << 16)); + + answer = roaring_bitmap_create(); + bool cow = is_cow(bm); + roaring_bitmap_set_copy_on_write(answer, cow); + + ans_ra = &answer->high_low_container; + + if (in_offset == 0) { + ans_ra = &answer->high_low_container; + + for (int i = 0, j = 0; i < length; ++i) { + int64_t key = ra_get_key_at_index(bm_ra, (uint16_t)i); + key += container_offset; + + if (key < 0 || key >= (1 << 16)) { + continue; + } + ra_append_copy(ans_ra, bm_ra, (uint16_t)i, cow); + ans_ra->keys[j++] = (uint16_t)key; + } + return answer; + } + + uint8_t t; + const container_t *c; + container_t *lo, *hi, **lo_ptr, **hi_ptr; + int64_t k; + + for (int i = 0; i < length; ++i) { + lo = hi = NULL; + lo_ptr = hi_ptr = NULL; + + k = ra_get_key_at_index(bm_ra, (uint16_t)i) + container_offset; + if (k >= 0 && k < (1 << 16)) { + lo_ptr = &lo; + } + if (k + 1 >= 0 && k + 1 < (1 << 16)) { + hi_ptr = &hi; + } + if (lo_ptr == NULL && hi_ptr == NULL) { + continue; + } + c = ra_get_container_at_index(bm_ra, (uint16_t)i, &t); + c = container_unwrap_shared(c, &t); + + container_add_offset(c, t, lo_ptr, hi_ptr, in_offset); + if (lo != NULL) { + offset_append_with_merge(ans_ra, (int)k, lo, t); + } + if (hi != NULL) { + ra_append(ans_ra, (uint16_t)(k + 1), hi, t); + } + // the `lo` and `hi` container type always keep same as container `c`. + // in the case of `container_add_offset` on bitset container, `lo` and + // `hi` may has small cardinality, they must be repaired to array + // container. + } + + roaring_bitmap_repair_after_lazy(answer); // do required type conversions. + return answer; +} + +roaring_bitmap_t *roaring_bitmap_lazy_or(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2, + const bool bitsetconversion) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + if (0 == length1) { + return roaring_bitmap_copy(x2); + } + if (0 == length2) { + return roaring_bitmap_copy(x1); + } + roaring_bitmap_t *answer = + roaring_bitmap_create_with_capacity(length1 + length2); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c; + if (bitsetconversion && + (get_container_type(c1, type1) != BITSET_CONTAINER_TYPE) && + (get_container_type(c2, type2) != BITSET_CONTAINER_TYPE)) { + container_t *newc1 = + container_mutable_unwrap_shared(c1, &type1); + newc1 = container_to_bitset(newc1, type1); + type1 = BITSET_CONTAINER_TYPE; + c = container_lazy_ior(newc1, type1, c2, type2, &result_type); + if (c != newc1) { // should not happen + container_free(newc1, type1); + } + } else { + c = container_lazy_or(c1, type1, c2, type2, &result_type); + } + // since we assume that the initial containers are non-empty, + // the + // result here + // can only be non-empty + ra_append(&answer->high_low_container, s1, c, result_type); + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + c1 = get_copy_of_container(c1, &type1, is_cow(x1)); + if (is_cow(x1)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c1, + type1); + } + ra_append(&answer->high_low_container, s1, c1, type1); + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_append(&answer->high_low_container, s2, c2, type2); + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + } + } + if (pos1 == length1) { + ra_append_copy_range(&answer->high_low_container, + &x2->high_low_container, pos2, length2, + is_cow(x2)); + } else if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); + } + return answer; +} + +void roaring_bitmap_lazy_or_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2, + const bool bitsetconversion) { + uint8_t result_type = 0; + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; + + if (0 == length2) return; + + if (0 == length1) { + roaring_bitmap_overwrite(x1, x2); + return; + } + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + if (!container_is_full(c1, type1)) { + if ((bitsetconversion == false) || + (get_container_type(c1, type1) == BITSET_CONTAINER_TYPE)) { + c1 = get_writable_copy_if_shared(c1, &type1); + } else { + // convert to bitset + container_t *old_c1 = c1; + uint8_t old_type1 = type1; + c1 = container_mutable_unwrap_shared(c1, &type1); + c1 = container_to_bitset(c1, type1); + container_free(old_c1, old_type1); + type1 = BITSET_CONTAINER_TYPE; + } + + container_t *c2 = ra_get_container_at_index( + &x2->high_low_container, (uint16_t)pos2, &type2); + container_t *c = + container_lazy_ior(c1, type1, c2, type2, &result_type); + + if (c != c1) { // in this instance a new container was created, + // and we need to free the old one + container_free(c1, type1); + } + + ra_set_container_at_index(&x1->high_low_container, pos1, c, + result_type); + } + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + // container_t *c2_clone = container_clone(c2, type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, + type2); + pos1++; + length1++; + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + } + } + if (pos1 == length1) { + ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, + pos2, length2, is_cow(x2)); + } +} + +roaring_bitmap_t *roaring_bitmap_lazy_xor(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + if (0 == length1) { + return roaring_bitmap_copy(x2); + } + if (0 == length2) { + return roaring_bitmap_copy(x1); + } + roaring_bitmap_t *answer = + roaring_bitmap_create_with_capacity(length1 + length2); + roaring_bitmap_set_copy_on_write(answer, is_cow(x1) || is_cow(x2)); + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + container_t *c = + container_lazy_xor(c1, type1, c2, type2, &result_type); + + if (container_nonzero_cardinality(c, result_type)) { + ra_append(&answer->high_low_container, s1, c, result_type); + } else { + container_free(c, result_type); + } + + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + c1 = get_copy_of_container(c1, &type1, is_cow(x1)); + if (is_cow(x1)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c1, + type1); + } + ra_append(&answer->high_low_container, s1, c1, type1); + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_append(&answer->high_low_container, s2, c2, type2); + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + } + } + if (pos1 == length1) { + ra_append_copy_range(&answer->high_low_container, + &x2->high_low_container, pos2, length2, + is_cow(x2)); + } else if (pos2 == length2) { + ra_append_copy_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1, + is_cow(x1)); + } + return answer; +} + +void roaring_bitmap_lazy_xor_inplace(roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + assert(x1 != x2); + uint8_t result_type = 0; + int length1 = x1->high_low_container.size; + const int length2 = x2->high_low_container.size; + + if (0 == length2) return; + + if (0 == length1) { + roaring_bitmap_overwrite(x1, x2); + return; + } + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + + // We do the computation "in place" only when c1 is not a shared + // container. Rationale: using a shared container safely with in + // place computation would require making a copy and then doing the + // computation in place which is likely less efficient than avoiding + // in place entirely and always generating a new container. + + container_t *c; + if (type1 == SHARED_CONTAINER_TYPE) { + c = container_lazy_xor(c1, type1, c2, type2, &result_type); + shared_container_free(CAST_shared(c1)); // release + } else { + c = container_lazy_ixor(c1, type1, c2, type2, &result_type); + } + + if (container_nonzero_cardinality(c, result_type)) { + ra_set_container_at_index(&x1->high_low_container, pos1, c, + result_type); + ++pos1; + } else { + container_free(c, result_type); + ra_remove_at_index(&x1->high_low_container, pos1); + --length1; + } + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + // container_t *c2_clone = container_clone(c2, type2); + c2 = get_copy_of_container(c2, &type2, is_cow(x2)); + if (is_cow(x2)) { + ra_set_container_at_index(&x2->high_low_container, pos2, c2, + type2); + } + ra_insert_new_key_value_at(&x1->high_low_container, pos1, s2, c2, + type2); + pos1++; + length1++; + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + } + } + if (pos1 == length1) { + ra_append_copy_range(&x1->high_low_container, &x2->high_low_container, + pos2, length2, is_cow(x2)); + } +} + +void roaring_bitmap_repair_after_lazy(roaring_bitmap_t *r) { + roaring_array_t *ra = &r->high_low_container; + + for (int i = 0; i < ra->size; ++i) { + const uint8_t old_type = ra->typecodes[i]; + container_t *old_c = ra->containers[i]; + uint8_t new_type = old_type; + container_t *new_c = container_repair_after_lazy(old_c, &new_type); + ra->containers[i] = new_c; + ra->typecodes[i] = new_type; + } +} + +/** + * roaring_bitmap_rank returns the number of integers that are smaller or equal + * to x. + */ +uint64_t roaring_bitmap_rank(const roaring_bitmap_t *bm, uint32_t x) { + uint64_t size = 0; + uint32_t xhigh = x >> 16; + for (int i = 0; i < bm->high_low_container.size; i++) { + uint32_t key = bm->high_low_container.keys[i]; + if (xhigh > key) { + size += + container_get_cardinality(bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i]); + } else if (xhigh == key) { + return size + container_rank(bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i], + x & 0xFFFF); + } else { + return size; + } + } + return size; +} +void roaring_bitmap_rank_many(const roaring_bitmap_t *bm, const uint32_t *begin, + const uint32_t *end, uint64_t *ans) { + uint64_t size = 0; + + int i = 0; + const uint32_t *iter = begin; + while (i < bm->high_low_container.size && iter != end) { + uint32_t x = *iter; + uint32_t xhigh = x >> 16; + uint32_t key = bm->high_low_container.keys[i]; + if (xhigh > key) { + size += + container_get_cardinality(bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i]); + i++; + } else if (xhigh == key) { + uint32_t consumed = container_rank_many( + bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i], size, iter, end, ans); + iter += consumed; + ans += consumed; + } else { + *(ans++) = size; + iter++; + } + } +} + +/** + * roaring_bitmap_get_index returns the index of x, if not exsist return -1. + */ +int64_t roaring_bitmap_get_index(const roaring_bitmap_t *bm, uint32_t x) { + int64_t index = 0; + const uint16_t xhigh = x >> 16; + int32_t high_idx = ra_get_index(&bm->high_low_container, xhigh); + if (high_idx < 0) return -1; + + for (int i = 0; i < bm->high_low_container.size; i++) { + uint32_t key = bm->high_low_container.keys[i]; + if (xhigh > key) { + index += + container_get_cardinality(bm->high_low_container.containers[i], + bm->high_low_container.typecodes[i]); + } else if (xhigh == key) { + int32_t low_idx = container_get_index( + bm->high_low_container.containers[high_idx], + bm->high_low_container.typecodes[high_idx], x & 0xFFFF); + if (low_idx < 0) return -1; + return index + low_idx; + } else { + return -1; + } + } + return index; +} + +/** + * roaring_bitmap_smallest returns the smallest value in the set. + * Returns UINT32_MAX if the set is empty. + */ +uint32_t roaring_bitmap_minimum(const roaring_bitmap_t *bm) { + if (bm->high_low_container.size > 0) { + container_t *c = bm->high_low_container.containers[0]; + uint8_t type = bm->high_low_container.typecodes[0]; + uint32_t key = bm->high_low_container.keys[0]; + uint32_t lowvalue = container_minimum(c, type); + return lowvalue | (key << 16); + } + return UINT32_MAX; +} + +/** + * roaring_bitmap_smallest returns the greatest value in the set. + * Returns 0 if the set is empty. + */ +uint32_t roaring_bitmap_maximum(const roaring_bitmap_t *bm) { + if (bm->high_low_container.size > 0) { + container_t *container = + bm->high_low_container.containers[bm->high_low_container.size - 1]; + uint8_t typecode = + bm->high_low_container.typecodes[bm->high_low_container.size - 1]; + uint32_t key = + bm->high_low_container.keys[bm->high_low_container.size - 1]; + uint32_t lowvalue = container_maximum(container, typecode); + return lowvalue | (key << 16); + } + return 0; +} + +bool roaring_bitmap_select(const roaring_bitmap_t *bm, uint32_t rank, + uint32_t *element) { + container_t *container; + uint8_t typecode; + uint16_t key; + uint32_t start_rank = 0; + int i = 0; + bool valid = false; + while (!valid && i < bm->high_low_container.size) { + container = bm->high_low_container.containers[i]; + typecode = bm->high_low_container.typecodes[i]; + valid = + container_select(container, typecode, &start_rank, rank, element); + i++; + } + + if (valid) { + key = bm->high_low_container.keys[i - 1]; + *element |= (((uint32_t)key) << 16); // w/o cast, key promotes signed + return true; + } else + return false; +} + +bool roaring_bitmap_intersect(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + uint64_t answer = 0; + int pos1 = 0, pos2 = 0; + + while (pos1 < length1 && pos2 < length2) { + const uint16_t s1 = + ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + const uint16_t s2 = + ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + if (s1 == s2) { + uint8_t type1, type2; + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + if (container_intersect(c1, type1, c2, type2)) return true; + ++pos1; + ++pos2; + } else if (s1 < s2) { // s1 < s2 + pos1 = ra_advance_until(&x1->high_low_container, s2, pos1); + } else { // s1 > s2 + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + } + } + return answer != 0; +} + +bool roaring_bitmap_intersect_with_range(const roaring_bitmap_t *bm, uint64_t x, + uint64_t y) { + if (x >= y) { + // Empty range. + return false; + } + roaring_uint32_iterator_t it; + roaring_iterator_init(bm, &it); + if (!roaring_uint32_iterator_move_equalorlarger(&it, (uint32_t)x)) { + // No values above x. + return false; + } + if (it.current_value >= y) { + // No values below y. + return false; + } + return true; +} + +uint64_t roaring_bitmap_and_cardinality(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const int length1 = x1->high_low_container.size, + length2 = x2->high_low_container.size; + uint64_t answer = 0; + int pos1 = 0, pos2 = 0; + while (pos1 < length1 && pos2 < length2) { + const uint16_t s1 = + ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + const uint16_t s2 = + ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + if (s1 == s2) { + uint8_t type1, type2; + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + answer += container_and_cardinality(c1, type1, c2, type2); + ++pos1; + ++pos2; + } else if (s1 < s2) { // s1 < s2 + pos1 = ra_advance_until(&x1->high_low_container, s2, pos1); + } else { // s1 > s2 + pos2 = ra_advance_until(&x2->high_low_container, s1, pos2); + } + } + return answer; +} + +double roaring_bitmap_jaccard_index(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const uint64_t c1 = roaring_bitmap_get_cardinality(x1); + const uint64_t c2 = roaring_bitmap_get_cardinality(x2); + const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); + return (double)inter / (double)(c1 + c2 - inter); +} + +uint64_t roaring_bitmap_or_cardinality(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const uint64_t c1 = roaring_bitmap_get_cardinality(x1); + const uint64_t c2 = roaring_bitmap_get_cardinality(x2); + const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); + return c1 + c2 - inter; +} + +uint64_t roaring_bitmap_andnot_cardinality(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const uint64_t c1 = roaring_bitmap_get_cardinality(x1); + const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); + return c1 - inter; +} + +uint64_t roaring_bitmap_xor_cardinality(const roaring_bitmap_t *x1, + const roaring_bitmap_t *x2) { + const uint64_t c1 = roaring_bitmap_get_cardinality(x1); + const uint64_t c2 = roaring_bitmap_get_cardinality(x2); + const uint64_t inter = roaring_bitmap_and_cardinality(x1, x2); + return c1 + c2 - 2 * inter; +} + +bool roaring_bitmap_contains(const roaring_bitmap_t *r, uint32_t val) { + const uint16_t hb = val >> 16; + /* + * the next function call involves a binary search and lots of branching. + */ + int32_t i = ra_get_index(&r->high_low_container, hb); + if (i < 0) return false; + + uint8_t typecode; + // next call ought to be cheap + container_t *container = ra_get_container_at_index(&r->high_low_container, + (uint16_t)i, &typecode); + // rest might be a tad expensive, possibly involving another round of binary + // search + return container_contains(container, val & 0xFFFF, typecode); +} + +/** + * Check whether a range of values from range_start (included) to range_end + * (excluded) is present + */ +bool roaring_bitmap_contains_range(const roaring_bitmap_t *r, + uint64_t range_start, uint64_t range_end) { + if (range_end >= UINT64_C(0x100000000)) { + range_end = UINT64_C(0x100000000); + } + if (range_start >= range_end) + return true; // empty range are always contained! + if (range_end - range_start == 1) + return roaring_bitmap_contains(r, (uint32_t)range_start); + uint16_t hb_rs = (uint16_t)(range_start >> 16); + uint16_t hb_re = (uint16_t)((range_end - 1) >> 16); + const int32_t span = hb_re - hb_rs; + const int32_t hlc_sz = ra_get_size(&r->high_low_container); + if (hlc_sz < span + 1) { + return false; + } + int32_t is = ra_get_index(&r->high_low_container, hb_rs); + int32_t ie = ra_get_index(&r->high_low_container, hb_re); + if ((ie < 0) || (is < 0) || ((ie - is) != span) || ie >= hlc_sz) { + return false; + } + const uint32_t lb_rs = range_start & 0xFFFF; + const uint32_t lb_re = ((range_end - 1) & 0xFFFF) + 1; + uint8_t type; + container_t *c = + ra_get_container_at_index(&r->high_low_container, (uint16_t)is, &type); + if (hb_rs == hb_re) { + return container_contains_range(c, lb_rs, lb_re, type); + } + if (!container_contains_range(c, lb_rs, 1 << 16, type)) { + return false; + } + c = ra_get_container_at_index(&r->high_low_container, (uint16_t)ie, &type); + if (!container_contains_range(c, 0, lb_re, type)) { + return false; + } + for (int32_t i = is + 1; i < ie; ++i) { + c = ra_get_container_at_index(&r->high_low_container, (uint16_t)i, + &type); + if (!container_is_full(c, type)) { + return false; + } + } + return true; +} + +bool roaring_bitmap_is_strict_subset(const roaring_bitmap_t *r1, + const roaring_bitmap_t *r2) { + return (roaring_bitmap_get_cardinality(r2) > + roaring_bitmap_get_cardinality(r1) && + roaring_bitmap_is_subset(r1, r2)); +} + +/* + * FROZEN SERIALIZATION FORMAT DESCRIPTION + * + * -- (beginning must be aligned by 32 bytes) -- + * <bitset_data> uint64_t[BITSET_CONTAINER_SIZE_IN_WORDS * + * num_bitset_containers] <run_data> rle16_t[total number of rle elements in + * all run containers] <array_data> uint16_t[total number of array elements in + * all array containers] <keys> uint16_t[num_containers] <counts> + * uint16_t[num_containers] <typecodes> uint8_t[num_containers] <header> + * uint32_t + * + * <header> is a 4-byte value which is a bit union of FROZEN_COOKIE (15 bits) + * and the number of containers (17 bits). + * + * <counts> stores number of elements for every container. + * Its meaning depends on container type. + * For array and bitset containers, this value is the container cardinality + * minus one. For run container, it is the number of rle_t elements (n_runs). + * + * <bitset_data>,<array_data>,<run_data> are flat arrays of elements of + * all containers of respective type. + * + * <*_data> and <keys> are kept close together because they are not accessed + * during deserilization. This may reduce IO in case of large mmaped bitmaps. + * All members have their native alignments during deserilization except + * <header>, which is not guaranteed to be aligned by 4 bytes. + */ + +size_t roaring_bitmap_frozen_size_in_bytes(const roaring_bitmap_t *rb) { + const roaring_array_t *ra = &rb->high_low_container; + size_t num_bytes = 0; + for (int32_t i = 0; i < ra->size; i++) { + switch (ra->typecodes[i]) { + case BITSET_CONTAINER_TYPE: { + num_bytes += BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + break; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(ra->containers[i]); + num_bytes += rc->n_runs * sizeof(rle16_t); + break; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = + const_CAST_array(ra->containers[i]); + num_bytes += ac->cardinality * sizeof(uint16_t); + break; + } + default: + roaring_unreachable; + } + } + num_bytes += (2 + 2 + 1) * ra->size; // keys, counts, typecodes + num_bytes += 4; // header + return num_bytes; +} + +inline static void *arena_alloc(char **arena, size_t num_bytes) { + char *res = *arena; + *arena += num_bytes; + return res; +} + +void roaring_bitmap_frozen_serialize(const roaring_bitmap_t *rb, char *buf) { + /* + * Note: we do not require user to supply a specifically aligned buffer. + * Thus we have to use memcpy() everywhere. + */ + + const roaring_array_t *ra = &rb->high_low_container; + + size_t bitset_zone_size = 0; + size_t run_zone_size = 0; + size_t array_zone_size = 0; + for (int32_t i = 0; i < ra->size; i++) { + switch (ra->typecodes[i]) { + case BITSET_CONTAINER_TYPE: { + bitset_zone_size += + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + break; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(ra->containers[i]); + run_zone_size += rc->n_runs * sizeof(rle16_t); + break; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = + const_CAST_array(ra->containers[i]); + array_zone_size += ac->cardinality * sizeof(uint16_t); + break; + } + default: + roaring_unreachable; + } + } + + uint64_t *bitset_zone = (uint64_t *)arena_alloc(&buf, bitset_zone_size); + rle16_t *run_zone = (rle16_t *)arena_alloc(&buf, run_zone_size); + uint16_t *array_zone = (uint16_t *)arena_alloc(&buf, array_zone_size); + uint16_t *key_zone = (uint16_t *)arena_alloc(&buf, 2 * ra->size); + uint16_t *count_zone = (uint16_t *)arena_alloc(&buf, 2 * ra->size); + uint8_t *typecode_zone = (uint8_t *)arena_alloc(&buf, ra->size); + uint32_t *header_zone = (uint32_t *)arena_alloc(&buf, 4); + + for (int32_t i = 0; i < ra->size; i++) { + uint16_t count; + switch (ra->typecodes[i]) { + case BITSET_CONTAINER_TYPE: { + const bitset_container_t *bc = + const_CAST_bitset(ra->containers[i]); + memcpy(bitset_zone, bc->words, + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t)); + bitset_zone += BITSET_CONTAINER_SIZE_IN_WORDS; + if (bc->cardinality != BITSET_UNKNOWN_CARDINALITY) { + count = (uint16_t)(bc->cardinality - 1); + } else { + count = + (uint16_t)(bitset_container_compute_cardinality(bc) - + 1); + } + break; + } + case RUN_CONTAINER_TYPE: { + const run_container_t *rc = const_CAST_run(ra->containers[i]); + size_t num_bytes = rc->n_runs * sizeof(rle16_t); + memcpy(run_zone, rc->runs, num_bytes); + run_zone += rc->n_runs; + count = (uint16_t)rc->n_runs; + break; + } + case ARRAY_CONTAINER_TYPE: { + const array_container_t *ac = + const_CAST_array(ra->containers[i]); + size_t num_bytes = ac->cardinality * sizeof(uint16_t); + memcpy(array_zone, ac->array, num_bytes); + array_zone += ac->cardinality; + count = (uint16_t)(ac->cardinality - 1); + break; + } + default: + roaring_unreachable; + } + memcpy(&count_zone[i], &count, 2); + } + memcpy(key_zone, ra->keys, ra->size * sizeof(uint16_t)); + memcpy(typecode_zone, ra->typecodes, ra->size * sizeof(uint8_t)); + uint32_t header = ((uint32_t)ra->size << 15) | FROZEN_COOKIE; + memcpy(header_zone, &header, 4); +} + +const roaring_bitmap_t *roaring_bitmap_frozen_view(const char *buf, + size_t length) { + if ((uintptr_t)buf % 32 != 0) { + return NULL; + } + + // cookie and num_containers + if (length < 4) { + return NULL; + } + uint32_t header; + memcpy(&header, buf + length - 4, 4); // header may be misaligned + if ((header & 0x7FFF) != FROZEN_COOKIE) { + return NULL; + } + int32_t num_containers = (header >> 15); + + // typecodes, counts and keys + if (length < 4 + (size_t)num_containers * (1 + 2 + 2)) { + return NULL; + } + uint16_t *keys = (uint16_t *)(buf + length - 4 - num_containers * 5); + uint16_t *counts = (uint16_t *)(buf + length - 4 - num_containers * 3); + uint8_t *typecodes = (uint8_t *)(buf + length - 4 - num_containers * 1); + + // {bitset,array,run}_zone + int32_t num_bitset_containers = 0; + int32_t num_run_containers = 0; + int32_t num_array_containers = 0; + size_t bitset_zone_size = 0; + size_t run_zone_size = 0; + size_t array_zone_size = 0; + for (int32_t i = 0; i < num_containers; i++) { + switch (typecodes[i]) { + case BITSET_CONTAINER_TYPE: + num_bitset_containers++; + bitset_zone_size += + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + break; + case RUN_CONTAINER_TYPE: + num_run_containers++; + run_zone_size += counts[i] * sizeof(rle16_t); + break; + case ARRAY_CONTAINER_TYPE: + num_array_containers++; + array_zone_size += (counts[i] + UINT32_C(1)) * sizeof(uint16_t); + break; + default: + return NULL; + } + } + if (length != bitset_zone_size + run_zone_size + array_zone_size + + 5 * num_containers + 4) { + return NULL; + } + uint64_t *bitset_zone = (uint64_t *)(buf); + rle16_t *run_zone = (rle16_t *)(buf + bitset_zone_size); + uint16_t *array_zone = (uint16_t *)(buf + bitset_zone_size + run_zone_size); + + size_t alloc_size = 0; + alloc_size += sizeof(roaring_bitmap_t); + alloc_size += num_containers * sizeof(container_t *); + alloc_size += num_bitset_containers * sizeof(bitset_container_t); + alloc_size += num_run_containers * sizeof(run_container_t); + alloc_size += num_array_containers * sizeof(array_container_t); + + char *arena = (char *)roaring_malloc(alloc_size); + if (arena == NULL) { + return NULL; + } + + roaring_bitmap_t *rb = + (roaring_bitmap_t *)arena_alloc(&arena, sizeof(roaring_bitmap_t)); + rb->high_low_container.flags = ROARING_FLAG_FROZEN; + rb->high_low_container.allocation_size = num_containers; + rb->high_low_container.size = num_containers; + rb->high_low_container.keys = (uint16_t *)keys; + rb->high_low_container.typecodes = (uint8_t *)typecodes; + rb->high_low_container.containers = (container_t **)arena_alloc( + &arena, sizeof(container_t *) * num_containers); + // Ensure offset of high_low_container.containers is known distance used in + // C++ wrapper. sizeof(roaring_bitmap_t) is used as it is the size of the + // only allocation that precedes high_low_container.containers. If this is + // changed (new allocation or changed order), this offset will also need to + // be changed in the C++ wrapper. + assert(rb == + (roaring_bitmap_t *)((char *)rb->high_low_container.containers - + sizeof(roaring_bitmap_t))); + for (int32_t i = 0; i < num_containers; i++) { + switch (typecodes[i]) { + case BITSET_CONTAINER_TYPE: { + bitset_container_t *bitset = (bitset_container_t *)arena_alloc( + &arena, sizeof(bitset_container_t)); + bitset->words = bitset_zone; + bitset->cardinality = counts[i] + UINT32_C(1); + rb->high_low_container.containers[i] = bitset; + bitset_zone += BITSET_CONTAINER_SIZE_IN_WORDS; + break; + } + case RUN_CONTAINER_TYPE: { + run_container_t *run = (run_container_t *)arena_alloc( + &arena, sizeof(run_container_t)); + run->capacity = counts[i]; + run->n_runs = counts[i]; + run->runs = run_zone; + rb->high_low_container.containers[i] = run; + run_zone += run->n_runs; + break; + } + case ARRAY_CONTAINER_TYPE: { + array_container_t *array = (array_container_t *)arena_alloc( + &arena, sizeof(array_container_t)); + array->capacity = counts[i] + UINT32_C(1); + array->cardinality = counts[i] + UINT32_C(1); + array->array = array_zone; + rb->high_low_container.containers[i] = array; + array_zone += counts[i] + UINT32_C(1); + break; + } + default: + roaring_free(arena); + return NULL; + } + } + + return rb; +} + +ALLOW_UNALIGNED +roaring_bitmap_t *roaring_bitmap_portable_deserialize_frozen(const char *buf) { + char *start_of_buf = (char *)buf; + uint32_t cookie; + int32_t num_containers; + uint16_t *descriptive_headers; + uint32_t *offset_headers = NULL; + const char *run_flag_bitset = NULL; + bool hasrun = false; + + // deserialize cookie + memcpy(&cookie, buf, sizeof(uint32_t)); + buf += sizeof(uint32_t); + if (cookie == SERIAL_COOKIE_NO_RUNCONTAINER) { + memcpy(&num_containers, buf, sizeof(int32_t)); + buf += sizeof(int32_t); + descriptive_headers = (uint16_t *)buf; + buf += num_containers * 2 * sizeof(uint16_t); + offset_headers = (uint32_t *)buf; + buf += num_containers * sizeof(uint32_t); + } else if ((cookie & 0xFFFF) == SERIAL_COOKIE) { + num_containers = (cookie >> 16) + 1; + hasrun = true; + int32_t run_flag_bitset_size = (num_containers + 7) / 8; + run_flag_bitset = buf; + buf += run_flag_bitset_size; + descriptive_headers = (uint16_t *)buf; + buf += num_containers * 2 * sizeof(uint16_t); + if (num_containers >= NO_OFFSET_THRESHOLD) { + offset_headers = (uint32_t *)buf; + buf += num_containers * sizeof(uint32_t); + } + } else { + return NULL; + } + + // calculate total size for allocation + int32_t num_bitset_containers = 0; + int32_t num_run_containers = 0; + int32_t num_array_containers = 0; + + for (int32_t i = 0; i < num_containers; i++) { + uint16_t tmp; + memcpy(&tmp, descriptive_headers + 2 * i + 1, sizeof(tmp)); + uint32_t cardinality = tmp + 1; + bool isbitmap = (cardinality > DEFAULT_MAX_SIZE); + bool isrun = false; + if (hasrun) { + if ((run_flag_bitset[i / 8] & (1 << (i % 8))) != 0) { + isbitmap = false; + isrun = true; + } + } + + if (isbitmap) { + num_bitset_containers++; + } else if (isrun) { + num_run_containers++; + } else { + num_array_containers++; + } + } + + size_t alloc_size = 0; + alloc_size += sizeof(roaring_bitmap_t); + alloc_size += num_containers * sizeof(container_t *); + alloc_size += num_bitset_containers * sizeof(bitset_container_t); + alloc_size += num_run_containers * sizeof(run_container_t); + alloc_size += num_array_containers * sizeof(array_container_t); + alloc_size += num_containers * sizeof(uint16_t); // keys + alloc_size += num_containers * sizeof(uint8_t); // typecodes + + // allocate bitmap and construct containers + char *arena = (char *)roaring_malloc(alloc_size); + if (arena == NULL) { + return NULL; + } + + roaring_bitmap_t *rb = + (roaring_bitmap_t *)arena_alloc(&arena, sizeof(roaring_bitmap_t)); + rb->high_low_container.flags = ROARING_FLAG_FROZEN; + rb->high_low_container.allocation_size = num_containers; + rb->high_low_container.size = num_containers; + rb->high_low_container.containers = (container_t **)arena_alloc( + &arena, sizeof(container_t *) * num_containers); + + uint16_t *keys = + (uint16_t *)arena_alloc(&arena, num_containers * sizeof(uint16_t)); + uint8_t *typecodes = + (uint8_t *)arena_alloc(&arena, num_containers * sizeof(uint8_t)); + + rb->high_low_container.keys = keys; + rb->high_low_container.typecodes = typecodes; + + for (int32_t i = 0; i < num_containers; i++) { + uint16_t tmp; + memcpy(&tmp, descriptive_headers + 2 * i + 1, sizeof(tmp)); + int32_t cardinality = tmp + 1; + bool isbitmap = (cardinality > DEFAULT_MAX_SIZE); + bool isrun = false; + if (hasrun) { + if ((run_flag_bitset[i / 8] & (1 << (i % 8))) != 0) { + isbitmap = false; + isrun = true; + } + } + + keys[i] = descriptive_headers[2 * i]; + + if (isbitmap) { + typecodes[i] = BITSET_CONTAINER_TYPE; + bitset_container_t *c = (bitset_container_t *)arena_alloc( + &arena, sizeof(bitset_container_t)); + c->cardinality = cardinality; + if (offset_headers != NULL) { + c->words = (uint64_t *)(start_of_buf + offset_headers[i]); + } else { + c->words = (uint64_t *)buf; + buf += BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + } + rb->high_low_container.containers[i] = c; + } else if (isrun) { + typecodes[i] = RUN_CONTAINER_TYPE; + run_container_t *c = + (run_container_t *)arena_alloc(&arena, sizeof(run_container_t)); + c->capacity = cardinality; + uint16_t n_runs; + if (offset_headers != NULL) { + memcpy(&n_runs, start_of_buf + offset_headers[i], + sizeof(uint16_t)); + c->n_runs = n_runs; + c->runs = (rle16_t *)(start_of_buf + offset_headers[i] + + sizeof(uint16_t)); + } else { + memcpy(&n_runs, buf, sizeof(uint16_t)); + c->n_runs = n_runs; + buf += sizeof(uint16_t); + c->runs = (rle16_t *)buf; + buf += c->n_runs * sizeof(rle16_t); + } + rb->high_low_container.containers[i] = c; + } else { + typecodes[i] = ARRAY_CONTAINER_TYPE; + array_container_t *c = (array_container_t *)arena_alloc( + &arena, sizeof(array_container_t)); + c->cardinality = cardinality; + c->capacity = cardinality; + if (offset_headers != NULL) { + c->array = (uint16_t *)(start_of_buf + offset_headers[i]); + } else { + c->array = (uint16_t *)buf; + buf += cardinality * sizeof(uint16_t); + } + rb->high_low_container.containers[i] = c; + } + } + + return rb; +} + +bool roaring_bitmap_to_bitset(const roaring_bitmap_t *r, bitset_t *bitset) { + uint32_t max_value = roaring_bitmap_maximum(r); + size_t new_array_size = (size_t)(((uint64_t)max_value + 63) / 64); + bool resize_ok = bitset_resize(bitset, new_array_size, true); + if (!resize_ok) { + return false; + } + const roaring_array_t *ra = &r->high_low_container; + for (int i = 0; i < ra->size; ++i) { + uint64_t *words = bitset->array + (ra->keys[i] << 10); + uint8_t type = ra->typecodes[i]; + const container_t *c = ra->containers[i]; + if (type == SHARED_CONTAINER_TYPE) { + c = container_unwrap_shared(c, &type); + } + switch (type) { + case BITSET_CONTAINER_TYPE: { + size_t max_word_index = new_array_size - (ra->keys[i] << 10); + if (max_word_index > 1024) { + max_word_index = 1024; + } + const bitset_container_t *src = const_CAST_bitset(c); + memcpy(words, src->words, max_word_index * sizeof(uint64_t)); + } break; + case ARRAY_CONTAINER_TYPE: { + const array_container_t *src = const_CAST_array(c); + bitset_set_list(words, src->array, src->cardinality); + } break; + case RUN_CONTAINER_TYPE: { + const run_container_t *src = const_CAST_run(c); + for (int32_t rlepos = 0; rlepos < src->n_runs; ++rlepos) { + rle16_t rle = src->runs[rlepos]; + bitset_set_lenrange(words, rle.value, rle.length); + } + } break; + default: + roaring_unreachable; + } + } + return true; +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { +#endif diff --git a/contrib/libs/croaring/src/roaring64.c b/contrib/libs/croaring/src/roaring64.c new file mode 100644 index 00000000000..5f9f941bdc4 --- /dev/null +++ b/contrib/libs/croaring/src/roaring64.c @@ -0,0 +1,2091 @@ +#include <assert.h> +#include <stdarg.h> +#include <stdint.h> +#include <string.h> + +#include <roaring/art/art.h> +#include <roaring/portability.h> +#include <roaring/roaring64.h> + +// For serialization / deserialization +#include <roaring/roaring.h> +#include <roaring/roaring_array.h> +// containers.h last to avoid conflict with ROARING_CONTAINER_T. +#include <roaring/containers/containers.h> + +#ifdef __cplusplus +using namespace ::roaring::internal; + +extern "C" { +namespace roaring { +namespace api { +#endif + +// TODO: Copy on write. +// TODO: Error on failed allocation. + +typedef struct roaring64_bitmap_s { + art_t art; + uint8_t flags; +} roaring64_bitmap_t; + +// Leaf type of the ART used to keep the high 48 bits of each entry. +typedef struct roaring64_leaf_s { + art_val_t _pad; + uint8_t typecode; + container_t *container; +} roaring64_leaf_t; + +// Alias to make it easier to work with, since it's an internal-only type +// anyway. +typedef struct roaring64_leaf_s leaf_t; + +// Iterator struct to hold iteration state. +typedef struct roaring64_iterator_s { + const roaring64_bitmap_t *parent; + art_iterator_t art_it; + roaring_container_iterator_t container_it; + uint64_t high48; // Key that art_it points to. + + uint64_t value; + bool has_value; + + // If has_value is false, then the iterator is saturated. This field + // indicates the direction of saturation. If true, there are no more values + // in the forward direction. If false, there are no more values in the + // backward direction. + bool saturated_forward; +} roaring64_iterator_t; + +// Splits the given uint64 key into high 48 bit and low 16 bit components. +// Expects high48_out to be of length ART_KEY_BYTES. +static inline uint16_t split_key(uint64_t key, uint8_t high48_out[]) { + uint64_t tmp = croaring_htobe64(key); + memcpy(high48_out, (uint8_t *)(&tmp), ART_KEY_BYTES); + return (uint16_t)key; +} + +// Recombines the high 48 bit and low 16 bit components into a uint64 key. +// Expects high48_out to be of length ART_KEY_BYTES. +static inline uint64_t combine_key(const uint8_t high48[], uint16_t low16) { + uint64_t result = 0; + memcpy((uint8_t *)(&result), high48, ART_KEY_BYTES); + return croaring_be64toh(result) | low16; +} + +static inline uint64_t minimum(uint64_t a, uint64_t b) { + return (a < b) ? a : b; +} + +static inline leaf_t *create_leaf(container_t *container, uint8_t typecode) { + leaf_t *leaf = (leaf_t *)roaring_malloc(sizeof(leaf_t)); + leaf->container = container; + leaf->typecode = typecode; + return leaf; +} + +static inline leaf_t *copy_leaf_container(const leaf_t *leaf) { + leaf_t *result_leaf = (leaf_t *)roaring_malloc(sizeof(leaf_t)); + result_leaf->typecode = leaf->typecode; + // get_copy_of_container modifies the typecode passed in. + result_leaf->container = get_copy_of_container( + leaf->container, &result_leaf->typecode, /*copy_on_write=*/false); + return result_leaf; +} + +static inline void free_leaf(leaf_t *leaf) { roaring_free(leaf); } + +static inline int compare_high48(art_key_chunk_t key1[], + art_key_chunk_t key2[]) { + return art_compare_keys(key1, key2); +} + +static inline bool roaring64_iterator_init_at_leaf_first( + roaring64_iterator_t *it) { + it->high48 = combine_key(it->art_it.key, 0); + leaf_t *leaf = (leaf_t *)it->art_it.value; + uint16_t low16 = 0; + it->container_it = + container_init_iterator(leaf->container, leaf->typecode, &low16); + it->value = it->high48 | low16; + return (it->has_value = true); +} + +static inline bool roaring64_iterator_init_at_leaf_last( + roaring64_iterator_t *it) { + it->high48 = combine_key(it->art_it.key, 0); + leaf_t *leaf = (leaf_t *)it->art_it.value; + uint16_t low16 = 0; + it->container_it = + container_init_iterator_last(leaf->container, leaf->typecode, &low16); + it->value = it->high48 | low16; + return (it->has_value = true); +} + +static inline roaring64_iterator_t *roaring64_iterator_init_at( + const roaring64_bitmap_t *r, roaring64_iterator_t *it, bool first) { + it->parent = r; + it->art_it = art_init_iterator(&r->art, first); + it->has_value = it->art_it.value != NULL; + if (it->has_value) { + if (first) { + roaring64_iterator_init_at_leaf_first(it); + } else { + roaring64_iterator_init_at_leaf_last(it); + } + } else { + it->saturated_forward = first; + } + return it; +} + +roaring64_bitmap_t *roaring64_bitmap_create(void) { + roaring64_bitmap_t *r = + (roaring64_bitmap_t *)roaring_malloc(sizeof(roaring64_bitmap_t)); + r->art.root = NULL; + r->flags = 0; + return r; +} + +void roaring64_bitmap_free(roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + container_free(leaf->container, leaf->typecode); + free_leaf(leaf); + art_iterator_next(&it); + } + art_free(&r->art); + roaring_free(r); +} + +roaring64_bitmap_t *roaring64_bitmap_copy(const roaring64_bitmap_t *r) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); + + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + uint8_t result_typecode = leaf->typecode; + container_t *result_container = get_copy_of_container( + leaf->container, &result_typecode, /*copy_on_write=*/false); + leaf_t *result_leaf = create_leaf(result_container, result_typecode); + art_insert(&result->art, it.key, (art_val_t *)result_leaf); + art_iterator_next(&it); + } + return result; +} + +roaring64_bitmap_t *roaring64_bitmap_from_range(uint64_t min, uint64_t max, + uint64_t step) { + if (step == 0 || max <= min) { + return NULL; + } + roaring64_bitmap_t *r = roaring64_bitmap_create(); + if (step >= (1 << 16)) { + // Only one value per container. + for (uint64_t value = min; value < max; value += step) { + roaring64_bitmap_add(r, value); + if (value > UINT64_MAX - step) { + break; + } + } + return r; + } + do { + uint64_t high_bits = min & 0xFFFFFFFFFFFF0000; + uint16_t container_min = min & 0xFFFF; + uint32_t container_max = (uint32_t)minimum(max - high_bits, 1 << 16); + + uint8_t typecode; + container_t *container = container_from_range( + &typecode, container_min, container_max, (uint16_t)step); + + uint8_t high48[ART_KEY_BYTES]; + split_key(min, high48); + leaf_t *leaf = create_leaf(container, typecode); + art_insert(&r->art, high48, (art_val_t *)leaf); + + uint64_t gap = container_max - container_min + step - 1; + uint64_t increment = gap - (gap % step); + if (min > UINT64_MAX - increment) { + break; + } + min += increment; + } while (min < max); + return r; +} + +roaring64_bitmap_t *roaring64_bitmap_of_ptr(size_t n_args, + const uint64_t *vals) { + roaring64_bitmap_t *r = roaring64_bitmap_create(); + roaring64_bitmap_add_many(r, n_args, vals); + return r; +} + +roaring64_bitmap_t *roaring64_bitmap_of(size_t n_args, ...) { + roaring64_bitmap_t *r = roaring64_bitmap_create(); + roaring64_bulk_context_t context = {0}; + va_list ap; + va_start(ap, n_args); + for (size_t i = 0; i < n_args; i++) { + uint64_t val = va_arg(ap, uint64_t); + roaring64_bitmap_add_bulk(r, &context, val); + } + va_end(ap); + return r; +} + +static inline leaf_t *containerptr_roaring64_bitmap_add(roaring64_bitmap_t *r, + uint8_t *high48, + uint16_t low16, + leaf_t *leaf) { + if (leaf != NULL) { + uint8_t typecode2; + container_t *container2 = + container_add(leaf->container, low16, leaf->typecode, &typecode2); + if (container2 != leaf->container) { + container_free(leaf->container, leaf->typecode); + leaf->container = container2; + leaf->typecode = typecode2; + } + return leaf; + } else { + array_container_t *ac = array_container_create(); + uint8_t typecode; + container_t *container = + container_add(ac, low16, ARRAY_CONTAINER_TYPE, &typecode); + assert(ac == container); + leaf = create_leaf(container, typecode); + art_insert(&r->art, high48, (art_val_t *)leaf); + return leaf; + } +} + +void roaring64_bitmap_add(roaring64_bitmap_t *r, uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + containerptr_roaring64_bitmap_add(r, high48, low16, leaf); +} + +bool roaring64_bitmap_add_checked(roaring64_bitmap_t *r, uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + + int old_cardinality = 0; + if (leaf != NULL) { + old_cardinality = + container_get_cardinality(leaf->container, leaf->typecode); + } + leaf = containerptr_roaring64_bitmap_add(r, high48, low16, leaf); + int new_cardinality = + container_get_cardinality(leaf->container, leaf->typecode); + return old_cardinality != new_cardinality; +} + +void roaring64_bitmap_add_bulk(roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, + uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + if (context->leaf != NULL && + compare_high48(context->high_bytes, high48) == 0) { + // We're at a container with the correct high bits. + uint8_t typecode2; + container_t *container2 = + container_add(context->leaf->container, low16, + context->leaf->typecode, &typecode2); + if (container2 != context->leaf->container) { + container_free(context->leaf->container, context->leaf->typecode); + context->leaf->container = container2; + context->leaf->typecode = typecode2; + } + } else { + // We're not positioned anywhere yet or the high bits of the key + // differ. + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + context->leaf = + containerptr_roaring64_bitmap_add(r, high48, low16, leaf); + memcpy(context->high_bytes, high48, ART_KEY_BYTES); + } +} + +void roaring64_bitmap_add_many(roaring64_bitmap_t *r, size_t n_args, + const uint64_t *vals) { + if (n_args == 0) { + return; + } + const uint64_t *end = vals + n_args; + roaring64_bulk_context_t context = {0}; + for (const uint64_t *current_val = vals; current_val != end; + current_val++) { + roaring64_bitmap_add_bulk(r, &context, *current_val); + } +} + +static inline void add_range_closed_at(art_t *art, uint8_t *high48, + uint16_t min, uint16_t max) { + leaf_t *leaf = (leaf_t *)art_find(art, high48); + if (leaf != NULL) { + uint8_t typecode2; + container_t *container2 = container_add_range( + leaf->container, leaf->typecode, min, max, &typecode2); + if (container2 != leaf->container) { + container_free(leaf->container, leaf->typecode); + leaf->container = container2; + leaf->typecode = typecode2; + } + return; + } + uint8_t typecode; + // container_add_range is inclusive, but `container_range_of_ones` is + // exclusive. + container_t *container = container_range_of_ones(min, max + 1, &typecode); + leaf = create_leaf(container, typecode); + art_insert(art, high48, (art_val_t *)leaf); +} + +void roaring64_bitmap_add_range(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min >= max) { + return; + } + roaring64_bitmap_add_range_closed(r, min, max - 1); +} + +void roaring64_bitmap_add_range_closed(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min > max) { + return; + } + + art_t *art = &r->art; + uint8_t min_high48[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48); + uint8_t max_high48[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48); + if (compare_high48(min_high48, max_high48) == 0) { + // Only populate range within one container. + add_range_closed_at(art, min_high48, min_low16, max_low16); + return; + } + + // Populate a range across containers. Fill intermediate containers + // entirely. + add_range_closed_at(art, min_high48, min_low16, 0xffff); + uint64_t min_high_bits = min >> 16; + uint64_t max_high_bits = max >> 16; + for (uint64_t current = min_high_bits + 1; current < max_high_bits; + ++current) { + uint8_t current_high48[ART_KEY_BYTES]; + split_key(current << 16, current_high48); + add_range_closed_at(art, current_high48, 0, 0xffff); + } + add_range_closed_at(art, max_high48, 0, max_low16); +} + +bool roaring64_bitmap_contains(const roaring64_bitmap_t *r, uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + if (leaf != NULL) { + return container_contains(leaf->container, low16, leaf->typecode); + } + return false; +} + +bool roaring64_bitmap_contains_range(const roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min >= max) { + return true; + } + + uint8_t min_high48[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48); + uint8_t max_high48[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48); + uint64_t max_high48_bits = (max - 1) & 0xFFFFFFFFFFFF0000; // Inclusive + + art_iterator_t it = art_lower_bound(&r->art, min_high48); + if (it.value == NULL || combine_key(it.key, 0) > min) { + return false; + } + uint64_t prev_high48_bits = min & 0xFFFFFFFFFFFF0000; + while (it.value != NULL) { + uint64_t current_high48_bits = combine_key(it.key, 0); + if (current_high48_bits > max_high48_bits) { + // We've passed the end of the range with all containers containing + // the range. + return true; + } + if (current_high48_bits - prev_high48_bits > 0x10000) { + // There is a gap in the iterator that falls in the range. + return false; + } + + leaf_t *leaf = (leaf_t *)it.value; + uint32_t container_min = 0; + if (compare_high48(it.key, min_high48) == 0) { + container_min = min_low16; + } + uint32_t container_max = 0xFFFF + 1; // Exclusive + if (compare_high48(it.key, max_high48) == 0) { + container_max = max_low16; + } + + // For the first and last containers we use container_contains_range, + // for the intermediate containers we can use container_is_full. + if (container_min == 0 && container_max == 0xFFFF + 1) { + if (!container_is_full(leaf->container, leaf->typecode)) { + return false; + } + } else if (!container_contains_range(leaf->container, container_min, + container_max, leaf->typecode)) { + return false; + } + prev_high48_bits = current_high48_bits; + art_iterator_next(&it); + } + return prev_high48_bits == max_high48_bits; +} + +bool roaring64_bitmap_contains_bulk(const roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, + uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + + if (context->leaf == NULL || context->high_bytes != high48) { + // We're not positioned anywhere yet or the high bits of the key + // differ. + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + if (leaf == NULL) { + return false; + } + context->leaf = leaf; + memcpy(context->high_bytes, high48, ART_KEY_BYTES); + } + return container_contains(context->leaf->container, low16, + context->leaf->typecode); +} + +bool roaring64_bitmap_select(const roaring64_bitmap_t *r, uint64_t rank, + uint64_t *element) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + uint64_t start_rank = 0; + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + uint64_t cardinality = + container_get_cardinality(leaf->container, leaf->typecode); + if (start_rank + cardinality > rank) { + uint32_t uint32_start = 0; + uint32_t uint32_rank = rank - start_rank; + uint32_t uint32_element = 0; + if (container_select(leaf->container, leaf->typecode, &uint32_start, + uint32_rank, &uint32_element)) { + *element = combine_key(it.key, (uint16_t)uint32_element); + return true; + } + return false; + } + start_rank += cardinality; + art_iterator_next(&it); + } + return false; +} + +uint64_t roaring64_bitmap_rank(const roaring64_bitmap_t *r, uint64_t val) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + uint64_t rank = 0; + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + int compare_result = compare_high48(it.key, high48); + if (compare_result < 0) { + rank += container_get_cardinality(leaf->container, leaf->typecode); + } else if (compare_result == 0) { + return rank + + container_rank(leaf->container, leaf->typecode, low16); + } else { + return rank; + } + art_iterator_next(&it); + } + return rank; +} + +bool roaring64_bitmap_get_index(const roaring64_bitmap_t *r, uint64_t val, + uint64_t *out_index) { + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + uint64_t index = 0; + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + int compare_result = compare_high48(it.key, high48); + if (compare_result < 0) { + index += container_get_cardinality(leaf->container, leaf->typecode); + } else if (compare_result == 0) { + int index16 = + container_get_index(leaf->container, leaf->typecode, low16); + if (index16 < 0) { + return false; + } + *out_index = index + index16; + return true; + } else { + return false; + } + art_iterator_next(&it); + } + return false; +} + +static inline leaf_t *containerptr_roaring64_bitmap_remove( + roaring64_bitmap_t *r, uint8_t *high48, uint16_t low16, leaf_t *leaf) { + if (leaf == NULL) { + return NULL; + } + + container_t *container = leaf->container; + uint8_t typecode = leaf->typecode; + uint8_t typecode2; + container_t *container2 = + container_remove(container, low16, typecode, &typecode2); + if (container2 != container) { + container_free(container, typecode); + leaf->container = container2; + leaf->typecode = typecode2; + } + if (!container_nonzero_cardinality(container2, typecode2)) { + container_free(container2, typecode2); + leaf = (leaf_t *)art_erase(&r->art, high48); + if (leaf != NULL) { + free_leaf(leaf); + } + return NULL; + } + return leaf; +} + +void roaring64_bitmap_remove(roaring64_bitmap_t *r, uint64_t val) { + art_t *art = &r->art; + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + + leaf_t *leaf = (leaf_t *)art_find(art, high48); + containerptr_roaring64_bitmap_remove(r, high48, low16, leaf); +} + +bool roaring64_bitmap_remove_checked(roaring64_bitmap_t *r, uint64_t val) { + art_t *art = &r->art; + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + leaf_t *leaf = (leaf_t *)art_find(art, high48); + + if (leaf == NULL) { + return false; + } + int old_cardinality = + container_get_cardinality(leaf->container, leaf->typecode); + leaf = containerptr_roaring64_bitmap_remove(r, high48, low16, leaf); + if (leaf == NULL) { + return true; + } + int new_cardinality = + container_get_cardinality(leaf->container, leaf->typecode); + return new_cardinality != old_cardinality; +} + +void roaring64_bitmap_remove_bulk(roaring64_bitmap_t *r, + roaring64_bulk_context_t *context, + uint64_t val) { + art_t *art = &r->art; + uint8_t high48[ART_KEY_BYTES]; + uint16_t low16 = split_key(val, high48); + if (context->leaf != NULL && + compare_high48(context->high_bytes, high48) == 0) { + // We're at a container with the correct high bits. + uint8_t typecode2; + container_t *container2 = + container_remove(context->leaf->container, low16, + context->leaf->typecode, &typecode2); + if (container2 != context->leaf->container) { + container_free(context->leaf->container, context->leaf->typecode); + context->leaf->container = container2; + context->leaf->typecode = typecode2; + } + if (!container_nonzero_cardinality(container2, typecode2)) { + leaf_t *leaf = (leaf_t *)art_erase(art, high48); + container_free(container2, typecode2); + free_leaf(leaf); + } + } else { + // We're not positioned anywhere yet or the high bits of the key + // differ. + leaf_t *leaf = (leaf_t *)art_find(art, high48); + context->leaf = + containerptr_roaring64_bitmap_remove(r, high48, low16, leaf); + memcpy(context->high_bytes, high48, ART_KEY_BYTES); + } +} + +void roaring64_bitmap_remove_many(roaring64_bitmap_t *r, size_t n_args, + const uint64_t *vals) { + if (n_args == 0) { + return; + } + const uint64_t *end = vals + n_args; + roaring64_bulk_context_t context = {0}; + for (const uint64_t *current_val = vals; current_val != end; + current_val++) { + roaring64_bitmap_remove_bulk(r, &context, *current_val); + } +} + +static inline void remove_range_closed_at(art_t *art, uint8_t *high48, + uint16_t min, uint16_t max) { + leaf_t *leaf = (leaf_t *)art_find(art, high48); + if (leaf == NULL) { + return; + } + uint8_t typecode2; + container_t *container2 = container_remove_range( + leaf->container, leaf->typecode, min, max, &typecode2); + if (container2 != leaf->container) { + container_free(leaf->container, leaf->typecode); + if (container2 != NULL) { + leaf->container = container2; + leaf->typecode = typecode2; + } else { + art_erase(art, high48); + free_leaf(leaf); + } + } +} + +void roaring64_bitmap_remove_range(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min >= max) { + return; + } + roaring64_bitmap_remove_range_closed(r, min, max - 1); +} + +void roaring64_bitmap_remove_range_closed(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min > max) { + return; + } + + art_t *art = &r->art; + uint8_t min_high48[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48); + uint8_t max_high48[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48); + if (compare_high48(min_high48, max_high48) == 0) { + // Only remove a range within one container. + remove_range_closed_at(art, min_high48, min_low16, max_low16); + return; + } + + // Remove a range across containers. Remove intermediate containers + // entirely. + remove_range_closed_at(art, min_high48, min_low16, 0xffff); + + art_iterator_t it = art_upper_bound(art, min_high48); + while (it.value != NULL && art_compare_keys(it.key, max_high48) < 0) { + leaf_t *leaf = (leaf_t *)art_iterator_erase(art, &it); + container_free(leaf->container, leaf->typecode); + free_leaf(leaf); + } + remove_range_closed_at(art, max_high48, 0, max_low16); +} + +uint64_t roaring64_bitmap_get_cardinality(const roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + uint64_t cardinality = 0; + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + cardinality += + container_get_cardinality(leaf->container, leaf->typecode); + art_iterator_next(&it); + } + return cardinality; +} + +uint64_t roaring64_bitmap_range_cardinality(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max) { + if (min >= max) { + return 0; + } + max--; // A closed range is easier to work with. + + uint64_t cardinality = 0; + uint8_t min_high48[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48); + uint8_t max_high48[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48); + + art_iterator_t it = art_lower_bound(&r->art, min_high48); + while (it.value != NULL) { + int max_compare_result = compare_high48(it.key, max_high48); + if (max_compare_result > 0) { + // We're outside the range. + break; + } + + leaf_t *leaf = (leaf_t *)it.value; + if (max_compare_result == 0) { + // We're at the max high key, add only the range up to the low + // 16 bits of max. + cardinality += + container_rank(leaf->container, leaf->typecode, max_low16); + } else { + // We're not yet at the max high key, add the full container + // range. + cardinality += + container_get_cardinality(leaf->container, leaf->typecode); + } + if (compare_high48(it.key, min_high48) == 0 && min_low16 > 0) { + // We're at the min high key, remove the range up to the low 16 + // bits of min. + cardinality -= + container_rank(leaf->container, leaf->typecode, min_low16 - 1); + } + art_iterator_next(&it); + } + return cardinality; +} + +bool roaring64_bitmap_is_empty(const roaring64_bitmap_t *r) { + return art_is_empty(&r->art); +} + +uint64_t roaring64_bitmap_minimum(const roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + if (it.value == NULL) { + return UINT64_MAX; + } + leaf_t *leaf = (leaf_t *)it.value; + return combine_key(it.key, + container_minimum(leaf->container, leaf->typecode)); +} + +uint64_t roaring64_bitmap_maximum(const roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/false); + if (it.value == NULL) { + return 0; + } + leaf_t *leaf = (leaf_t *)it.value; + return combine_key(it.key, + container_maximum(leaf->container, leaf->typecode)); +} + +bool roaring64_bitmap_run_optimize(roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + bool has_run_container = false; + while (it.value != NULL) { + leaf_t *leaf = (leaf_t *)it.value; + uint8_t new_typecode; + // We don't need to free the existing container if a new one was + // created, convert_run_optimize does that internally. + leaf->container = convert_run_optimize(leaf->container, leaf->typecode, + &new_typecode); + leaf->typecode = new_typecode; + has_run_container |= new_typecode == RUN_CONTAINER_TYPE; + art_iterator_next(&it); + } + return has_run_container; +} + +static bool roaring64_leaf_internal_validate(const art_val_t *val, + const char **reason) { + leaf_t *leaf = (leaf_t *)val; + return container_internal_validate(leaf->container, leaf->typecode, reason); +} + +bool roaring64_bitmap_internal_validate(const roaring64_bitmap_t *r, + const char **reason) { + return art_internal_validate(&r->art, reason, + roaring64_leaf_internal_validate); +} + +bool roaring64_bitmap_equals(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL && it2.value != NULL) { + if (compare_high48(it1.key, it2.key) != 0) { + return false; + } + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + if (!container_equals(leaf1->container, leaf1->typecode, + leaf2->container, leaf2->typecode)) { + return false; + } + art_iterator_next(&it1); + art_iterator_next(&it2); + } + return it1.value == NULL && it2.value == NULL; +} + +bool roaring64_bitmap_is_subset(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL) { + bool it2_present = it2.value != NULL; + + int compare_result = 0; + if (it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + if (!container_is_subset(leaf1->container, leaf1->typecode, + leaf2->container, leaf2->typecode)) { + return false; + } + art_iterator_next(&it1); + art_iterator_next(&it2); + } + } + if (!it2_present || compare_result < 0) { + return false; + } else if (compare_result > 0) { + art_iterator_lower_bound(&it2, it1.key); + } + } + return true; +} + +bool roaring64_bitmap_is_strict_subset(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + return roaring64_bitmap_get_cardinality(r1) < + roaring64_bitmap_get_cardinality(r2) && + roaring64_bitmap_is_subset(r1, r2); +} + +roaring64_bitmap_t *roaring64_bitmap_and(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); + + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL && it2.value != NULL) { + // Cases: + // 1. it1 < it2 -> it1++ + // 2. it1 == it1 -> output it1 & it2, it1++, it2++ + // 3. it1 > it2 -> it2++ + int compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2: iterators at the same high key position. + leaf_t *result_leaf = (leaf_t *)roaring_malloc(sizeof(leaf_t)); + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + result_leaf->container = container_and( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &result_leaf->typecode); + + if (container_nonzero_cardinality(result_leaf->container, + result_leaf->typecode)) { + art_insert(&result->art, it1.key, (art_val_t *)result_leaf); + } else { + container_free(result_leaf->container, result_leaf->typecode); + free_leaf(result_leaf); + } + art_iterator_next(&it1); + art_iterator_next(&it2); + } else if (compare_result < 0) { + // Case 1: it1 is before it2. + art_iterator_lower_bound(&it1, it2.key); + } else { + // Case 3: it2 is before it1. + art_iterator_lower_bound(&it2, it1.key); + } + } + return result; +} + +uint64_t roaring64_bitmap_and_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t result = 0; + + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL && it2.value != NULL) { + // Cases: + // 1. it1 < it2 -> it1++ + // 2. it1 == it1 -> output cardinaltiy it1 & it2, it1++, it2++ + // 3. it1 > it2 -> it2++ + int compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + result += + container_and_cardinality(leaf1->container, leaf1->typecode, + leaf2->container, leaf2->typecode); + art_iterator_next(&it1); + art_iterator_next(&it2); + } else if (compare_result < 0) { + // Case 1: it1 is before it2. + art_iterator_lower_bound(&it1, it2.key); + } else { + // Case 3: it2 is before it1. + art_iterator_lower_bound(&it2, it1.key); + } + } + return result; +} + +// Inplace and (modifies its first argument). +void roaring64_bitmap_and_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + if (r1 == r2) { + return; + } + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL) { + // Cases: + // 1. !it2_present -> erase it1 + // 2. it2_present + // a. it1 < it2 -> erase it1 + // b. it1 == it2 -> output it1 & it2, it1++, it2++ + // c. it1 > it2 -> it2++ + bool it2_present = it2.value != NULL; + int compare_result = 0; + if (it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2a: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + + // We do the computation "in place" only when c1 is not a + // shared container. Rationale: using a shared container + // safely with in place computation would require making a + // copy and then doing the computation in place which is + // likely less efficient than avoiding in place entirely and + // always generating a new container. + uint8_t typecode2; + container_t *container2; + if (leaf1->typecode == SHARED_CONTAINER_TYPE) { + container2 = container_and( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &typecode2); + } else { + container2 = container_iand( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &typecode2); + } + + if (container2 != leaf1->container) { + container_free(leaf1->container, leaf1->typecode); + leaf1->container = container2; + leaf1->typecode = typecode2; + } + if (!container_nonzero_cardinality(container2, typecode2)) { + container_free(container2, typecode2); + art_iterator_erase(&r1->art, &it1); + free_leaf(leaf1); + } else { + // Only advance the iterator if we didn't delete the + // leaf, as erasing advances by itself. + art_iterator_next(&it1); + } + art_iterator_next(&it2); + } + } + + if (!it2_present || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + leaf_t *leaf = (leaf_t *)art_iterator_erase(&r1->art, &it1); + assert(leaf != NULL); + container_free(leaf->container, leaf->typecode); + free_leaf(leaf); + } else if (compare_result > 0) { + // Case 2c: it1 is after it2. + art_iterator_lower_bound(&it2, it1.key); + } + } +} + +bool roaring64_bitmap_intersect(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + bool intersect = false; + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL && it2.value != NULL) { + // Cases: + // 1. it1 < it2 -> it1++ + // 2. it1 == it1 -> intersect |= it1 & it2, it1++, it2++ + // 3. it1 > it2 -> it2++ + int compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + intersect |= container_intersect(leaf1->container, leaf1->typecode, + leaf2->container, leaf2->typecode); + art_iterator_next(&it1); + art_iterator_next(&it2); + } else if (compare_result < 0) { + // Case 1: it1 is before it2. + art_iterator_lower_bound(&it1, it2.key); + } else { + // Case 3: it2 is before it1. + art_iterator_lower_bound(&it2, it1.key); + } + } + return intersect; +} + +bool roaring64_bitmap_intersect_with_range(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max) { + if (min >= max) { + return false; + } + roaring64_iterator_t it; + roaring64_iterator_init_at(r, &it, /*first=*/true); + if (!roaring64_iterator_move_equalorlarger(&it, min)) { + return false; + } + return roaring64_iterator_has_value(&it) && + roaring64_iterator_value(&it) < max; +} + +double roaring64_bitmap_jaccard_index(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t c1 = roaring64_bitmap_get_cardinality(r1); + uint64_t c2 = roaring64_bitmap_get_cardinality(r2); + uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); + return (double)inter / (double)(c1 + c2 - inter); +} + +roaring64_bitmap_t *roaring64_bitmap_or(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); + + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL || it2.value != NULL) { + bool it1_present = it1.value != NULL; + bool it2_present = it2.value != NULL; + + // Cases: + // 1. it1_present && !it2_present -> output it1, it1++ + // 2. !it1_present && it2_present -> output it2, it2++ + // 3. it1_present && it2_present + // a. it1 < it2 -> output it1, it1++ + // b. it1 == it2 -> output it1 | it2, it1++, it2++ + // c. it1 > it2 -> output it2, it2++ + int compare_result = 0; + if (it1_present && it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 3b: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + leaf_t *result_leaf = (leaf_t *)roaring_malloc(sizeof(leaf_t)); + result_leaf->container = container_or( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &result_leaf->typecode); + art_insert(&result->art, it1.key, (art_val_t *)result_leaf); + art_iterator_next(&it1); + art_iterator_next(&it2); + } + } + if ((it1_present && !it2_present) || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + leaf_t *result_leaf = copy_leaf_container((leaf_t *)it1.value); + art_insert(&result->art, it1.key, (art_val_t *)result_leaf); + art_iterator_next(&it1); + } else if ((!it1_present && it2_present) || compare_result > 0) { + // Cases 2 and 3c: it2 is the only iterator or is before it1. + leaf_t *result_leaf = copy_leaf_container((leaf_t *)it2.value); + art_insert(&result->art, it2.key, (art_val_t *)result_leaf); + art_iterator_next(&it2); + } + } + return result; +} + +uint64_t roaring64_bitmap_or_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t c1 = roaring64_bitmap_get_cardinality(r1); + uint64_t c2 = roaring64_bitmap_get_cardinality(r2); + uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); + return c1 + c2 - inter; +} + +void roaring64_bitmap_or_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + if (r1 == r2) { + return; + } + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL || it2.value != NULL) { + bool it1_present = it1.value != NULL; + bool it2_present = it2.value != NULL; + + // Cases: + // 1. it1_present && !it2_present -> it1++ + // 2. !it1_present && it2_present -> add it2, it2++ + // 3. it1_present && it2_present + // a. it1 < it2 -> it1++ + // b. it1 == it2 -> it1 | it2, it1++, it2++ + // c. it1 > it2 -> add it2, it2++ + int compare_result = 0; + if (it1_present && it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 3b: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + uint8_t typecode2; + container_t *container2; + if (leaf1->typecode == SHARED_CONTAINER_TYPE) { + container2 = container_or(leaf1->container, leaf1->typecode, + leaf2->container, leaf2->typecode, + &typecode2); + } else { + container2 = container_ior( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &typecode2); + } + if (container2 != leaf1->container) { + container_free(leaf1->container, leaf1->typecode); + leaf1->container = container2; + leaf1->typecode = typecode2; + } + art_iterator_next(&it1); + art_iterator_next(&it2); + } + } + if ((it1_present && !it2_present) || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + art_iterator_next(&it1); + } else if ((!it1_present && it2_present) || compare_result > 0) { + // Cases 2 and 3c: it2 is the only iterator or is before it1. + leaf_t *result_leaf = copy_leaf_container((leaf_t *)it2.value); + art_iterator_insert(&r1->art, &it1, it2.key, + (art_val_t *)result_leaf); + art_iterator_next(&it2); + } + } +} + +roaring64_bitmap_t *roaring64_bitmap_xor(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); + + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL || it2.value != NULL) { + bool it1_present = it1.value != NULL; + bool it2_present = it2.value != NULL; + + // Cases: + // 1. it1_present && !it2_present -> output it1, it1++ + // 2. !it1_present && it2_present -> output it2, it2++ + // 3. it1_present && it2_present + // a. it1 < it2 -> output it1, it1++ + // b. it1 == it2 -> output it1 ^ it2, it1++, it2++ + // c. it1 > it2 -> output it2, it2++ + int compare_result = 0; + if (it1_present && it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 3b: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + leaf_t *result_leaf = (leaf_t *)roaring_malloc(sizeof(leaf_t)); + result_leaf->container = container_xor( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &result_leaf->typecode); + if (container_nonzero_cardinality(result_leaf->container, + result_leaf->typecode)) { + art_insert(&result->art, it1.key, (art_val_t *)result_leaf); + } else { + container_free(result_leaf->container, + result_leaf->typecode); + free_leaf(result_leaf); + } + art_iterator_next(&it1); + art_iterator_next(&it2); + } + } + if ((it1_present && !it2_present) || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + leaf_t *result_leaf = copy_leaf_container((leaf_t *)it1.value); + art_insert(&result->art, it1.key, (art_val_t *)result_leaf); + art_iterator_next(&it1); + } else if ((!it1_present && it2_present) || compare_result > 0) { + // Cases 2 and 3c: it2 is the only iterator or is before it1. + leaf_t *result_leaf = copy_leaf_container((leaf_t *)it2.value); + art_insert(&result->art, it2.key, (art_val_t *)result_leaf); + art_iterator_next(&it2); + } + } + return result; +} + +uint64_t roaring64_bitmap_xor_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t c1 = roaring64_bitmap_get_cardinality(r1); + uint64_t c2 = roaring64_bitmap_get_cardinality(r2); + uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); + return c1 + c2 - 2 * inter; +} + +void roaring64_bitmap_xor_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + assert(r1 != r2); + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL || it2.value != NULL) { + bool it1_present = it1.value != NULL; + bool it2_present = it2.value != NULL; + + // Cases: + // 1. it1_present && !it2_present -> it1++ + // 2. !it1_present && it2_present -> add it2, it2++ + // 3. it1_present && it2_present + // a. it1 < it2 -> it1++ + // b. it1 == it2 -> it1 ^ it2, it1++, it2++ + // c. it1 > it2 -> add it2, it2++ + int compare_result = 0; + if (it1_present && it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 3b: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + container_t *container1 = leaf1->container; + uint8_t typecode1 = leaf1->typecode; + uint8_t typecode2; + container_t *container2; + if (leaf1->typecode == SHARED_CONTAINER_TYPE) { + container2 = container_xor( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &typecode2); + if (container2 != container1) { + // We only free when doing container_xor, not + // container_ixor, as ixor frees the original + // internally. + container_free(container1, typecode1); + } + } else { + container2 = container_ixor( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &typecode2); + } + leaf1->container = container2; + leaf1->typecode = typecode2; + + if (!container_nonzero_cardinality(container2, typecode2)) { + container_free(container2, typecode2); + art_iterator_erase(&r1->art, &it1); + free_leaf(leaf1); + } else { + // Only advance the iterator if we didn't delete the + // leaf, as erasing advances by itself. + art_iterator_next(&it1); + } + art_iterator_next(&it2); + } + } + if ((it1_present && !it2_present) || compare_result < 0) { + // Cases 1 and 3a: it1 is the only iterator or is before it2. + art_iterator_next(&it1); + } else if ((!it1_present && it2_present) || compare_result > 0) { + // Cases 2 and 3c: it2 is the only iterator or is before it1. + leaf_t *result_leaf = copy_leaf_container((leaf_t *)it2.value); + if (it1_present) { + art_iterator_insert(&r1->art, &it1, it2.key, + (art_val_t *)result_leaf); + art_iterator_next(&it1); + } else { + art_insert(&r1->art, it2.key, (art_val_t *)result_leaf); + } + art_iterator_next(&it2); + } + } +} + +roaring64_bitmap_t *roaring64_bitmap_andnot(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + roaring64_bitmap_t *result = roaring64_bitmap_create(); + + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL) { + // Cases: + // 1. it1_present && !it2_present -> output it1, it1++ + // 2. it1_present && it2_present + // a. it1 < it2 -> output it1, it1++ + // b. it1 == it2 -> output it1 - it2, it1++, it2++ + // c. it1 > it2 -> it2++ + bool it2_present = it2.value != NULL; + int compare_result = 0; + if (it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2b: iterators at the same high key position. + leaf_t *result_leaf = (leaf_t *)roaring_malloc(sizeof(leaf_t)); + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + result_leaf->container = container_andnot( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &result_leaf->typecode); + + if (container_nonzero_cardinality(result_leaf->container, + result_leaf->typecode)) { + art_insert(&result->art, it1.key, (art_val_t *)result_leaf); + } else { + container_free(result_leaf->container, + result_leaf->typecode); + free_leaf(result_leaf); + } + art_iterator_next(&it1); + art_iterator_next(&it2); + } + } + if (!it2_present || compare_result < 0) { + // Cases 1 and 2a: it1 is the only iterator or is before it2. + leaf_t *result_leaf = copy_leaf_container((leaf_t *)it1.value); + art_insert(&result->art, it1.key, (art_val_t *)result_leaf); + art_iterator_next(&it1); + } else if (compare_result > 0) { + // Case 2c: it1 is after it2. + art_iterator_next(&it2); + } + } + return result; +} + +uint64_t roaring64_bitmap_andnot_cardinality(const roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + uint64_t c1 = roaring64_bitmap_get_cardinality(r1); + uint64_t inter = roaring64_bitmap_and_cardinality(r1, r2); + return c1 - inter; +} + +void roaring64_bitmap_andnot_inplace(roaring64_bitmap_t *r1, + const roaring64_bitmap_t *r2) { + art_iterator_t it1 = art_init_iterator(&r1->art, /*first=*/true); + art_iterator_t it2 = art_init_iterator(&r2->art, /*first=*/true); + + while (it1.value != NULL) { + // Cases: + // 1. it1_present && !it2_present -> it1++ + // 2. it1_present && it2_present + // a. it1 < it2 -> it1++ + // b. it1 == it2 -> it1 - it2, it1++, it2++ + // c. it1 > it2 -> it2++ + bool it2_present = it2.value != NULL; + int compare_result = 0; + if (it2_present) { + compare_result = compare_high48(it1.key, it2.key); + if (compare_result == 0) { + // Case 2b: iterators at the same high key position. + leaf_t *leaf1 = (leaf_t *)it1.value; + leaf_t *leaf2 = (leaf_t *)it2.value; + container_t *container1 = leaf1->container; + uint8_t typecode1 = leaf1->typecode; + uint8_t typecode2; + container_t *container2; + if (leaf1->typecode == SHARED_CONTAINER_TYPE) { + container2 = container_andnot( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &typecode2); + if (container2 != container1) { + // We only free when doing container_andnot, not + // container_iandnot, as iandnot frees the original + // internally. + container_free(container1, typecode1); + } + } else { + container2 = container_iandnot( + leaf1->container, leaf1->typecode, leaf2->container, + leaf2->typecode, &typecode2); + } + if (container2 != container1) { + leaf1->container = container2; + leaf1->typecode = typecode2; + } + + if (!container_nonzero_cardinality(container2, typecode2)) { + container_free(container2, typecode2); + art_iterator_erase(&r1->art, &it1); + free_leaf(leaf1); + } else { + // Only advance the iterator if we didn't delete the + // leaf, as erasing advances by itself. + art_iterator_next(&it1); + } + art_iterator_next(&it2); + } + } + if (!it2_present || compare_result < 0) { + // Cases 1 and 2a: it1 is the only iterator or is before it2. + art_iterator_next(&it1); + } else if (compare_result > 0) { + // Case 2c: it1 is after it2. + art_iterator_next(&it2); + } + } +} + +/** + * Flips the leaf at high48 in the range [min, max), returning a new leaf with a + * new container. If the high48 key is not found in the existing bitmap, a new + * container is created. Returns null if the negation results in an empty range. + */ +static leaf_t *roaring64_flip_leaf(const roaring64_bitmap_t *r, + uint8_t high48[], uint32_t min, + uint32_t max) { + leaf_t *leaf1 = (leaf_t *)art_find(&r->art, high48); + container_t *container2; + uint8_t typecode2; + if (leaf1 == NULL) { + // No container at this key, create a full container. + container2 = container_range_of_ones(min, max, &typecode2); + } else if (min == 0 && max > 0xFFFF) { + // Flip whole container. + container2 = + container_not(leaf1->container, leaf1->typecode, &typecode2); + } else { + // Partially flip a container. + container2 = container_not_range(leaf1->container, leaf1->typecode, min, + max, &typecode2); + } + if (container_nonzero_cardinality(container2, typecode2)) { + return create_leaf(container2, typecode2); + } + container_free(container2, typecode2); + return NULL; +} + +/** + * Flips the leaf at high48 in the range [min, max). If the high48 key is not + * found in the bitmap, a new container is created. Deletes the leaf and + * associated container if the negation results in an empty range. + */ +static void roaring64_flip_leaf_inplace(roaring64_bitmap_t *r, uint8_t high48[], + uint32_t min, uint32_t max) { + leaf_t *leaf = (leaf_t *)art_find(&r->art, high48); + container_t *container2; + uint8_t typecode2; + if (leaf == NULL) { + // No container at this key, insert a full container. + container2 = container_range_of_ones(min, max, &typecode2); + art_insert(&r->art, high48, + (art_val_t *)create_leaf(container2, typecode2)); + return; + } + + if (min == 0 && max > 0xFFFF) { + // Flip whole container. + container2 = + container_inot(leaf->container, leaf->typecode, &typecode2); + } else { + // Partially flip a container. + container2 = container_inot_range(leaf->container, leaf->typecode, min, + max, &typecode2); + } + + leaf->container = container2; + leaf->typecode = typecode2; + + if (!container_nonzero_cardinality(leaf->container, leaf->typecode)) { + art_erase(&r->art, high48); + container_free(leaf->container, leaf->typecode); + free_leaf(leaf); + } +} + +roaring64_bitmap_t *roaring64_bitmap_flip(const roaring64_bitmap_t *r, + uint64_t min, uint64_t max) { + if (min >= max) { + return roaring64_bitmap_copy(r); + } + return roaring64_bitmap_flip_closed(r, min, max - 1); +} + +roaring64_bitmap_t *roaring64_bitmap_flip_closed(const roaring64_bitmap_t *r1, + uint64_t min, uint64_t max) { + if (min > max) { + return roaring64_bitmap_copy(r1); + } + uint8_t min_high48_key[ART_KEY_BYTES]; + uint16_t min_low16 = split_key(min, min_high48_key); + uint8_t max_high48_key[ART_KEY_BYTES]; + uint16_t max_low16 = split_key(max, max_high48_key); + uint64_t min_high48_bits = (min & 0xFFFFFFFFFFFF0000ULL) >> 16; + uint64_t max_high48_bits = (max & 0xFFFFFFFFFFFF0000ULL) >> 16; + + roaring64_bitmap_t *r2 = roaring64_bitmap_create(); + art_iterator_t it = art_init_iterator(&r1->art, /*first=*/true); + + // Copy the containers before min unchanged. + while (it.value != NULL && compare_high48(it.key, min_high48_key) < 0) { + leaf_t *leaf1 = (leaf_t *)it.value; + uint8_t typecode2 = leaf1->typecode; + container_t *container2 = get_copy_of_container( + leaf1->container, &typecode2, /*copy_on_write=*/false); + art_insert(&r2->art, it.key, + (art_val_t *)create_leaf(container2, typecode2)); + art_iterator_next(&it); + } + + // Flip the range (including non-existent containers!) between min and max. + for (uint64_t high48_bits = min_high48_bits; high48_bits <= max_high48_bits; + high48_bits++) { + uint8_t current_high48_key[ART_KEY_BYTES]; + split_key(high48_bits << 16, current_high48_key); + + uint32_t min_container = 0; + if (high48_bits == min_high48_bits) { + min_container = min_low16; + } + uint32_t max_container = 0xFFFF + 1; // Exclusive range. + if (high48_bits == max_high48_bits) { + max_container = max_low16 + 1; // Exclusive. + } + + leaf_t *leaf = roaring64_flip_leaf(r1, current_high48_key, + min_container, max_container); + if (leaf != NULL) { + art_insert(&r2->art, current_high48_key, (art_val_t *)leaf); + } + } + + // Copy the containers after max unchanged. + it = art_upper_bound(&r1->art, max_high48_key); + while (it.value != NULL) { + leaf_t *leaf1 = (leaf_t *)it.value; + uint8_t typecode2 = leaf1->typecode; + container_t *container2 = get_copy_of_container( + leaf1->container, &typecode2, /*copy_on_write=*/false); + art_insert(&r2->art, it.key, + (art_val_t *)create_leaf(container2, typecode2)); + art_iterator_next(&it); + } + + return r2; +} + +void roaring64_bitmap_flip_inplace(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min >= max) { + return; + } + roaring64_bitmap_flip_closed_inplace(r, min, max - 1); +} + +void roaring64_bitmap_flip_closed_inplace(roaring64_bitmap_t *r, uint64_t min, + uint64_t max) { + if (min > max) { + return; + } + uint16_t min_low16 = (uint16_t)min; + uint16_t max_low16 = (uint16_t)max; + uint64_t min_high48_bits = (min & 0xFFFFFFFFFFFF0000ULL) >> 16; + uint64_t max_high48_bits = (max & 0xFFFFFFFFFFFF0000ULL) >> 16; + + // Flip the range (including non-existent containers!) between min and max. + for (uint64_t high48_bits = min_high48_bits; high48_bits <= max_high48_bits; + high48_bits++) { + uint8_t current_high48_key[ART_KEY_BYTES]; + split_key(high48_bits << 16, current_high48_key); + + uint32_t min_container = 0; + if (high48_bits == min_high48_bits) { + min_container = min_low16; + } + uint32_t max_container = 0xFFFF + 1; // Exclusive range. + if (high48_bits == max_high48_bits) { + max_container = max_low16 + 1; // Exclusive. + } + + roaring64_flip_leaf_inplace(r, current_high48_key, min_container, + max_container); + } +} + +// Returns the number of distinct high 32-bit entries in the bitmap. +static inline uint64_t count_high32(const roaring64_bitmap_t *r) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + uint64_t high32_count = 0; + uint32_t prev_high32; + while (it.value != NULL) { + uint32_t current_high32 = (uint32_t)(combine_key(it.key, 0) >> 32); + if (high32_count == 0 || prev_high32 != current_high32) { + high32_count++; + prev_high32 = current_high32; + } + art_iterator_next(&it); + } + return high32_count; +} + +// Frees the (32-bit!) bitmap without freeing the containers. +static inline void roaring_bitmap_free_without_containers(roaring_bitmap_t *r) { + ra_clear_without_containers(&r->high_low_container); + roaring_free(r); +} + +size_t roaring64_bitmap_portable_size_in_bytes(const roaring64_bitmap_t *r) { + // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + size_t size = 0; + + // Write as uint64 the distinct number of "buckets", where a bucket is + // defined as the most significant 32 bits of an element. + uint64_t high32_count; + size += sizeof(high32_count); + + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + uint32_t prev_high32; + roaring_bitmap_t *bitmap32 = NULL; + + // Iterate through buckets ordered by increasing keys. + while (it.value != NULL) { + uint32_t current_high32 = (uint32_t)(combine_key(it.key, 0) >> 32); + if (bitmap32 == NULL || prev_high32 != current_high32) { + if (bitmap32 != NULL) { + // Write as uint32 the most significant 32 bits of the bucket. + size += sizeof(prev_high32); + + // Write the 32-bit Roaring bitmaps representing the least + // significant bits of a set of elements. + size += roaring_bitmap_portable_size_in_bytes(bitmap32); + roaring_bitmap_free_without_containers(bitmap32); + } + + // Start a new 32-bit bitmap with the current high 32 bits. + art_iterator_t it2 = it; + uint32_t containers_with_high32 = 0; + while (it2.value != NULL && (uint32_t)(combine_key(it2.key, 0) >> + 32) == current_high32) { + containers_with_high32++; + art_iterator_next(&it2); + } + bitmap32 = + roaring_bitmap_create_with_capacity(containers_with_high32); + + prev_high32 = current_high32; + } + leaf_t *leaf = (leaf_t *)it.value; + ra_append(&bitmap32->high_low_container, + (uint16_t)(current_high32 >> 16), leaf->container, + leaf->typecode); + art_iterator_next(&it); + } + + if (bitmap32 != NULL) { + // Write as uint32 the most significant 32 bits of the bucket. + size += sizeof(prev_high32); + + // Write the 32-bit Roaring bitmaps representing the least + // significant bits of a set of elements. + size += roaring_bitmap_portable_size_in_bytes(bitmap32); + roaring_bitmap_free_without_containers(bitmap32); + } + + return size; +} + +size_t roaring64_bitmap_portable_serialize(const roaring64_bitmap_t *r, + char *buf) { + // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + if (buf == NULL) { + return 0; + } + const char *initial_buf = buf; + + // Write as uint64 the distinct number of "buckets", where a bucket is + // defined as the most significant 32 bits of an element. + uint64_t high32_count = count_high32(r); + memcpy(buf, &high32_count, sizeof(high32_count)); + buf += sizeof(high32_count); + + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + uint32_t prev_high32; + roaring_bitmap_t *bitmap32 = NULL; + + // Iterate through buckets ordered by increasing keys. + while (it.value != NULL) { + uint64_t current_high48 = combine_key(it.key, 0); + uint32_t current_high32 = (uint32_t)(current_high48 >> 32); + if (bitmap32 == NULL || prev_high32 != current_high32) { + if (bitmap32 != NULL) { + // Write as uint32 the most significant 32 bits of the bucket. + memcpy(buf, &prev_high32, sizeof(prev_high32)); + buf += sizeof(prev_high32); + + // Write the 32-bit Roaring bitmaps representing the least + // significant bits of a set of elements. + buf += roaring_bitmap_portable_serialize(bitmap32, buf); + roaring_bitmap_free_without_containers(bitmap32); + } + + // Start a new 32-bit bitmap with the current high 32 bits. + art_iterator_t it2 = it; + uint32_t containers_with_high32 = 0; + while (it2.value != NULL && + (uint32_t)combine_key(it2.key, 0) == current_high32) { + containers_with_high32++; + art_iterator_next(&it2); + } + bitmap32 = + roaring_bitmap_create_with_capacity(containers_with_high32); + + prev_high32 = current_high32; + } + leaf_t *leaf = (leaf_t *)it.value; + ra_append(&bitmap32->high_low_container, + (uint16_t)(current_high48 >> 16), leaf->container, + leaf->typecode); + art_iterator_next(&it); + } + + if (bitmap32 != NULL) { + // Write as uint32 the most significant 32 bits of the bucket. + memcpy(buf, &prev_high32, sizeof(prev_high32)); + buf += sizeof(prev_high32); + + // Write the 32-bit Roaring bitmaps representing the least + // significant bits of a set of elements. + buf += roaring_bitmap_portable_serialize(bitmap32, buf); + roaring_bitmap_free_without_containers(bitmap32); + } + + return buf - initial_buf; +} + +size_t roaring64_bitmap_portable_deserialize_size(const char *buf, + size_t maxbytes) { + // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + if (buf == NULL) { + return 0; + } + size_t read_bytes = 0; + + // Read as uint64 the distinct number of "buckets", where a bucket is + // defined as the most significant 32 bits of an element. + uint64_t buckets; + if (read_bytes + sizeof(buckets) > maxbytes) { + return 0; + } + memcpy(&buckets, buf, sizeof(buckets)); + buf += sizeof(buckets); + read_bytes += sizeof(buckets); + + // Buckets should be 32 bits with 4 bits of zero padding. + if (buckets > UINT32_MAX) { + return 0; + } + + // Iterate through buckets ordered by increasing keys. + for (uint64_t bucket = 0; bucket < buckets; ++bucket) { + // Read as uint32 the most significant 32 bits of the bucket. + uint32_t high32; + if (read_bytes + sizeof(high32) > maxbytes) { + return 0; + } + buf += sizeof(high32); + read_bytes += sizeof(high32); + + // Read the 32-bit Roaring bitmaps representing the least significant + // bits of a set of elements. + size_t bitmap32_size = roaring_bitmap_portable_deserialize_size( + buf, maxbytes - read_bytes); + if (bitmap32_size == 0) { + return 0; + } + buf += bitmap32_size; + read_bytes += bitmap32_size; + } + return read_bytes; +} + +roaring64_bitmap_t *roaring64_bitmap_portable_deserialize_safe( + const char *buf, size_t maxbytes) { + // https://github.com/RoaringBitmap/RoaringFormatSpec#extension-for-64-bit-implementations + if (buf == NULL) { + return NULL; + } + size_t read_bytes = 0; + + // Read as uint64 the distinct number of "buckets", where a bucket is + // defined as the most significant 32 bits of an element. + uint64_t buckets; + if (read_bytes + sizeof(buckets) > maxbytes) { + return NULL; + } + memcpy(&buckets, buf, sizeof(buckets)); + buf += sizeof(buckets); + read_bytes += sizeof(buckets); + + // Buckets should be 32 bits with 4 bits of zero padding. + if (buckets > UINT32_MAX) { + return NULL; + } + + roaring64_bitmap_t *r = roaring64_bitmap_create(); + // Iterate through buckets ordered by increasing keys. + for (uint64_t bucket = 0; bucket < buckets; ++bucket) { + // Read as uint32 the most significant 32 bits of the bucket. + uint32_t high32; + if (read_bytes + sizeof(high32) > maxbytes) { + roaring64_bitmap_free(r); + return NULL; + } + memcpy(&high32, buf, sizeof(high32)); + buf += sizeof(high32); + read_bytes += sizeof(high32); + + // Read the 32-bit Roaring bitmaps representing the least significant + // bits of a set of elements. + size_t bitmap32_size = roaring_bitmap_portable_deserialize_size( + buf, maxbytes - read_bytes); + if (bitmap32_size == 0) { + roaring64_bitmap_free(r); + return NULL; + } + + roaring_bitmap_t *bitmap32 = roaring_bitmap_portable_deserialize_safe( + buf, maxbytes - read_bytes); + if (bitmap32 == NULL) { + roaring64_bitmap_free(r); + return NULL; + } + buf += bitmap32_size; + read_bytes += bitmap32_size; + + // Insert all containers of the 32-bit bitmap into the 64-bit bitmap. + uint32_t r32_size = ra_get_size(&bitmap32->high_low_container); + for (size_t i = 0; i < r32_size; ++i) { + uint16_t key16 = + ra_get_key_at_index(&bitmap32->high_low_container, (uint16_t)i); + uint8_t typecode; + container_t *container = ra_get_container_at_index( + &bitmap32->high_low_container, (uint16_t)i, &typecode); + + uint64_t high48_bits = + (((uint64_t)high32) << 32) | (((uint64_t)key16) << 16); + uint8_t high48[ART_KEY_BYTES]; + split_key(high48_bits, high48); + leaf_t *leaf = create_leaf(container, typecode); + art_insert(&r->art, high48, (art_val_t *)leaf); + } + roaring_bitmap_free_without_containers(bitmap32); + } + return r; +} + +bool roaring64_bitmap_iterate(const roaring64_bitmap_t *r, + roaring_iterator64 iterator, void *ptr) { + art_iterator_t it = art_init_iterator(&r->art, /*first=*/true); + while (it.value != NULL) { + uint64_t high48 = combine_key(it.key, 0); + uint64_t high32 = high48 & 0xFFFFFFFF00000000ULL; + uint32_t low32 = high48; + leaf_t *leaf = (leaf_t *)it.value; + if (!container_iterate64(leaf->container, leaf->typecode, low32, + iterator, high32, ptr)) { + return false; + } + art_iterator_next(&it); + } + return true; +} + +void roaring64_bitmap_to_uint64_array(const roaring64_bitmap_t *r, + uint64_t *out) { + roaring64_iterator_t it = {0}; + roaring64_iterator_init_at(r, &it, /*first=*/true); + roaring64_iterator_read(&it, out, UINT64_MAX); +} + +roaring64_iterator_t *roaring64_iterator_create(const roaring64_bitmap_t *r) { + roaring64_iterator_t *it = + (roaring64_iterator_t *)roaring_malloc(sizeof(roaring64_iterator_t)); + return roaring64_iterator_init_at(r, it, /*first=*/true); +} + +roaring64_iterator_t *roaring64_iterator_create_last( + const roaring64_bitmap_t *r) { + roaring64_iterator_t *it = + (roaring64_iterator_t *)roaring_malloc(sizeof(roaring64_iterator_t)); + return roaring64_iterator_init_at(r, it, /*first=*/false); +} + +void roaring64_iterator_reinit(const roaring64_bitmap_t *r, + roaring64_iterator_t *it) { + roaring64_iterator_init_at(r, it, /*first=*/true); +} + +void roaring64_iterator_reinit_last(const roaring64_bitmap_t *r, + roaring64_iterator_t *it) { + roaring64_iterator_init_at(r, it, /*first=*/false); +} + +roaring64_iterator_t *roaring64_iterator_copy(const roaring64_iterator_t *it) { + roaring64_iterator_t *new_it = + (roaring64_iterator_t *)roaring_malloc(sizeof(roaring64_iterator_t)); + memcpy(new_it, it, sizeof(*it)); + return new_it; +} + +void roaring64_iterator_free(roaring64_iterator_t *it) { roaring_free(it); } + +bool roaring64_iterator_has_value(const roaring64_iterator_t *it) { + return it->has_value; +} + +uint64_t roaring64_iterator_value(const roaring64_iterator_t *it) { + return it->value; +} + +bool roaring64_iterator_advance(roaring64_iterator_t *it) { + if (it->art_it.value == NULL) { + if (it->saturated_forward) { + return (it->has_value = false); + } + roaring64_iterator_init_at(it->parent, it, /*first=*/true); + return it->has_value; + } + leaf_t *leaf = (leaf_t *)it->art_it.value; + uint16_t low16 = (uint16_t)it->value; + if (container_iterator_next(leaf->container, leaf->typecode, + &it->container_it, &low16)) { + it->value = it->high48 | low16; + return (it->has_value = true); + } + if (art_iterator_next(&it->art_it)) { + return roaring64_iterator_init_at_leaf_first(it); + } + it->saturated_forward = true; + return (it->has_value = false); +} + +bool roaring64_iterator_previous(roaring64_iterator_t *it) { + if (it->art_it.value == NULL) { + if (!it->saturated_forward) { + // Saturated backward. + return (it->has_value = false); + } + roaring64_iterator_init_at(it->parent, it, /*first=*/false); + return it->has_value; + } + leaf_t *leaf = (leaf_t *)it->art_it.value; + uint16_t low16 = (uint16_t)it->value; + if (container_iterator_prev(leaf->container, leaf->typecode, + &it->container_it, &low16)) { + it->value = it->high48 | low16; + return (it->has_value = true); + } + if (art_iterator_prev(&it->art_it)) { + return roaring64_iterator_init_at_leaf_last(it); + } + it->saturated_forward = false; // Saturated backward. + return (it->has_value = false); +} + +bool roaring64_iterator_move_equalorlarger(roaring64_iterator_t *it, + uint64_t val) { + uint8_t val_high48[ART_KEY_BYTES]; + uint16_t val_low16 = split_key(val, val_high48); + if (!it->has_value || it->high48 != (val & 0xFFFFFFFFFFFF0000)) { + // The ART iterator is before or after the high48 bits of `val` (or + // beyond the ART altogether), so we need to move to a leaf with a key + // equal or greater. + if (!art_iterator_lower_bound(&it->art_it, val_high48)) { + // Only smaller keys found. + it->saturated_forward = true; + return (it->has_value = false); + } + it->high48 = combine_key(it->art_it.key, 0); + // Fall through to the next if statement. + } + + if (it->high48 == (val & 0xFFFFFFFFFFFF0000)) { + // We're at equal high bits, check if a suitable value can be found in + // this container. + leaf_t *leaf = (leaf_t *)it->art_it.value; + uint16_t low16 = (uint16_t)it->value; + if (container_iterator_lower_bound(leaf->container, leaf->typecode, + &it->container_it, &low16, + val_low16)) { + it->value = it->high48 | low16; + return (it->has_value = true); + } + // Only smaller entries in this container, move to the next. + if (!art_iterator_next(&it->art_it)) { + it->saturated_forward = true; + return (it->has_value = false); + } + } + + // We're at a leaf with high bits greater than `val`, so the first entry in + // this container is our result. + return roaring64_iterator_init_at_leaf_first(it); +} + +uint64_t roaring64_iterator_read(roaring64_iterator_t *it, uint64_t *buf, + uint64_t count) { + uint64_t consumed = 0; + while (it->has_value && consumed < count) { + uint32_t container_consumed; + leaf_t *leaf = (leaf_t *)it->art_it.value; + uint16_t low16 = (uint16_t)it->value; + uint32_t container_count = UINT32_MAX; + if (count - consumed < (uint64_t)UINT32_MAX) { + container_count = count - consumed; + } + bool has_value = container_iterator_read_into_uint64( + leaf->container, leaf->typecode, &it->container_it, it->high48, buf, + container_count, &container_consumed, &low16); + consumed += container_consumed; + buf += container_consumed; + if (has_value) { + it->has_value = true; + it->value = it->high48 | low16; + assert(consumed == count); + return consumed; + } + it->has_value = art_iterator_next(&it->art_it); + if (it->has_value) { + roaring64_iterator_init_at_leaf_first(it); + } + } + return consumed; +} + +#ifdef __cplusplus +} // extern "C" +} // namespace roaring +} // namespace api +#endif diff --git a/contrib/libs/croaring/src/roaring_array.c b/contrib/libs/croaring/src/roaring_array.c new file mode 100644 index 00000000000..fabc8b5b649 --- /dev/null +++ b/contrib/libs/croaring/src/roaring_array.c @@ -0,0 +1,888 @@ +#include <assert.h> +#include <inttypes.h> +#include <stdbool.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <roaring/containers/bitset.h> +#include <roaring/containers/containers.h> +#include <roaring/memory.h> +#include <roaring/roaring_array.h> + +#ifdef __cplusplus +extern "C" { +namespace roaring { +namespace internal { +#endif + +// Convention: [0,ra->size) all elements are initialized +// [ra->size, ra->allocation_size) is junk and contains nothing needing freeing + +extern inline int32_t ra_get_size(const roaring_array_t *ra); +extern inline int32_t ra_get_index(const roaring_array_t *ra, uint16_t x); + +extern inline container_t *ra_get_container_at_index(const roaring_array_t *ra, + uint16_t i, + uint8_t *typecode); + +extern inline void ra_unshare_container_at_index(roaring_array_t *ra, + uint16_t i); + +extern inline void ra_replace_key_and_container_at_index(roaring_array_t *ra, + int32_t i, + uint16_t key, + container_t *c, + uint8_t typecode); + +extern inline void ra_set_container_at_index(const roaring_array_t *ra, + int32_t i, container_t *c, + uint8_t typecode); + +static bool realloc_array(roaring_array_t *ra, int32_t new_capacity) { + // + // Note: not implemented using C's realloc(), because the memory layout is + // Struct-of-Arrays vs. Array-of-Structs: + // https://github.com/RoaringBitmap/CRoaring/issues/256 + + if (new_capacity == 0) { + roaring_free(ra->containers); + ra->containers = NULL; + ra->keys = NULL; + ra->typecodes = NULL; + ra->allocation_size = 0; + return true; + } + const size_t memoryneeded = + new_capacity * + (sizeof(uint16_t) + sizeof(container_t *) + sizeof(uint8_t)); + void *bigalloc = roaring_malloc(memoryneeded); + if (!bigalloc) return false; + void *oldbigalloc = ra->containers; + container_t **newcontainers = (container_t **)bigalloc; + uint16_t *newkeys = (uint16_t *)(newcontainers + new_capacity); + uint8_t *newtypecodes = (uint8_t *)(newkeys + new_capacity); + assert((char *)(newtypecodes + new_capacity) == + (char *)bigalloc + memoryneeded); + if (ra->size > 0) { + memcpy(newcontainers, ra->containers, sizeof(container_t *) * ra->size); + memcpy(newkeys, ra->keys, sizeof(uint16_t) * ra->size); + memcpy(newtypecodes, ra->typecodes, sizeof(uint8_t) * ra->size); + } + ra->containers = newcontainers; + ra->keys = newkeys; + ra->typecodes = newtypecodes; + ra->allocation_size = new_capacity; + roaring_free(oldbigalloc); + return true; +} + +bool ra_init_with_capacity(roaring_array_t *new_ra, uint32_t cap) { + if (!new_ra) return false; + ra_init(new_ra); + + // Containers hold 64Ki elements, so 64Ki containers is enough to hold + // `0x10000 * 0x10000` (all 2^32) elements + if (cap > 0x10000) { + cap = 0x10000; + } + + if (cap > 0) { + void *bigalloc = roaring_malloc( + cap * (sizeof(uint16_t) + sizeof(container_t *) + sizeof(uint8_t))); + if (bigalloc == NULL) return false; + new_ra->containers = (container_t **)bigalloc; + new_ra->keys = (uint16_t *)(new_ra->containers + cap); + new_ra->typecodes = (uint8_t *)(new_ra->keys + cap); + // Narrowing is safe because of above check + new_ra->allocation_size = (int32_t)cap; + } + return true; +} + +int ra_shrink_to_fit(roaring_array_t *ra) { + int savings = (ra->allocation_size - ra->size) * + (sizeof(uint16_t) + sizeof(container_t *) + sizeof(uint8_t)); + if (!realloc_array(ra, ra->size)) { + return 0; + } + ra->allocation_size = ra->size; + return savings; +} + +void ra_init(roaring_array_t *new_ra) { + if (!new_ra) { + return; + } + new_ra->keys = NULL; + new_ra->containers = NULL; + new_ra->typecodes = NULL; + + new_ra->allocation_size = 0; + new_ra->size = 0; + new_ra->flags = 0; +} + +bool ra_overwrite(const roaring_array_t *source, roaring_array_t *dest, + bool copy_on_write) { + ra_clear_containers(dest); // we are going to overwrite them + if (source->size == 0) { // Note: can't call memcpy(NULL), even w/size + dest->size = 0; // <--- This is important. + return true; // output was just cleared, so they match + } + if (dest->allocation_size < source->size) { + if (!realloc_array(dest, source->size)) { + return false; + } + } + dest->size = source->size; + memcpy(dest->keys, source->keys, dest->size * sizeof(uint16_t)); + // we go through the containers, turning them into shared containers... + if (copy_on_write) { + for (int32_t i = 0; i < dest->size; ++i) { + source->containers[i] = get_copy_of_container( + source->containers[i], &source->typecodes[i], copy_on_write); + } + // we do a shallow copy to the other bitmap + memcpy(dest->containers, source->containers, + dest->size * sizeof(container_t *)); + memcpy(dest->typecodes, source->typecodes, + dest->size * sizeof(uint8_t)); + } else { + memcpy(dest->typecodes, source->typecodes, + dest->size * sizeof(uint8_t)); + for (int32_t i = 0; i < dest->size; i++) { + dest->containers[i] = + container_clone(source->containers[i], source->typecodes[i]); + if (dest->containers[i] == NULL) { + for (int32_t j = 0; j < i; j++) { + container_free(dest->containers[j], dest->typecodes[j]); + } + ra_clear_without_containers(dest); + return false; + } + } + } + return true; +} + +void ra_clear_containers(roaring_array_t *ra) { + for (int32_t i = 0; i < ra->size; ++i) { + container_free(ra->containers[i], ra->typecodes[i]); + } +} + +void ra_reset(roaring_array_t *ra) { + ra_clear_containers(ra); + ra->size = 0; + ra_shrink_to_fit(ra); +} + +void ra_clear_without_containers(roaring_array_t *ra) { + roaring_free( + ra->containers); // keys and typecodes are allocated with containers + ra->size = 0; + ra->allocation_size = 0; + ra->containers = NULL; + ra->keys = NULL; + ra->typecodes = NULL; +} + +void ra_clear(roaring_array_t *ra) { + ra_clear_containers(ra); + ra_clear_without_containers(ra); +} + +bool extend_array(roaring_array_t *ra, int32_t k) { + int32_t desired_size = ra->size + k; + const int32_t max_containers = 65536; + assert(desired_size <= max_containers); + if (desired_size > ra->allocation_size) { + int32_t new_capacity = + (ra->size < 1024) ? 2 * desired_size : 5 * desired_size / 4; + if (new_capacity > max_containers) { + new_capacity = max_containers; + } + + return realloc_array(ra, new_capacity); + } + return true; +} + +void ra_append(roaring_array_t *ra, uint16_t key, container_t *c, + uint8_t typecode) { + extend_array(ra, 1); + const int32_t pos = ra->size; + + ra->keys[pos] = key; + ra->containers[pos] = c; + ra->typecodes[pos] = typecode; + ra->size++; +} + +void ra_append_copy(roaring_array_t *ra, const roaring_array_t *sa, + uint16_t index, bool copy_on_write) { + extend_array(ra, 1); + const int32_t pos = ra->size; + + // old contents is junk that does not need freeing + ra->keys[pos] = sa->keys[index]; + // the shared container will be in two bitmaps + if (copy_on_write) { + sa->containers[index] = get_copy_of_container( + sa->containers[index], &sa->typecodes[index], copy_on_write); + ra->containers[pos] = sa->containers[index]; + ra->typecodes[pos] = sa->typecodes[index]; + } else { + ra->containers[pos] = + container_clone(sa->containers[index], sa->typecodes[index]); + ra->typecodes[pos] = sa->typecodes[index]; + } + ra->size++; +} + +void ra_append_copies_until(roaring_array_t *ra, const roaring_array_t *sa, + uint16_t stopping_key, bool copy_on_write) { + for (int32_t i = 0; i < sa->size; ++i) { + if (sa->keys[i] >= stopping_key) break; + ra_append_copy(ra, sa, (uint16_t)i, copy_on_write); + } +} + +void ra_append_copy_range(roaring_array_t *ra, const roaring_array_t *sa, + int32_t start_index, int32_t end_index, + bool copy_on_write) { + extend_array(ra, end_index - start_index); + for (int32_t i = start_index; i < end_index; ++i) { + const int32_t pos = ra->size; + ra->keys[pos] = sa->keys[i]; + if (copy_on_write) { + sa->containers[i] = get_copy_of_container( + sa->containers[i], &sa->typecodes[i], copy_on_write); + ra->containers[pos] = sa->containers[i]; + ra->typecodes[pos] = sa->typecodes[i]; + } else { + ra->containers[pos] = + container_clone(sa->containers[i], sa->typecodes[i]); + ra->typecodes[pos] = sa->typecodes[i]; + } + ra->size++; + } +} + +void ra_append_copies_after(roaring_array_t *ra, const roaring_array_t *sa, + uint16_t before_start, bool copy_on_write) { + int start_location = ra_get_index(sa, before_start); + if (start_location >= 0) + ++start_location; + else + start_location = -start_location - 1; + ra_append_copy_range(ra, sa, start_location, sa->size, copy_on_write); +} + +void ra_append_move_range(roaring_array_t *ra, roaring_array_t *sa, + int32_t start_index, int32_t end_index) { + extend_array(ra, end_index - start_index); + + for (int32_t i = start_index; i < end_index; ++i) { + const int32_t pos = ra->size; + + ra->keys[pos] = sa->keys[i]; + ra->containers[pos] = sa->containers[i]; + ra->typecodes[pos] = sa->typecodes[i]; + ra->size++; + } +} + +void ra_append_range(roaring_array_t *ra, roaring_array_t *sa, + int32_t start_index, int32_t end_index, + bool copy_on_write) { + extend_array(ra, end_index - start_index); + + for (int32_t i = start_index; i < end_index; ++i) { + const int32_t pos = ra->size; + ra->keys[pos] = sa->keys[i]; + if (copy_on_write) { + sa->containers[i] = get_copy_of_container( + sa->containers[i], &sa->typecodes[i], copy_on_write); + ra->containers[pos] = sa->containers[i]; + ra->typecodes[pos] = sa->typecodes[i]; + } else { + ra->containers[pos] = + container_clone(sa->containers[i], sa->typecodes[i]); + ra->typecodes[pos] = sa->typecodes[i]; + } + ra->size++; + } +} + +container_t *ra_get_container(roaring_array_t *ra, uint16_t x, + uint8_t *typecode) { + int i = binarySearch(ra->keys, (int32_t)ra->size, x); + if (i < 0) return NULL; + *typecode = ra->typecodes[i]; + return ra->containers[i]; +} + +extern inline container_t *ra_get_container_at_index(const roaring_array_t *ra, + uint16_t i, + uint8_t *typecode); + +extern inline uint16_t ra_get_key_at_index(const roaring_array_t *ra, + uint16_t i); + +extern inline int32_t ra_get_index(const roaring_array_t *ra, uint16_t x); + +extern inline int32_t ra_advance_until(const roaring_array_t *ra, uint16_t x, + int32_t pos); + +// everything skipped over is freed +int32_t ra_advance_until_freeing(roaring_array_t *ra, uint16_t x, int32_t pos) { + while (pos < ra->size && ra->keys[pos] < x) { + container_free(ra->containers[pos], ra->typecodes[pos]); + ++pos; + } + return pos; +} + +void ra_insert_new_key_value_at(roaring_array_t *ra, int32_t i, uint16_t key, + container_t *c, uint8_t typecode) { + extend_array(ra, 1); + // May be an optimization opportunity with DIY memmove + memmove(&(ra->keys[i + 1]), &(ra->keys[i]), + sizeof(uint16_t) * (ra->size - i)); + memmove(&(ra->containers[i + 1]), &(ra->containers[i]), + sizeof(container_t *) * (ra->size - i)); + memmove(&(ra->typecodes[i + 1]), &(ra->typecodes[i]), + sizeof(uint8_t) * (ra->size - i)); + ra->keys[i] = key; + ra->containers[i] = c; + ra->typecodes[i] = typecode; + ra->size++; +} + +// note: Java routine set things to 0, enabling GC. +// Java called it "resize" but it was always used to downsize. +// Allowing upsize would break the conventions about +// valid containers below ra->size. + +void ra_downsize(roaring_array_t *ra, int32_t new_length) { + assert(new_length <= ra->size); + ra->size = new_length; +} + +void ra_remove_at_index(roaring_array_t *ra, int32_t i) { + memmove(&(ra->containers[i]), &(ra->containers[i + 1]), + sizeof(container_t *) * (ra->size - i - 1)); + memmove(&(ra->keys[i]), &(ra->keys[i + 1]), + sizeof(uint16_t) * (ra->size - i - 1)); + memmove(&(ra->typecodes[i]), &(ra->typecodes[i + 1]), + sizeof(uint8_t) * (ra->size - i - 1)); + ra->size--; +} + +void ra_remove_at_index_and_free(roaring_array_t *ra, int32_t i) { + container_free(ra->containers[i], ra->typecodes[i]); + ra_remove_at_index(ra, i); +} + +// used in inplace andNot only, to slide left the containers from +// the mutated RoaringBitmap that are after the largest container of +// the argument RoaringBitmap. In use it should be followed by a call to +// downsize. +// +void ra_copy_range(roaring_array_t *ra, uint32_t begin, uint32_t end, + uint32_t new_begin) { + assert(begin <= end); + assert(new_begin < begin); + + const int range = end - begin; + + // We ensure to previously have freed overwritten containers + // that are not copied elsewhere + + memmove(&(ra->containers[new_begin]), &(ra->containers[begin]), + sizeof(container_t *) * range); + memmove(&(ra->keys[new_begin]), &(ra->keys[begin]), + sizeof(uint16_t) * range); + memmove(&(ra->typecodes[new_begin]), &(ra->typecodes[begin]), + sizeof(uint8_t) * range); +} + +void ra_shift_tail(roaring_array_t *ra, int32_t count, int32_t distance) { + if (distance > 0) { + extend_array(ra, distance); + } + int32_t srcpos = ra->size - count; + int32_t dstpos = srcpos + distance; + memmove(&(ra->keys[dstpos]), &(ra->keys[srcpos]), sizeof(uint16_t) * count); + memmove(&(ra->containers[dstpos]), &(ra->containers[srcpos]), + sizeof(container_t *) * count); + memmove(&(ra->typecodes[dstpos]), &(ra->typecodes[srcpos]), + sizeof(uint8_t) * count); + ra->size += distance; +} + +void ra_to_uint32_array(const roaring_array_t *ra, uint32_t *ans) { + size_t ctr = 0; + for (int32_t i = 0; i < ra->size; ++i) { + int num_added = container_to_uint32_array( + ans + ctr, ra->containers[i], ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16); + ctr += num_added; + } +} + +bool ra_range_uint32_array(const roaring_array_t *ra, size_t offset, + size_t limit, uint32_t *ans) { + size_t ctr = 0; + size_t dtr = 0; + + size_t t_limit = 0; + + bool first = false; + size_t first_skip = 0; + + uint32_t *t_ans = NULL; + size_t cur_len = 0; + + for (int i = 0; i < ra->size; ++i) { + const container_t *c = + container_unwrap_shared(ra->containers[i], &ra->typecodes[i]); + switch (ra->typecodes[i]) { + case BITSET_CONTAINER_TYPE: + t_limit = (const_CAST_bitset(c))->cardinality; + break; + case ARRAY_CONTAINER_TYPE: + t_limit = (const_CAST_array(c))->cardinality; + break; + case RUN_CONTAINER_TYPE: + t_limit = run_container_cardinality(const_CAST_run(c)); + break; + } + if (ctr + t_limit - 1 >= offset && ctr < offset + limit) { + if (!first) { + // first_skip = t_limit - (ctr + t_limit - offset); + first_skip = offset - ctr; + first = true; + t_ans = (uint32_t *)roaring_malloc(sizeof(*t_ans) * + (first_skip + limit)); + if (t_ans == NULL) { + return false; + } + memset(t_ans, 0, sizeof(*t_ans) * (first_skip + limit)); + cur_len = first_skip + limit; + } + if (dtr + t_limit > cur_len) { + uint32_t *append_ans = (uint32_t *)roaring_malloc( + sizeof(*append_ans) * (cur_len + t_limit)); + if (append_ans == NULL) { + if (t_ans != NULL) roaring_free(t_ans); + return false; + } + memset(append_ans, 0, + sizeof(*append_ans) * (cur_len + t_limit)); + cur_len = cur_len + t_limit; + memcpy(append_ans, t_ans, dtr * sizeof(uint32_t)); + roaring_free(t_ans); + t_ans = append_ans; + } + switch (ra->typecodes[i]) { + case BITSET_CONTAINER_TYPE: + container_to_uint32_array(t_ans + dtr, const_CAST_bitset(c), + ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16); + break; + case ARRAY_CONTAINER_TYPE: + container_to_uint32_array(t_ans + dtr, const_CAST_array(c), + ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16); + break; + case RUN_CONTAINER_TYPE: + container_to_uint32_array(t_ans + dtr, const_CAST_run(c), + ra->typecodes[i], + ((uint32_t)ra->keys[i]) << 16); + break; + } + dtr += t_limit; + } + ctr += t_limit; + if (dtr - first_skip >= limit) break; + } + if (t_ans != NULL) { + memcpy(ans, t_ans + first_skip, limit * sizeof(uint32_t)); + free(t_ans); + } + return true; +} + +bool ra_has_run_container(const roaring_array_t *ra) { + for (int32_t k = 0; k < ra->size; ++k) { + if (get_container_type(ra->containers[k], ra->typecodes[k]) == + RUN_CONTAINER_TYPE) + return true; + } + return false; +} + +uint32_t ra_portable_header_size(const roaring_array_t *ra) { + if (ra_has_run_container(ra)) { + if (ra->size < + NO_OFFSET_THRESHOLD) { // for small bitmaps, we omit the offsets + return 4 + (ra->size + 7) / 8 + 4 * ra->size; + } + return 4 + (ra->size + 7) / 8 + + 8 * ra->size; // - 4 because we pack the size with the cookie + } else { + return 4 + 4 + 8 * ra->size; + } +} + +size_t ra_portable_size_in_bytes(const roaring_array_t *ra) { + size_t count = ra_portable_header_size(ra); + + for (int32_t k = 0; k < ra->size; ++k) { + count += container_size_in_bytes(ra->containers[k], ra->typecodes[k]); + } + return count; +} + +// This function is endian-sensitive. +size_t ra_portable_serialize(const roaring_array_t *ra, char *buf) { + char *initbuf = buf; + uint32_t startOffset = 0; + bool hasrun = ra_has_run_container(ra); + if (hasrun) { + uint32_t cookie = SERIAL_COOKIE | ((uint32_t)(ra->size - 1) << 16); + memcpy(buf, &cookie, sizeof(cookie)); + buf += sizeof(cookie); + uint32_t s = (ra->size + 7) / 8; + uint8_t *bitmapOfRunContainers = (uint8_t *)roaring_calloc(s, 1); + assert(bitmapOfRunContainers != NULL); // todo: handle + for (int32_t i = 0; i < ra->size; ++i) { + if (get_container_type(ra->containers[i], ra->typecodes[i]) == + RUN_CONTAINER_TYPE) { + bitmapOfRunContainers[i / 8] |= (1 << (i % 8)); + } + } + memcpy(buf, bitmapOfRunContainers, s); + buf += s; + roaring_free(bitmapOfRunContainers); + if (ra->size < NO_OFFSET_THRESHOLD) { + startOffset = 4 + 4 * ra->size + s; + } else { + startOffset = 4 + 8 * ra->size + s; + } + } else { // backwards compatibility + uint32_t cookie = SERIAL_COOKIE_NO_RUNCONTAINER; + + memcpy(buf, &cookie, sizeof(cookie)); + buf += sizeof(cookie); + memcpy(buf, &ra->size, sizeof(ra->size)); + buf += sizeof(ra->size); + + startOffset = 4 + 4 + 4 * ra->size + 4 * ra->size; + } + for (int32_t k = 0; k < ra->size; ++k) { + memcpy(buf, &ra->keys[k], sizeof(ra->keys[k])); + buf += sizeof(ra->keys[k]); + // get_cardinality returns a value in [1,1<<16], subtracting one + // we get [0,1<<16 - 1] which fits in 16 bits + uint16_t card = (uint16_t)(container_get_cardinality(ra->containers[k], + ra->typecodes[k]) - + 1); + memcpy(buf, &card, sizeof(card)); + buf += sizeof(card); + } + if ((!hasrun) || (ra->size >= NO_OFFSET_THRESHOLD)) { + // writing the containers offsets + for (int32_t k = 0; k < ra->size; k++) { + memcpy(buf, &startOffset, sizeof(startOffset)); + buf += sizeof(startOffset); + startOffset = + startOffset + + container_size_in_bytes(ra->containers[k], ra->typecodes[k]); + } + } + for (int32_t k = 0; k < ra->size; ++k) { + buf += container_write(ra->containers[k], ra->typecodes[k], buf); + } + return buf - initbuf; +} + +// Quickly checks whether there is a serialized bitmap at the pointer, +// not exceeding size "maxbytes" in bytes. This function does not allocate +// memory dynamically. +// +// This function returns 0 if and only if no valid bitmap is found. +// Otherwise, it returns how many bytes are occupied. +// +size_t ra_portable_deserialize_size(const char *buf, const size_t maxbytes) { + size_t bytestotal = sizeof(int32_t); // for cookie + if (bytestotal > maxbytes) return 0; + uint32_t cookie; + memcpy(&cookie, buf, sizeof(int32_t)); + buf += sizeof(uint32_t); + if ((cookie & 0xFFFF) != SERIAL_COOKIE && + cookie != SERIAL_COOKIE_NO_RUNCONTAINER) { + return 0; + } + int32_t size; + + if ((cookie & 0xFFFF) == SERIAL_COOKIE) + size = (cookie >> 16) + 1; + else { + bytestotal += sizeof(int32_t); + if (bytestotal > maxbytes) return 0; + memcpy(&size, buf, sizeof(int32_t)); + buf += sizeof(uint32_t); + } + if (size > (1 << 16)) { + return 0; + } + char *bitmapOfRunContainers = NULL; + bool hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE; + if (hasrun) { + int32_t s = (size + 7) / 8; + bytestotal += s; + if (bytestotal > maxbytes) return 0; + bitmapOfRunContainers = (char *)buf; + buf += s; + } + bytestotal += size * 2 * sizeof(uint16_t); + if (bytestotal > maxbytes) return 0; + uint16_t *keyscards = (uint16_t *)buf; + buf += size * 2 * sizeof(uint16_t); + if ((!hasrun) || (size >= NO_OFFSET_THRESHOLD)) { + // skipping the offsets + bytestotal += size * 4; + if (bytestotal > maxbytes) return 0; + buf += size * 4; + } + // Reading the containers + for (int32_t k = 0; k < size; ++k) { + uint16_t tmp; + memcpy(&tmp, keyscards + 2 * k + 1, sizeof(tmp)); + uint32_t thiscard = tmp + 1; + bool isbitmap = (thiscard > DEFAULT_MAX_SIZE); + bool isrun = false; + if (hasrun) { + if ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) { + isbitmap = false; + isrun = true; + } + } + if (isbitmap) { + size_t containersize = + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + bytestotal += containersize; + if (bytestotal > maxbytes) return 0; + buf += containersize; + } else if (isrun) { + bytestotal += sizeof(uint16_t); + if (bytestotal > maxbytes) return 0; + uint16_t n_runs; + memcpy(&n_runs, buf, sizeof(uint16_t)); + buf += sizeof(uint16_t); + size_t containersize = n_runs * sizeof(rle16_t); + bytestotal += containersize; + if (bytestotal > maxbytes) return 0; + buf += containersize; + } else { + size_t containersize = thiscard * sizeof(uint16_t); + bytestotal += containersize; + if (bytestotal > maxbytes) return 0; + buf += containersize; + } + } + return bytestotal; +} + +// This function populates answer from the content of buf (reading up to +// maxbytes bytes). The function returns false if a properly serialized bitmap +// cannot be found. If it returns true, readbytes is populated by how many bytes +// were read, we have that *readbytes <= maxbytes. +// +// This function is endian-sensitive. +bool ra_portable_deserialize(roaring_array_t *answer, const char *buf, + const size_t maxbytes, size_t *readbytes) { + *readbytes = sizeof(int32_t); // for cookie + if (*readbytes > maxbytes) { + // Ran out of bytes while reading first 4 bytes. + return false; + } + uint32_t cookie; + memcpy(&cookie, buf, sizeof(int32_t)); + buf += sizeof(uint32_t); + if ((cookie & 0xFFFF) != SERIAL_COOKIE && + cookie != SERIAL_COOKIE_NO_RUNCONTAINER) { + // "I failed to find one of the right cookies. + return false; + } + int32_t size; + + if ((cookie & 0xFFFF) == SERIAL_COOKIE) + size = (cookie >> 16) + 1; + else { + *readbytes += sizeof(int32_t); + if (*readbytes > maxbytes) { + // Ran out of bytes while reading second part of the cookie. + return false; + } + memcpy(&size, buf, sizeof(int32_t)); + buf += sizeof(uint32_t); + } + if (size < 0) { + // You cannot have a negative number of containers, the data must be + // corrupted. + return false; + } + if (size > (1 << 16)) { + // You cannot have so many containers, the data must be corrupted. + return false; + } + const char *bitmapOfRunContainers = NULL; + bool hasrun = (cookie & 0xFFFF) == SERIAL_COOKIE; + if (hasrun) { + int32_t s = (size + 7) / 8; + *readbytes += s; + if (*readbytes > maxbytes) { // data is corrupted? + // Ran out of bytes while reading run bitmap. + return false; + } + bitmapOfRunContainers = buf; + buf += s; + } + uint16_t *keyscards = (uint16_t *)buf; + + *readbytes += size * 2 * sizeof(uint16_t); + if (*readbytes > maxbytes) { + // Ran out of bytes while reading key-cardinality array. + return false; + } + buf += size * 2 * sizeof(uint16_t); + + bool is_ok = ra_init_with_capacity(answer, size); + if (!is_ok) { + // Failed to allocate memory for roaring array. Bailing out. + return false; + } + + for (int32_t k = 0; k < size; ++k) { + uint16_t tmp; + memcpy(&tmp, keyscards + 2 * k, sizeof(tmp)); + answer->keys[k] = tmp; + } + if ((!hasrun) || (size >= NO_OFFSET_THRESHOLD)) { + *readbytes += size * 4; + if (*readbytes > maxbytes) { // data is corrupted? + // Ran out of bytes while reading offsets. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + + // skipping the offsets + buf += size * 4; + } + // Reading the containers + for (int32_t k = 0; k < size; ++k) { + uint16_t tmp; + memcpy(&tmp, keyscards + 2 * k + 1, sizeof(tmp)); + uint32_t thiscard = tmp + 1; + bool isbitmap = (thiscard > DEFAULT_MAX_SIZE); + bool isrun = false; + if (hasrun) { + if ((bitmapOfRunContainers[k / 8] & (1 << (k % 8))) != 0) { + isbitmap = false; + isrun = true; + } + } + if (isbitmap) { + // we check that the read is allowed + size_t containersize = + BITSET_CONTAINER_SIZE_IN_WORDS * sizeof(uint64_t); + *readbytes += containersize; + if (*readbytes > maxbytes) { + // Running out of bytes while reading a bitset container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + // it is now safe to read + bitset_container_t *c = bitset_container_create(); + if (c == NULL) { // memory allocation failure + // Failed to allocate memory for a bitset container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + answer->size++; + buf += bitset_container_read(thiscard, c, buf); + answer->containers[k] = c; + answer->typecodes[k] = BITSET_CONTAINER_TYPE; + } else if (isrun) { + // we check that the read is allowed + *readbytes += sizeof(uint16_t); + if (*readbytes > maxbytes) { + // Running out of bytes while reading a run container (header). + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + uint16_t n_runs; + memcpy(&n_runs, buf, sizeof(uint16_t)); + size_t containersize = n_runs * sizeof(rle16_t); + *readbytes += containersize; + if (*readbytes > maxbytes) { // data is corrupted? + // Running out of bytes while reading a run container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + // it is now safe to read + + run_container_t *c = run_container_create(); + if (c == NULL) { // memory allocation failure + // Failed to allocate memory for a run container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + answer->size++; + buf += run_container_read(thiscard, c, buf); + answer->containers[k] = c; + answer->typecodes[k] = RUN_CONTAINER_TYPE; + } else { + // we check that the read is allowed + size_t containersize = thiscard * sizeof(uint16_t); + *readbytes += containersize; + if (*readbytes > maxbytes) { // data is corrupted? + // Running out of bytes while reading an array container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + // it is now safe to read + array_container_t *c = + array_container_create_given_capacity(thiscard); + if (c == NULL) { // memory allocation failure + // Failed to allocate memory for an array container. + ra_clear(answer); // we need to clear the containers already + // allocated, and the roaring array + return false; + } + answer->size++; + buf += array_container_read(thiscard, c, buf); + answer->containers[k] = c; + answer->typecodes[k] = ARRAY_CONTAINER_TYPE; + } + } + return true; +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace internal { +#endif diff --git a/contrib/libs/croaring/src/roaring_priority_queue.c b/contrib/libs/croaring/src/roaring_priority_queue.c new file mode 100644 index 00000000000..d53df8ee81e --- /dev/null +++ b/contrib/libs/croaring/src/roaring_priority_queue.c @@ -0,0 +1,253 @@ +#include <roaring/roaring.h> +#include <roaring/roaring_array.h> + +#ifdef __cplusplus +using namespace ::roaring::internal; + +extern "C" { +namespace roaring { +namespace api { +#endif + +struct roaring_pq_element_s { + uint64_t size; + bool is_temporary; + roaring_bitmap_t *bitmap; +}; + +typedef struct roaring_pq_element_s roaring_pq_element_t; + +struct roaring_pq_s { + roaring_pq_element_t *elements; + uint64_t size; +}; + +typedef struct roaring_pq_s roaring_pq_t; + +static inline bool compare(roaring_pq_element_t *t1, roaring_pq_element_t *t2) { + return t1->size < t2->size; +} + +static void pq_add(roaring_pq_t *pq, roaring_pq_element_t *t) { + uint64_t i = pq->size; + pq->elements[pq->size++] = *t; + while (i > 0) { + uint64_t p = (i - 1) >> 1; + roaring_pq_element_t ap = pq->elements[p]; + if (!compare(t, &ap)) break; + pq->elements[i] = ap; + i = p; + } + pq->elements[i] = *t; +} + +static void pq_free(roaring_pq_t *pq) { roaring_free(pq); } + +static void percolate_down(roaring_pq_t *pq, uint32_t i) { + uint32_t size = (uint32_t)pq->size; + uint32_t hsize = size >> 1; + roaring_pq_element_t ai = pq->elements[i]; + while (i < hsize) { + uint32_t l = (i << 1) + 1; + uint32_t r = l + 1; + roaring_pq_element_t bestc = pq->elements[l]; + if (r < size) { + if (compare(pq->elements + r, &bestc)) { + l = r; + bestc = pq->elements[r]; + } + } + if (!compare(&bestc, &ai)) { + break; + } + pq->elements[i] = bestc; + i = l; + } + pq->elements[i] = ai; +} + +static roaring_pq_t *create_pq(const roaring_bitmap_t **arr, uint32_t length) { + size_t alloc_size = + sizeof(roaring_pq_t) + sizeof(roaring_pq_element_t) * length; + roaring_pq_t *answer = (roaring_pq_t *)roaring_malloc(alloc_size); + answer->elements = (roaring_pq_element_t *)(answer + 1); + answer->size = length; + for (uint32_t i = 0; i < length; i++) { + answer->elements[i].bitmap = (roaring_bitmap_t *)arr[i]; + answer->elements[i].is_temporary = false; + answer->elements[i].size = + roaring_bitmap_portable_size_in_bytes(arr[i]); + } + for (int32_t i = (length >> 1); i >= 0; i--) { + percolate_down(answer, i); + } + return answer; +} + +static roaring_pq_element_t pq_poll(roaring_pq_t *pq) { + roaring_pq_element_t ans = *pq->elements; + if (pq->size > 1) { + pq->elements[0] = pq->elements[--pq->size]; + percolate_down(pq, 0); + } else + --pq->size; + // memmove(pq->elements,pq->elements+1,(pq->size-1)*sizeof(roaring_pq_element_t));--pq->size; + return ans; +} + +// this function consumes and frees the inputs +static roaring_bitmap_t *lazy_or_from_lazy_inputs(roaring_bitmap_t *x1, + roaring_bitmap_t *x2) { + uint8_t result_type = 0; + const int length1 = ra_get_size(&x1->high_low_container), + length2 = ra_get_size(&x2->high_low_container); + if (0 == length1) { + roaring_bitmap_free(x1); + return x2; + } + if (0 == length2) { + roaring_bitmap_free(x2); + return x1; + } + uint32_t neededcap = length1 > length2 ? length2 : length1; + roaring_bitmap_t *answer = roaring_bitmap_create_with_capacity(neededcap); + int pos1 = 0, pos2 = 0; + uint8_t type1, type2; + uint16_t s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + uint16_t s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + while (true) { + if (s1 == s2) { + // todo: unsharing can be inefficient as it may create a clone where + // none + // is needed, but it has the benefit of being easy to reason about. + + ra_unshare_container_at_index(&x1->high_low_container, + (uint16_t)pos1); + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + assert(type1 != SHARED_CONTAINER_TYPE); + + ra_unshare_container_at_index(&x2->high_low_container, + (uint16_t)pos2); + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + assert(type2 != SHARED_CONTAINER_TYPE); + + container_t *c; + + if ((type2 == BITSET_CONTAINER_TYPE) && + (type1 != BITSET_CONTAINER_TYPE)) { + c = container_lazy_ior(c2, type2, c1, type1, &result_type); + container_free(c1, type1); + if (c != c2) { + container_free(c2, type2); + } + } else { + c = container_lazy_ior(c1, type1, c2, type2, &result_type); + container_free(c2, type2); + if (c != c1) { + container_free(c1, type1); + } + } + // since we assume that the initial containers are non-empty, the + // result here + // can only be non-empty + ra_append(&answer->high_low_container, s1, c, result_type); + ++pos1; + ++pos2; + if (pos1 == length1) break; + if (pos2 == length2) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + + } else if (s1 < s2) { // s1 < s2 + container_t *c1 = ra_get_container_at_index(&x1->high_low_container, + (uint16_t)pos1, &type1); + ra_append(&answer->high_low_container, s1, c1, type1); + pos1++; + if (pos1 == length1) break; + s1 = ra_get_key_at_index(&x1->high_low_container, (uint16_t)pos1); + + } else { // s1 > s2 + container_t *c2 = ra_get_container_at_index(&x2->high_low_container, + (uint16_t)pos2, &type2); + ra_append(&answer->high_low_container, s2, c2, type2); + pos2++; + if (pos2 == length2) break; + s2 = ra_get_key_at_index(&x2->high_low_container, (uint16_t)pos2); + } + } + if (pos1 == length1) { + ra_append_move_range(&answer->high_low_container, + &x2->high_low_container, pos2, length2); + } else if (pos2 == length2) { + ra_append_move_range(&answer->high_low_container, + &x1->high_low_container, pos1, length1); + } + ra_clear_without_containers(&x1->high_low_container); + ra_clear_without_containers(&x2->high_low_container); + roaring_free(x1); + roaring_free(x2); + return answer; +} + +/** + * Compute the union of 'number' bitmaps using a heap. This can + * sometimes be faster than roaring_bitmap_or_many which uses + * a naive algorithm. Caller is responsible for freeing the + * result. + */ +roaring_bitmap_t *roaring_bitmap_or_many_heap(uint32_t number, + const roaring_bitmap_t **x) { + if (number == 0) { + return roaring_bitmap_create(); + } + if (number == 1) { + return roaring_bitmap_copy(x[0]); + } + roaring_pq_t *pq = create_pq(x, number); + while (pq->size > 1) { + roaring_pq_element_t x1 = pq_poll(pq); + roaring_pq_element_t x2 = pq_poll(pq); + + if (x1.is_temporary && x2.is_temporary) { + roaring_bitmap_t *newb = + lazy_or_from_lazy_inputs(x1.bitmap, x2.bitmap); + // should normally return a fresh new bitmap *except* that + // it can return x1.bitmap or x2.bitmap in degenerate cases + bool temporary = !((newb == x1.bitmap) && (newb == x2.bitmap)); + uint64_t bsize = roaring_bitmap_portable_size_in_bytes(newb); + roaring_pq_element_t newelement = { + .size = bsize, .is_temporary = temporary, .bitmap = newb}; + pq_add(pq, &newelement); + } else if (x2.is_temporary) { + roaring_bitmap_lazy_or_inplace(x2.bitmap, x1.bitmap, false); + x2.size = roaring_bitmap_portable_size_in_bytes(x2.bitmap); + pq_add(pq, &x2); + } else if (x1.is_temporary) { + roaring_bitmap_lazy_or_inplace(x1.bitmap, x2.bitmap, false); + x1.size = roaring_bitmap_portable_size_in_bytes(x1.bitmap); + + pq_add(pq, &x1); + } else { + roaring_bitmap_t *newb = + roaring_bitmap_lazy_or(x1.bitmap, x2.bitmap, false); + uint64_t bsize = roaring_bitmap_portable_size_in_bytes(newb); + roaring_pq_element_t newelement = { + .size = bsize, .is_temporary = true, .bitmap = newb}; + + pq_add(pq, &newelement); + } + } + roaring_pq_element_t X = pq_poll(pq); + roaring_bitmap_t *answer = X.bitmap; + roaring_bitmap_repair_after_lazy(answer); + pq_free(pq); + return answer; +} + +#ifdef __cplusplus +} +} +} // extern "C" { namespace roaring { namespace api { +#endif diff --git a/contrib/libs/croaring/ya.make b/contrib/libs/croaring/ya.make new file mode 100644 index 00000000000..4ca24ffd5df --- /dev/null +++ b/contrib/libs/croaring/ya.make @@ -0,0 +1,51 @@ +# Generated by devtools/yamaker from nixpkgs 22.11. + +LIBRARY() + +LICENSE( + Apache-2.0 AND + BSD-3-Clause AND + MIT +) + +LICENSE_TEXTS(.yandex_meta/licenses.list.txt) + +VERSION(3.0.0) + +ORIGINAL_SOURCE(https://github.com/RoaringBitmap/CRoaring/archive/v3.0.0.tar.gz) + +ADDINCL( + GLOBAL contrib/libs/croaring/include + contrib/libs/croaring/include/roaring +) + +NO_COMPILER_WARNINGS() + +NO_RUNTIME() + +SRCS( + src/array_util.c + src/art/art.c + src/bitset.c + src/bitset_util.c + src/containers/array.c + src/containers/bitset.c + src/containers/containers.c + src/containers/convert.c + src/containers/mixed_andnot.c + src/containers/mixed_equal.c + src/containers/mixed_intersection.c + src/containers/mixed_negation.c + src/containers/mixed_subset.c + src/containers/mixed_union.c + src/containers/mixed_xor.c + src/containers/run.c + src/isadetection.c + src/memory.c + src/roaring.c + src/roaring64.c + src/roaring_array.c + src/roaring_priority_queue.c +) + +END() diff --git a/contrib/libs/linux-headers/ya.make b/contrib/libs/linux-headers/ya.make index dcaf1a8101a..7a7d74a66ac 100644 --- a/contrib/libs/linux-headers/ya.make +++ b/contrib/libs/linux-headers/ya.make @@ -30,15 +30,27 @@ VERSION(6.5.9) ORIGINAL_SOURCE(mirror://kernel/linux/kernel/v6.x/linux-6.5.9.tar.xz) +IF (OPENSOURCE_REPLACE_LINUX_HEADERS AND EXPORT_CMAKE) + + OPENSOURCE_EXPORT_REPLACEMENT( + CMAKE linux-headers-generic + CMAKE_TARGET linux-headers-generic::linux-headers-generic + CONAN linux-headers-generic/${OPENSOURCE_REPLACE_LINUX_HEADERS} + ) + +ELSE() + + ADDINCL( + GLOBAL contrib/libs/linux-headers + GLOBAL contrib/libs/linux-headers/_nf + ) + +ENDIF() + DISABLE(NEED_PLATFORM_PEERDIRS) DISABLE(WITH_VALGRIND) -ADDINCL( - GLOBAL contrib/libs/linux-headers - GLOBAL contrib/libs/linux-headers/_nf -) - NO_PLATFORM() END() diff --git a/contrib/libs/openssl/crypto/ya.make b/contrib/libs/openssl/crypto/ya.make index be3f2b7328d..c6e9d82018f 100644 --- a/contrib/libs/openssl/crypto/ya.make +++ b/contrib/libs/openssl/crypto/ya.make @@ -13,15 +13,16 @@ LICENSE( LICENSE_TEXTS(.yandex_meta/licenses.list.txt) -# TODO(YMAKE-92) Move this information out of ya.make and allow per project configuration -IF (OPENSOURCE_PROJECT == "catboost") +IF (OPENSOURCE_REPLACE_OPENSSL) + OPENSOURCE_EXPORT_REPLACEMENT( CMAKE OpenSSL CMAKE_PACKAGE_COMPONENT Crypto CMAKE_TARGET OpenSSL::Crypto - CONAN openssl/1.1.1t + CONAN openssl/${OPENSOURCE_REPLACE_OPENSSL} ) -ENDIF() + +ENDIF() # IF (OPENSOURCE_REPLACE_OPENSSL) PEERDIR( contrib/libs/zlib @@ -37,8 +38,7 @@ ADDINCL( contrib/libs/openssl/include ) -# TODO(YMAKE-92) Move this information out of ya.make and allow per project configuration -IF (NOT EXPORT_CMAKE OR OPENSOURCE_PROJECT != "catboost") +IF (NOT EXPORT_CMAKE OR NOT OPENSOURCE_REPLACE_OPENSSL) IF (OS_LINUX) IF (ARCH_ARM64) @@ -1362,6 +1362,6 @@ IF (ARCADIA_OPENSSL_DISABLE_ARMV7_TICK) ) ENDIF() -ENDIF() +ENDIF() # IF (NOT EXPORT_CMAKE OR NOT OPENSOURCE_REPLACE_OPENSSL) END() diff --git a/contrib/libs/openssl/ya.make b/contrib/libs/openssl/ya.make index e8d70a9a317..802b2c76c3d 100644 --- a/contrib/libs/openssl/ya.make +++ b/contrib/libs/openssl/ya.make @@ -6,13 +6,20 @@ VERSION(1.1.1t) ORIGINAL_SOURCE(https://github.com/openssl/openssl/archive/OpenSSL_1_1_1t.tar.gz) -# TODO(YMAKE-92) Move this information out of ya.make and allow per project configuration -IF (OPENSOURCE_PROJECT == "catboost") +IF (OPENSOURCE_REPLACE_OPENSSL AND EXPORT_CMAKE) + OPENSOURCE_EXPORT_REPLACEMENT( CMAKE OpenSSL CMAKE_TARGET OpenSSL::OpenSSL - CONAN openssl/1.1.1t + CONAN openssl/${OPENSOURCE_REPLACE_OPENSSL} + ) + +ELSE() + + ADDINCL( + GLOBAL contrib/libs/openssl/include ) + ENDIF() LICENSE( @@ -33,12 +40,10 @@ PEERDIR( ) ADDINCL( - GLOBAL contrib/libs/openssl/include contrib/libs/openssl ) -# TODO(YMAKE-92) Move this information out of ya.make and allow per project configuration -IF (NOT EXPORT_CMAKE OR OPENSOURCE_PROJECT != "catboost") +IF (NOT EXPORT_CMAKE OR NOT OPENSOURCE_REPLACE_OPENSSL) IF (OS_LINUX) IF (ARCH_ARM64) @@ -335,7 +340,7 @@ IF (OS_ANDROID AND ARCH_ARM64) ) ENDIF() -ENDIF() +ENDIF() # IF (NOT EXPORT_CMAKE OR NOT OPENSOURCE_REPLACE_OPENSSL) END() diff --git a/contrib/libs/protobuf/ya.make b/contrib/libs/protobuf/ya.make index 258e3259599..b8ba3c36fc6 100644 --- a/contrib/libs/protobuf/ya.make +++ b/contrib/libs/protobuf/ya.make @@ -15,18 +15,28 @@ VERSION(3.19.0) ORIGINAL_SOURCE(https://github.com/protocolbuffers/protobuf/archive/v3.19.0.tar.gz) +IF (OPENSOURCE_REPLACE_PROTOBUF AND EXPORT_CMAKE) + + OPENSOURCE_EXPORT_REPLACEMENT( + CMAKE Protobuf + CMAKE_TARGET protobuf::libprotobuf protobuf::libprotoc + CONAN protobuf/${OPENSOURCE_REPLACE_PROTOBUF} "&& conan_require_tool" protobuf/${OPENSOURCE_REPLACE_PROTOBUF} "&& conan_import \"bin, protoc* -> ./bin\" && vanilla_protobuf" + ) + +ELSE() + + ADDINCL( + GLOBAL contrib/libs/protobuf/src + GLOBAL FOR proto contrib/libs/protobuf/src + ) + +ENDIF() + PEERDIR( contrib/libs/zlib library/cpp/sanitizer/include ) -ADDINCL( - GLOBAL contrib/libs/protobuf/src - GLOBAL FOR - proto - contrib/libs/protobuf/src -) - NO_COMPILER_WARNINGS() CFLAGS( diff --git a/contrib/libs/protoc/ya.make b/contrib/libs/protoc/ya.make index f1134013cb7..927ead0dbd0 100644 --- a/contrib/libs/protoc/ya.make +++ b/contrib/libs/protoc/ya.make @@ -13,16 +13,28 @@ VERSION(3.19.0) ORIGINAL_SOURCE(https://github.com/protocolbuffers/protobuf/archive/v3.19.0.tar.gz) +IF (OPENSOURCE_REPLACE_PROTOBUF AND EXPORT_CMAKE) + + OPENSOURCE_EXPORT_REPLACEMENT( + CMAKE Protobuf + CMAKE_TARGET protobuf::libprotobuf protobuf::libprotoc + CONAN protobuf/${OPENSOURCE_REPLACE_PROTOBUF} "&& conan_require_tool" protobuf/${OPENSOURCE_REPLACE_PROTOBUF} "&& conan_import \"bin, protoc* -> ./bin\" && vanilla_protobuf" + ) + +ELSE() + + ADDINCL( + GLOBAL contrib/libs/protoc/src + ) + +ENDIF() + LICENSE_TEXTS(.yandex_meta/licenses.list.txt) PEERDIR( contrib/libs/protobuf ) -ADDINCL( - GLOBAL contrib/libs/protoc/src -) - NO_COMPILER_WARNINGS() NO_UTIL() diff --git a/contrib/python/PyJWT/py2/.dist-info/METADATA b/contrib/python/PyJWT/py2/.dist-info/METADATA new file mode 100644 index 00000000000..47ee5589077 --- /dev/null +++ b/contrib/python/PyJWT/py2/.dist-info/METADATA @@ -0,0 +1,115 @@ +Metadata-Version: 2.1 +Name: PyJWT +Version: 1.7.1 +Summary: JSON Web Token implementation in Python +Home-page: http://github.com/jpadilla/pyjwt +Author: Jose Padilla +Author-email: [email protected] +License: MIT +Keywords: jwt json web token security signing +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Natural Language :: English +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Topic :: Utilities +Provides-Extra: crypto +Requires-Dist: cryptography (>=1.4) ; extra == 'crypto' +Provides-Extra: flake8 +Requires-Dist: flake8 ; extra == 'flake8' +Requires-Dist: flake8-import-order ; extra == 'flake8' +Requires-Dist: pep8-naming ; extra == 'flake8' +Provides-Extra: test +Requires-Dist: pytest (<5.0.0,>=4.0.1) ; extra == 'test' +Requires-Dist: pytest-cov (<3.0.0,>=2.6.0) ; extra == 'test' +Requires-Dist: pytest-runner (<5.0.0,>=4.2) ; extra == 'test' + +PyJWT +===== + +.. image:: https://travis-ci.com/jpadilla/pyjwt.svg?branch=master + :target: http://travis-ci.com/jpadilla/pyjwt?branch=master + +.. image:: https://ci.appveyor.com/api/projects/status/h8nt70aqtwhht39t?svg=true + :target: https://ci.appveyor.com/project/jpadilla/pyjwt + +.. image:: https://img.shields.io/pypi/v/pyjwt.svg + :target: https://pypi.python.org/pypi/pyjwt + +.. image:: https://coveralls.io/repos/jpadilla/pyjwt/badge.svg?branch=master + :target: https://coveralls.io/r/jpadilla/pyjwt?branch=master + +.. image:: https://readthedocs.org/projects/pyjwt/badge/?version=latest + :target: https://pyjwt.readthedocs.io + +A Python implementation of `RFC 7519 <https://tools.ietf.org/html/rfc7519>`_. Original implementation was written by `@progrium <https://github.com/progrium>`_. + +Sponsor +------- + ++--------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| |auth0-logo| | If you want to quickly add secure token-based authentication to Python projects, feel free to check Auth0's Python SDK and free plan at `auth0.com/overview <https://auth0.com/overview?utm_source=GHsponsor&utm_medium=GHsponsor&utm_campaign=pyjwt&utm_content=auth>`_. | ++--------------+-----------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +.. |auth0-logo| image:: https://user-images.githubusercontent.com/83319/31722733-de95bbde-b3ea-11e7-96bf-4f4e8f915588.png + +Installing +---------- + +Install with **pip**: + +.. code-block:: sh + + $ pip install PyJWT + + +Usage +----- + +.. code:: python + + >>> import jwt + >>> encoded = jwt.encode({'some': 'payload'}, 'secret', algorithm='HS256') + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzb21lIjoicGF5bG9hZCJ9.4twFt5NiznN84AWoo1d7KO1T_yoc0Z6XOpOVswacPZg' + + >>> jwt.decode(encoded, 'secret', algorithms=['HS256']) + {'some': 'payload'} + + +Command line +------------ + +Usage:: + + pyjwt [options] INPUT + +Decoding examples:: + + pyjwt --key=secret decode TOKEN + pyjwt decode --no-verify TOKEN + +See more options executing ``pyjwt --help``. + + +Documentation +------------- + +View the full docs online at https://pyjwt.readthedocs.io/en/latest/ + + +Tests +----- + +You can run tests from the project root after cloning with: + +.. code-block:: sh + + $ python setup.py test + + diff --git a/contrib/python/PyJWT/py2/.dist-info/entry_points.txt b/contrib/python/PyJWT/py2/.dist-info/entry_points.txt new file mode 100644 index 00000000000..78717b26610 --- /dev/null +++ b/contrib/python/PyJWT/py2/.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +pyjwt = jwt.__main__:main + diff --git a/contrib/python/PyJWT/py2/.dist-info/top_level.txt b/contrib/python/PyJWT/py2/.dist-info/top_level.txt new file mode 100644 index 00000000000..27ccc9bc3a9 --- /dev/null +++ b/contrib/python/PyJWT/py2/.dist-info/top_level.txt @@ -0,0 +1 @@ +jwt diff --git a/contrib/python/PyJWT/py2/jwt/__init__.py b/contrib/python/PyJWT/py2/jwt/__init__.py new file mode 100644 index 00000000000..946983f0221 --- /dev/null +++ b/contrib/python/PyJWT/py2/jwt/__init__.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# flake8: noqa + +""" +JSON Web Token implementation + +Minimum implementation based on this spec: +http://self-issued.info/docs/draft-jones-json-web-token-01.html +""" + + +__title__ = 'pyjwt' +__version__ = '1.7.1' +__author__ = 'José Padilla' +__license__ = 'MIT' +__copyright__ = 'Copyright 2015-2018 José Padilla' + + +from .api_jwt import ( + encode, decode, register_algorithm, unregister_algorithm, + get_unverified_header, PyJWT +) +from .api_jws import PyJWS +from .exceptions import ( + InvalidTokenError, DecodeError, InvalidAlgorithmError, + InvalidAudienceError, ExpiredSignatureError, ImmatureSignatureError, + InvalidIssuedAtError, InvalidIssuerError, ExpiredSignature, + InvalidAudience, InvalidIssuer, MissingRequiredClaimError, + InvalidSignatureError, + PyJWTError, +) diff --git a/contrib/python/PyJWT/py2/jwt/__main__.py b/contrib/python/PyJWT/py2/jwt/__main__.py new file mode 100644 index 00000000000..bf50aabf4af --- /dev/null +++ b/contrib/python/PyJWT/py2/jwt/__main__.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python + +from __future__ import absolute_import, print_function + +import argparse +import json +import sys +import time + +from . import DecodeError, __version__, decode, encode + + +def encode_payload(args): + # Try to encode + if args.key is None: + raise ValueError('Key is required when encoding. See --help for usage.') + + # Build payload object to encode + payload = {} + + for arg in args.payload: + k, v = arg.split('=', 1) + + # exp +offset special case? + if k == 'exp' and v[0] == '+' and len(v) > 1: + v = str(int(time.time()+int(v[1:]))) + + # Cast to integer? + if v.isdigit(): + v = int(v) + else: + # Cast to float? + try: + v = float(v) + except ValueError: + pass + + # Cast to true, false, or null? + constants = {'true': True, 'false': False, 'null': None} + + if v in constants: + v = constants[v] + + payload[k] = v + + token = encode( + payload, + key=args.key, + algorithm=args.algorithm + ) + + return token.decode('utf-8') + + +def decode_payload(args): + try: + if args.token: + token = args.token + else: + if sys.stdin.isatty(): + token = sys.stdin.readline().strip() + else: + raise IOError('Cannot read from stdin: terminal not a TTY') + + token = token.encode('utf-8') + data = decode(token, key=args.key, verify=args.verify) + + return json.dumps(data) + + except DecodeError as e: + raise DecodeError('There was an error decoding the token: %s' % e) + + +def build_argparser(): + + usage = ''' + Encodes or decodes JSON Web Tokens based on input. + + %(prog)s [options] <command> [options] input + + Decoding examples: + + %(prog)s --key=secret decode json.web.token + %(prog)s decode --no-verify json.web.token + + Encoding requires the key option and takes space separated key/value pairs + separated by equals (=) as input. Examples: + + %(prog)s --key=secret encode iss=me exp=1302049071 + %(prog)s --key=secret encode foo=bar exp=+10 + + The exp key is special and can take an offset to current Unix time. + ''' + + arg_parser = argparse.ArgumentParser( + prog='pyjwt', + usage=usage + ) + + arg_parser.add_argument( + '-v', '--version', + action='version', + version='%(prog)s ' + __version__ + ) + + arg_parser.add_argument( + '--key', + dest='key', + metavar='KEY', + default=None, + help='set the secret key to sign with' + ) + + arg_parser.add_argument( + '--alg', + dest='algorithm', + metavar='ALG', + default='HS256', + help='set crypto algorithm to sign with. default=HS256' + ) + + subparsers = arg_parser.add_subparsers( + title='PyJWT subcommands', + description='valid subcommands', + help='additional help' + ) + + # Encode subcommand + encode_parser = subparsers.add_parser('encode', help='use to encode a supplied payload') + + payload_help = """Payload to encode. Must be a space separated list of key/value + pairs separated by equals (=) sign.""" + + encode_parser.add_argument('payload', nargs='+', help=payload_help) + encode_parser.set_defaults(func=encode_payload) + + # Decode subcommand + decode_parser = subparsers.add_parser('decode', help='use to decode a supplied JSON web token') + decode_parser.add_argument( + 'token', + help='JSON web token to decode.', + nargs='?') + + decode_parser.add_argument( + '-n', '--no-verify', + action='store_false', + dest='verify', + default=True, + help='ignore signature and claims verification on decode' + ) + + decode_parser.set_defaults(func=decode_payload) + + return arg_parser + + +def main(): + arg_parser = build_argparser() + + try: + arguments = arg_parser.parse_args(sys.argv[1:]) + + output = arguments.func(arguments) + + print(output) + except Exception as e: + print('There was an unforseen error: ', e) + arg_parser.print_help() diff --git a/contrib/python/PyJWT/py2/jwt/algorithms.py b/contrib/python/PyJWT/py2/jwt/algorithms.py new file mode 100644 index 00000000000..13436883417 --- /dev/null +++ b/contrib/python/PyJWT/py2/jwt/algorithms.py @@ -0,0 +1,403 @@ +import hashlib +import hmac +import json + + +from .compat import constant_time_compare, string_types +from .exceptions import InvalidKeyError +from .utils import ( + base64url_decode, base64url_encode, der_to_raw_signature, + force_bytes, force_unicode, from_base64url_uint, raw_to_der_signature, + to_base64url_uint +) + +try: + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.serialization import ( + load_pem_private_key, load_pem_public_key, load_ssh_public_key + ) + from cryptography.hazmat.primitives.asymmetric.rsa import ( + RSAPrivateKey, RSAPublicKey, RSAPrivateNumbers, RSAPublicNumbers, + rsa_recover_prime_factors, rsa_crt_dmp1, rsa_crt_dmq1, rsa_crt_iqmp + ) + from cryptography.hazmat.primitives.asymmetric.ec import ( + EllipticCurvePrivateKey, EllipticCurvePublicKey + ) + from cryptography.hazmat.primitives.asymmetric import ec, padding + from cryptography.hazmat.backends import default_backend + from cryptography.exceptions import InvalidSignature + + has_crypto = True +except ImportError: + has_crypto = False + +requires_cryptography = set(['RS256', 'RS384', 'RS512', 'ES256', 'ES384', + 'ES521', 'ES512', 'PS256', 'PS384', 'PS512']) + + +def get_default_algorithms(): + """ + Returns the algorithms that are implemented by the library. + """ + default_algorithms = { + 'none': NoneAlgorithm(), + 'HS256': HMACAlgorithm(HMACAlgorithm.SHA256), + 'HS384': HMACAlgorithm(HMACAlgorithm.SHA384), + 'HS512': HMACAlgorithm(HMACAlgorithm.SHA512) + } + + if has_crypto: + default_algorithms.update({ + 'RS256': RSAAlgorithm(RSAAlgorithm.SHA256), + 'RS384': RSAAlgorithm(RSAAlgorithm.SHA384), + 'RS512': RSAAlgorithm(RSAAlgorithm.SHA512), + 'ES256': ECAlgorithm(ECAlgorithm.SHA256), + 'ES384': ECAlgorithm(ECAlgorithm.SHA384), + 'ES521': ECAlgorithm(ECAlgorithm.SHA512), + 'ES512': ECAlgorithm(ECAlgorithm.SHA512), # Backward compat for #219 fix + 'PS256': RSAPSSAlgorithm(RSAPSSAlgorithm.SHA256), + 'PS384': RSAPSSAlgorithm(RSAPSSAlgorithm.SHA384), + 'PS512': RSAPSSAlgorithm(RSAPSSAlgorithm.SHA512) + }) + + return default_algorithms + + +class Algorithm(object): + """ + The interface for an algorithm used to sign and verify tokens. + """ + def prepare_key(self, key): + """ + Performs necessary validation and conversions on the key and returns + the key value in the proper format for sign() and verify(). + """ + raise NotImplementedError + + def sign(self, msg, key): + """ + Returns a digital signature for the specified message + using the specified key value. + """ + raise NotImplementedError + + def verify(self, msg, key, sig): + """ + Verifies that the specified digital signature is valid + for the specified message and key values. + """ + raise NotImplementedError + + @staticmethod + def to_jwk(key_obj): + """ + Serializes a given RSA key into a JWK + """ + raise NotImplementedError + + @staticmethod + def from_jwk(jwk): + """ + Deserializes a given RSA key from JWK back into a PublicKey or PrivateKey object + """ + raise NotImplementedError + + +class NoneAlgorithm(Algorithm): + """ + Placeholder for use when no signing or verification + operations are required. + """ + def prepare_key(self, key): + if key == '': + key = None + + if key is not None: + raise InvalidKeyError('When alg = "none", key value must be None.') + + return key + + def sign(self, msg, key): + return b'' + + def verify(self, msg, key, sig): + return False + + +class HMACAlgorithm(Algorithm): + """ + Performs signing and verification operations using HMAC + and the specified hash function. + """ + SHA256 = hashlib.sha256 + SHA384 = hashlib.sha384 + SHA512 = hashlib.sha512 + + def __init__(self, hash_alg): + self.hash_alg = hash_alg + + def prepare_key(self, key): + key = force_bytes(key) + + invalid_strings = [ + b'-----BEGIN PUBLIC KEY-----', + b'-----BEGIN CERTIFICATE-----', + b'-----BEGIN RSA PUBLIC KEY-----', + b'ssh-rsa' + ] + + if any([string_value in key for string_value in invalid_strings]): + raise InvalidKeyError( + 'The specified key is an asymmetric key or x509 certificate and' + ' should not be used as an HMAC secret.') + + return key + + @staticmethod + def to_jwk(key_obj): + return json.dumps({ + 'k': force_unicode(base64url_encode(force_bytes(key_obj))), + 'kty': 'oct' + }) + + @staticmethod + def from_jwk(jwk): + obj = json.loads(jwk) + + if obj.get('kty') != 'oct': + raise InvalidKeyError('Not an HMAC key') + + return base64url_decode(obj['k']) + + def sign(self, msg, key): + return hmac.new(key, msg, self.hash_alg).digest() + + def verify(self, msg, key, sig): + return constant_time_compare(sig, self.sign(msg, key)) + + +if has_crypto: + + class RSAAlgorithm(Algorithm): + """ + Performs signing and verification operations using + RSASSA-PKCS-v1_5 and the specified hash function. + """ + SHA256 = hashes.SHA256 + SHA384 = hashes.SHA384 + SHA512 = hashes.SHA512 + + def __init__(self, hash_alg): + self.hash_alg = hash_alg + + def prepare_key(self, key): + if isinstance(key, RSAPrivateKey) or \ + isinstance(key, RSAPublicKey): + return key + + if isinstance(key, string_types): + key = force_bytes(key) + + try: + if key.startswith(b'ssh-rsa'): + key = load_ssh_public_key(key, backend=default_backend()) + else: + key = load_pem_private_key(key, password=None, backend=default_backend()) + except ValueError: + key = load_pem_public_key(key, backend=default_backend()) + else: + raise TypeError('Expecting a PEM-formatted key.') + + return key + + @staticmethod + def to_jwk(key_obj): + obj = None + + if getattr(key_obj, 'private_numbers', None): + # Private key + numbers = key_obj.private_numbers() + + obj = { + 'kty': 'RSA', + 'key_ops': ['sign'], + 'n': force_unicode(to_base64url_uint(numbers.public_numbers.n)), + 'e': force_unicode(to_base64url_uint(numbers.public_numbers.e)), + 'd': force_unicode(to_base64url_uint(numbers.d)), + 'p': force_unicode(to_base64url_uint(numbers.p)), + 'q': force_unicode(to_base64url_uint(numbers.q)), + 'dp': force_unicode(to_base64url_uint(numbers.dmp1)), + 'dq': force_unicode(to_base64url_uint(numbers.dmq1)), + 'qi': force_unicode(to_base64url_uint(numbers.iqmp)) + } + + elif getattr(key_obj, 'verify', None): + # Public key + numbers = key_obj.public_numbers() + + obj = { + 'kty': 'RSA', + 'key_ops': ['verify'], + 'n': force_unicode(to_base64url_uint(numbers.n)), + 'e': force_unicode(to_base64url_uint(numbers.e)) + } + else: + raise InvalidKeyError('Not a public or private key') + + return json.dumps(obj) + + @staticmethod + def from_jwk(jwk): + try: + obj = json.loads(jwk) + except ValueError: + raise InvalidKeyError('Key is not valid JSON') + + if obj.get('kty') != 'RSA': + raise InvalidKeyError('Not an RSA key') + + if 'd' in obj and 'e' in obj and 'n' in obj: + # Private key + if 'oth' in obj: + raise InvalidKeyError('Unsupported RSA private key: > 2 primes not supported') + + other_props = ['p', 'q', 'dp', 'dq', 'qi'] + props_found = [prop in obj for prop in other_props] + any_props_found = any(props_found) + + if any_props_found and not all(props_found): + raise InvalidKeyError('RSA key must include all parameters if any are present besides d') + + public_numbers = RSAPublicNumbers( + from_base64url_uint(obj['e']), from_base64url_uint(obj['n']) + ) + + if any_props_found: + numbers = RSAPrivateNumbers( + d=from_base64url_uint(obj['d']), + p=from_base64url_uint(obj['p']), + q=from_base64url_uint(obj['q']), + dmp1=from_base64url_uint(obj['dp']), + dmq1=from_base64url_uint(obj['dq']), + iqmp=from_base64url_uint(obj['qi']), + public_numbers=public_numbers + ) + else: + d = from_base64url_uint(obj['d']) + p, q = rsa_recover_prime_factors( + public_numbers.n, d, public_numbers.e + ) + + numbers = RSAPrivateNumbers( + d=d, + p=p, + q=q, + dmp1=rsa_crt_dmp1(d, p), + dmq1=rsa_crt_dmq1(d, q), + iqmp=rsa_crt_iqmp(p, q), + public_numbers=public_numbers + ) + + return numbers.private_key(default_backend()) + elif 'n' in obj and 'e' in obj: + # Public key + numbers = RSAPublicNumbers( + from_base64url_uint(obj['e']), from_base64url_uint(obj['n']) + ) + + return numbers.public_key(default_backend()) + else: + raise InvalidKeyError('Not a public or private key') + + def sign(self, msg, key): + return key.sign(msg, padding.PKCS1v15(), self.hash_alg()) + + def verify(self, msg, key, sig): + try: + key.verify(sig, msg, padding.PKCS1v15(), self.hash_alg()) + return True + except InvalidSignature: + return False + + class ECAlgorithm(Algorithm): + """ + Performs signing and verification operations using + ECDSA and the specified hash function + """ + SHA256 = hashes.SHA256 + SHA384 = hashes.SHA384 + SHA512 = hashes.SHA512 + + def __init__(self, hash_alg): + self.hash_alg = hash_alg + + def prepare_key(self, key): + if isinstance(key, EllipticCurvePrivateKey) or \ + isinstance(key, EllipticCurvePublicKey): + return key + + if isinstance(key, string_types): + key = force_bytes(key) + + # Attempt to load key. We don't know if it's + # a Signing Key or a Verifying Key, so we try + # the Verifying Key first. + try: + if key.startswith(b'ecdsa-sha2-'): + key = load_ssh_public_key(key, backend=default_backend()) + else: + key = load_pem_public_key(key, backend=default_backend()) + except ValueError: + key = load_pem_private_key(key, password=None, backend=default_backend()) + + else: + raise TypeError('Expecting a PEM-formatted key.') + + return key + + def sign(self, msg, key): + der_sig = key.sign(msg, ec.ECDSA(self.hash_alg())) + + return der_to_raw_signature(der_sig, key.curve) + + def verify(self, msg, key, sig): + try: + der_sig = raw_to_der_signature(sig, key.curve) + except ValueError: + return False + + try: + key.verify(der_sig, msg, ec.ECDSA(self.hash_alg())) + return True + except InvalidSignature: + return False + + class RSAPSSAlgorithm(RSAAlgorithm): + """ + Performs a signature using RSASSA-PSS with MGF1 + """ + + def sign(self, msg, key): + return key.sign( + msg, + padding.PSS( + mgf=padding.MGF1(self.hash_alg()), + salt_length=self.hash_alg.digest_size + ), + self.hash_alg() + ) + + def verify(self, msg, key, sig): + try: + key.verify( + sig, + msg, + padding.PSS( + mgf=padding.MGF1(self.hash_alg()), + salt_length=self.hash_alg.digest_size + ), + self.hash_alg() + ) + return True + except InvalidSignature: + return False diff --git a/contrib/python/PyJWT/py2/jwt/api_jws.py b/contrib/python/PyJWT/py2/jwt/api_jws.py new file mode 100644 index 00000000000..a9354adb06c --- /dev/null +++ b/contrib/python/PyJWT/py2/jwt/api_jws.py @@ -0,0 +1,242 @@ +import binascii +import json +import warnings +try: + # import required by mypy to perform type checking, not used for normal execution + from typing import Callable, Dict, List, Optional, Union # NOQA +except ImportError: + pass + +from .algorithms import ( + Algorithm, get_default_algorithms, has_crypto, requires_cryptography # NOQA +) +from .compat import Mapping, binary_type, string_types, text_type +from .exceptions import ( + DecodeError, InvalidAlgorithmError, InvalidSignatureError, + InvalidTokenError +) +from .utils import base64url_decode, base64url_encode, force_bytes, merge_dict + + +class PyJWS(object): + header_typ = 'JWT' + + def __init__(self, algorithms=None, options=None): + self._algorithms = get_default_algorithms() + self._valid_algs = (set(algorithms) if algorithms is not None + else set(self._algorithms)) + + # Remove algorithms that aren't on the whitelist + for key in list(self._algorithms.keys()): + if key not in self._valid_algs: + del self._algorithms[key] + + if not options: + options = {} + + self.options = merge_dict(self._get_default_options(), options) + + @staticmethod + def _get_default_options(): + return { + 'verify_signature': True + } + + def register_algorithm(self, alg_id, alg_obj): + """ + Registers a new Algorithm for use when creating and verifying tokens. + """ + if alg_id in self._algorithms: + raise ValueError('Algorithm already has a handler.') + + if not isinstance(alg_obj, Algorithm): + raise TypeError('Object is not of type `Algorithm`') + + self._algorithms[alg_id] = alg_obj + self._valid_algs.add(alg_id) + + def unregister_algorithm(self, alg_id): + """ + Unregisters an Algorithm for use when creating and verifying tokens + Throws KeyError if algorithm is not registered. + """ + if alg_id not in self._algorithms: + raise KeyError('The specified algorithm could not be removed' + ' because it is not registered.') + + del self._algorithms[alg_id] + self._valid_algs.remove(alg_id) + + def get_algorithms(self): + """ + Returns a list of supported values for the 'alg' parameter. + """ + return list(self._valid_algs) + + def encode(self, + payload, # type: Union[Dict, bytes] + key, # type: str + algorithm='HS256', # type: str + headers=None, # type: Optional[Dict] + json_encoder=None # type: Optional[Callable] + ): + segments = [] + + if algorithm is None: + algorithm = 'none' + + if algorithm not in self._valid_algs: + pass + + # Header + header = {'typ': self.header_typ, 'alg': algorithm} + + if headers: + self._validate_headers(headers) + header.update(headers) + + json_header = force_bytes( + json.dumps( + header, + separators=(',', ':'), + cls=json_encoder + ) + ) + + segments.append(base64url_encode(json_header)) + segments.append(base64url_encode(payload)) + + # Segments + signing_input = b'.'.join(segments) + try: + alg_obj = self._algorithms[algorithm] + key = alg_obj.prepare_key(key) + signature = alg_obj.sign(signing_input, key) + + except KeyError: + if not has_crypto and algorithm in requires_cryptography: + raise NotImplementedError( + "Algorithm '%s' could not be found. Do you have cryptography " + "installed?" % algorithm + ) + else: + raise NotImplementedError('Algorithm not supported') + + segments.append(base64url_encode(signature)) + + return b'.'.join(segments) + + def decode(self, + jwt, # type: str + key='', # type: str + verify=True, # type: bool + algorithms=None, # type: List[str] + options=None, # type: Dict + **kwargs): + + merged_options = merge_dict(self.options, options) + verify_signature = merged_options['verify_signature'] + + if verify_signature and not algorithms: + warnings.warn( + 'It is strongly recommended that you pass in a ' + + 'value for the "algorithms" argument when calling decode(). ' + + 'This argument will be mandatory in a future version.', + DeprecationWarning + ) + + payload, signing_input, header, signature = self._load(jwt) + + if not verify: + warnings.warn('The verify parameter is deprecated. ' + 'Please use verify_signature in options instead.', + DeprecationWarning, stacklevel=2) + elif verify_signature: + self._verify_signature(payload, signing_input, header, signature, + key, algorithms) + + return payload + + def get_unverified_header(self, jwt): + """Returns back the JWT header parameters as a dict() + + Note: The signature is not verified so the header parameters + should not be fully trusted until signature verification is complete + """ + headers = self._load(jwt)[2] + self._validate_headers(headers) + + return headers + + def _load(self, jwt): + if isinstance(jwt, text_type): + jwt = jwt.encode('utf-8') + + if not issubclass(type(jwt), binary_type): + raise DecodeError("Invalid token type. Token must be a {0}".format( + binary_type)) + + try: + signing_input, crypto_segment = jwt.rsplit(b'.', 1) + header_segment, payload_segment = signing_input.split(b'.', 1) + except ValueError: + raise DecodeError('Not enough segments') + + try: + header_data = base64url_decode(header_segment) + except (TypeError, binascii.Error): + raise DecodeError('Invalid header padding') + + try: + header = json.loads(header_data.decode('utf-8')) + except ValueError as e: + raise DecodeError('Invalid header string: %s' % e) + + if not isinstance(header, Mapping): + raise DecodeError('Invalid header string: must be a json object') + + try: + payload = base64url_decode(payload_segment) + except (TypeError, binascii.Error): + raise DecodeError('Invalid payload padding') + + try: + signature = base64url_decode(crypto_segment) + except (TypeError, binascii.Error): + raise DecodeError('Invalid crypto padding') + + return (payload, signing_input, header, signature) + + def _verify_signature(self, payload, signing_input, header, signature, + key='', algorithms=None): + + alg = header.get('alg') + + if algorithms is not None and alg not in algorithms: + raise InvalidAlgorithmError('The specified alg value is not allowed') + + try: + alg_obj = self._algorithms[alg] + key = alg_obj.prepare_key(key) + + if not alg_obj.verify(signing_input, key, signature): + raise InvalidSignatureError('Signature verification failed') + + except KeyError: + raise InvalidAlgorithmError('Algorithm not supported') + + def _validate_headers(self, headers): + if 'kid' in headers: + self._validate_kid(headers['kid']) + + def _validate_kid(self, kid): + if not isinstance(kid, string_types): + raise InvalidTokenError('Key ID header parameter must be a string') + + +_jws_global_obj = PyJWS() +encode = _jws_global_obj.encode +decode = _jws_global_obj.decode +register_algorithm = _jws_global_obj.register_algorithm +unregister_algorithm = _jws_global_obj.unregister_algorithm +get_unverified_header = _jws_global_obj.get_unverified_header diff --git a/contrib/python/PyJWT/py2/jwt/api_jwt.py b/contrib/python/PyJWT/py2/jwt/api_jwt.py new file mode 100644 index 00000000000..85504acf930 --- /dev/null +++ b/contrib/python/PyJWT/py2/jwt/api_jwt.py @@ -0,0 +1,222 @@ +import json +import warnings +from calendar import timegm +from datetime import datetime, timedelta +try: + # import required by mypy to perform type checking, not used for normal execution + from typing import Callable, Dict, List, Optional, Union # NOQA +except ImportError: + pass + +from .api_jws import PyJWS +from .algorithms import Algorithm, get_default_algorithms # NOQA +from .compat import Iterable, Mapping, string_types +from .exceptions import ( + DecodeError, ExpiredSignatureError, ImmatureSignatureError, + InvalidAudienceError, InvalidIssuedAtError, + InvalidIssuerError, MissingRequiredClaimError +) +from .utils import merge_dict + + +class PyJWT(PyJWS): + header_type = 'JWT' + + @staticmethod + def _get_default_options(): + # type: () -> Dict[str, bool] + return { + 'verify_signature': True, + 'verify_exp': True, + 'verify_nbf': True, + 'verify_iat': True, + 'verify_aud': True, + 'verify_iss': True, + 'require_exp': False, + 'require_iat': False, + 'require_nbf': False + } + + def encode(self, + payload, # type: Union[Dict, bytes] + key, # type: str + algorithm='HS256', # type: str + headers=None, # type: Optional[Dict] + json_encoder=None # type: Optional[Callable] + ): + # Check that we get a mapping + if not isinstance(payload, Mapping): + raise TypeError('Expecting a mapping object, as JWT only supports ' + 'JSON objects as payloads.') + + # Payload + for time_claim in ['exp', 'iat', 'nbf']: + # Convert datetime to a intDate value in known time-format claims + if isinstance(payload.get(time_claim), datetime): + payload[time_claim] = timegm(payload[time_claim].utctimetuple()) # type: ignore + + json_payload = json.dumps( + payload, + separators=(',', ':'), + cls=json_encoder + ).encode('utf-8') + + return super(PyJWT, self).encode( + json_payload, key, algorithm, headers, json_encoder + ) + + def decode(self, + jwt, # type: str + key='', # type: str + verify=True, # type: bool + algorithms=None, # type: List[str] + options=None, # type: Dict + **kwargs): + + if verify and not algorithms: + warnings.warn( + 'It is strongly recommended that you pass in a ' + + 'value for the "algorithms" argument when calling decode(). ' + + 'This argument will be mandatory in a future version.', + DeprecationWarning + ) + + payload, _, _, _ = self._load(jwt) + + if options is None: + options = {'verify_signature': verify} + else: + options.setdefault('verify_signature', verify) + + decoded = super(PyJWT, self).decode( + jwt, key=key, algorithms=algorithms, options=options, **kwargs + ) + + try: + payload = json.loads(decoded.decode('utf-8')) + except ValueError as e: + raise DecodeError('Invalid payload string: %s' % e) + if not isinstance(payload, Mapping): + raise DecodeError('Invalid payload string: must be a json object') + + if verify: + merged_options = merge_dict(self.options, options) + self._validate_claims(payload, merged_options, **kwargs) + + return payload + + def _validate_claims(self, payload, options, audience=None, issuer=None, + leeway=0, **kwargs): + + if 'verify_expiration' in kwargs: + options['verify_exp'] = kwargs.get('verify_expiration', True) + warnings.warn('The verify_expiration parameter is deprecated. ' + 'Please use verify_exp in options instead.', + DeprecationWarning) + + if isinstance(leeway, timedelta): + leeway = leeway.total_seconds() + + if not isinstance(audience, (string_types, type(None), Iterable)): + raise TypeError('audience must be a string, iterable, or None') + + self._validate_required_claims(payload, options) + + now = timegm(datetime.utcnow().utctimetuple()) + + if 'iat' in payload and options.get('verify_iat'): + self._validate_iat(payload, now, leeway) + + if 'nbf' in payload and options.get('verify_nbf'): + self._validate_nbf(payload, now, leeway) + + if 'exp' in payload and options.get('verify_exp'): + self._validate_exp(payload, now, leeway) + + if options.get('verify_iss'): + self._validate_iss(payload, issuer) + + if options.get('verify_aud'): + self._validate_aud(payload, audience) + + def _validate_required_claims(self, payload, options): + if options.get('require_exp') and payload.get('exp') is None: + raise MissingRequiredClaimError('exp') + + if options.get('require_iat') and payload.get('iat') is None: + raise MissingRequiredClaimError('iat') + + if options.get('require_nbf') and payload.get('nbf') is None: + raise MissingRequiredClaimError('nbf') + + def _validate_iat(self, payload, now, leeway): + try: + int(payload['iat']) + except ValueError: + raise InvalidIssuedAtError('Issued At claim (iat) must be an integer.') + + def _validate_nbf(self, payload, now, leeway): + try: + nbf = int(payload['nbf']) + except ValueError: + raise DecodeError('Not Before claim (nbf) must be an integer.') + + if nbf > (now + leeway): + raise ImmatureSignatureError('The token is not yet valid (nbf)') + + def _validate_exp(self, payload, now, leeway): + try: + exp = int(payload['exp']) + except ValueError: + raise DecodeError('Expiration Time claim (exp) must be an' + ' integer.') + + if exp < (now - leeway): + raise ExpiredSignatureError('Signature has expired') + + def _validate_aud(self, payload, audience): + if audience is None and 'aud' not in payload: + return + + if audience is not None and 'aud' not in payload: + # Application specified an audience, but it could not be + # verified since the token does not contain a claim. + raise MissingRequiredClaimError('aud') + + if audience is None and 'aud' in payload: + # Application did not specify an audience, but + # the token has the 'aud' claim + raise InvalidAudienceError('Invalid audience') + + audience_claims = payload['aud'] + + if isinstance(audience_claims, string_types): + audience_claims = [audience_claims] + if not isinstance(audience_claims, list): + raise InvalidAudienceError('Invalid claim format in token') + if any(not isinstance(c, string_types) for c in audience_claims): + raise InvalidAudienceError('Invalid claim format in token') + + if isinstance(audience, string_types): + audience = [audience] + + if not any(aud in audience_claims for aud in audience): + raise InvalidAudienceError('Invalid audience') + + def _validate_iss(self, payload, issuer): + if issuer is None: + return + + if 'iss' not in payload: + raise MissingRequiredClaimError('iss') + + if payload['iss'] != issuer: + raise InvalidIssuerError('Invalid issuer') + + +_jwt_global_obj = PyJWT() +encode = _jwt_global_obj.encode +decode = _jwt_global_obj.decode +register_algorithm = _jwt_global_obj.register_algorithm +unregister_algorithm = _jwt_global_obj.unregister_algorithm +get_unverified_header = _jwt_global_obj.get_unverified_header diff --git a/contrib/python/PyJWT/py2/jwt/compat.py b/contrib/python/PyJWT/py2/jwt/compat.py new file mode 100644 index 00000000000..e79e258e56d --- /dev/null +++ b/contrib/python/PyJWT/py2/jwt/compat.py @@ -0,0 +1,68 @@ +""" +The `compat` module provides support for backwards compatibility with older +versions of python, and compatibility wrappers around optional packages. +""" +# flake8: noqa +import hmac +import struct +import sys + + +PY3 = sys.version_info[0] == 3 + + +if PY3: + text_type = str + binary_type = bytes +else: + text_type = unicode + binary_type = str + +string_types = (text_type, binary_type) + +try: + # Importing ABCs from collections will be removed in PY3.8 + from collections.abc import Iterable, Mapping +except ImportError: + from collections import Iterable, Mapping + +try: + constant_time_compare = hmac.compare_digest +except AttributeError: + # Fallback for Python < 2.7 + def constant_time_compare(val1, val2): + """ + Returns True if the two strings are equal, False otherwise. + + The time taken is independent of the number of characters that match. + """ + if len(val1) != len(val2): + return False + + result = 0 + + for x, y in zip(val1, val2): + result |= ord(x) ^ ord(y) + + return result == 0 + +# Use int.to_bytes if it exists (Python 3) +if getattr(int, 'to_bytes', None): + def bytes_from_int(val): + remaining = val + byte_length = 0 + + while remaining != 0: + remaining = remaining >> 8 + byte_length += 1 + + return val.to_bytes(byte_length, 'big', signed=False) +else: + def bytes_from_int(val): + buf = [] + while val: + val, remainder = divmod(val, 256) + buf.append(remainder) + + buf.reverse() + return struct.pack('%sB' % len(buf), *buf) diff --git a/contrib/python/tzlocal/py3/tzlocal/py.typed b/contrib/python/PyJWT/py2/jwt/contrib/__init__.py index e69de29bb2d..e69de29bb2d 100644 --- a/contrib/python/tzlocal/py3/tzlocal/py.typed +++ b/contrib/python/PyJWT/py2/jwt/contrib/__init__.py diff --git a/contrib/python/PyJWT/py2/jwt/contrib/algorithms/__init__.py b/contrib/python/PyJWT/py2/jwt/contrib/algorithms/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/contrib/python/PyJWT/py2/jwt/contrib/algorithms/__init__.py diff --git a/contrib/python/PyJWT/py2/jwt/contrib/algorithms/py_ecdsa.py b/contrib/python/PyJWT/py2/jwt/contrib/algorithms/py_ecdsa.py new file mode 100644 index 00000000000..bf0dea5ae28 --- /dev/null +++ b/contrib/python/PyJWT/py2/jwt/contrib/algorithms/py_ecdsa.py @@ -0,0 +1,60 @@ +# Note: This file is named py_ecdsa.py because import behavior in Python 2 +# would cause ecdsa.py to squash the ecdsa library that it depends upon. + +import hashlib + +import ecdsa + +from jwt.algorithms import Algorithm +from jwt.compat import string_types, text_type + + +class ECAlgorithm(Algorithm): + """ + Performs signing and verification operations using + ECDSA and the specified hash function + + This class requires the ecdsa package to be installed. + + This is based off of the implementation in PyJWT 0.3.2 + """ + SHA256 = hashlib.sha256 + SHA384 = hashlib.sha384 + SHA512 = hashlib.sha512 + + def __init__(self, hash_alg): + self.hash_alg = hash_alg + + def prepare_key(self, key): + + if isinstance(key, ecdsa.SigningKey) or \ + isinstance(key, ecdsa.VerifyingKey): + return key + + if isinstance(key, string_types): + if isinstance(key, text_type): + key = key.encode('utf-8') + + # Attempt to load key. We don't know if it's + # a Signing Key or a Verifying Key, so we try + # the Verifying Key first. + try: + key = ecdsa.VerifyingKey.from_pem(key) + except ecdsa.der.UnexpectedDER: + key = ecdsa.SigningKey.from_pem(key) + + else: + raise TypeError('Expecting a PEM-formatted key.') + + return key + + def sign(self, msg, key): + return key.sign(msg, hashfunc=self.hash_alg, + sigencode=ecdsa.util.sigencode_string) + + def verify(self, msg, key, sig): + try: + return key.verify(sig, msg, hashfunc=self.hash_alg, + sigdecode=ecdsa.util.sigdecode_string) + except AssertionError: + return False diff --git a/contrib/python/PyJWT/py2/jwt/contrib/algorithms/pycrypto.py b/contrib/python/PyJWT/py2/jwt/contrib/algorithms/pycrypto.py new file mode 100644 index 00000000000..e49cdbfe40f --- /dev/null +++ b/contrib/python/PyJWT/py2/jwt/contrib/algorithms/pycrypto.py @@ -0,0 +1,46 @@ +import Crypto.Hash.SHA256 +import Crypto.Hash.SHA384 +import Crypto.Hash.SHA512 +from Crypto.PublicKey import RSA +from Crypto.Signature import PKCS1_v1_5 + +from jwt.algorithms import Algorithm +from jwt.compat import string_types, text_type + + +class RSAAlgorithm(Algorithm): + """ + Performs signing and verification operations using + RSASSA-PKCS-v1_5 and the specified hash function. + + This class requires PyCrypto package to be installed. + + This is based off of the implementation in PyJWT 0.3.2 + """ + SHA256 = Crypto.Hash.SHA256 + SHA384 = Crypto.Hash.SHA384 + SHA512 = Crypto.Hash.SHA512 + + def __init__(self, hash_alg): + self.hash_alg = hash_alg + + def prepare_key(self, key): + + if isinstance(key, RSA._RSAobj): + return key + + if isinstance(key, string_types): + if isinstance(key, text_type): + key = key.encode('utf-8') + + key = RSA.importKey(key) + else: + raise TypeError('Expecting a PEM- or RSA-formatted key.') + + return key + + def sign(self, msg, key): + return PKCS1_v1_5.new(key).sign(self.hash_alg.new(msg)) + + def verify(self, msg, key, sig): + return PKCS1_v1_5.new(key).verify(self.hash_alg.new(msg), sig) diff --git a/contrib/python/PyJWT/py2/jwt/exceptions.py b/contrib/python/PyJWT/py2/jwt/exceptions.py new file mode 100644 index 00000000000..2a6aa596ba0 --- /dev/null +++ b/contrib/python/PyJWT/py2/jwt/exceptions.py @@ -0,0 +1,59 @@ +class PyJWTError(Exception): + """ + Base class for all exceptions + """ + pass + + +class InvalidTokenError(PyJWTError): + pass + + +class DecodeError(InvalidTokenError): + pass + + +class InvalidSignatureError(DecodeError): + pass + + +class ExpiredSignatureError(InvalidTokenError): + pass + + +class InvalidAudienceError(InvalidTokenError): + pass + + +class InvalidIssuerError(InvalidTokenError): + pass + + +class InvalidIssuedAtError(InvalidTokenError): + pass + + +class ImmatureSignatureError(InvalidTokenError): + pass + + +class InvalidKeyError(PyJWTError): + pass + + +class InvalidAlgorithmError(InvalidTokenError): + pass + + +class MissingRequiredClaimError(InvalidTokenError): + def __init__(self, claim): + self.claim = claim + + def __str__(self): + return 'Token is missing the "%s" claim' % self.claim + + +# Compatibility aliases (deprecated) +ExpiredSignature = ExpiredSignatureError +InvalidAudience = InvalidAudienceError +InvalidIssuer = InvalidIssuerError diff --git a/contrib/python/PyJWT/py2/jwt/help.py b/contrib/python/PyJWT/py2/jwt/help.py new file mode 100644 index 00000000000..55e39ebb271 --- /dev/null +++ b/contrib/python/PyJWT/py2/jwt/help.py @@ -0,0 +1,61 @@ +from __future__ import print_function + +import json +import platform +import sys + +from . import __version__ as pyjwt_version + +try: + import cryptography +except ImportError: + cryptography = None + +try: + import ecdsa +except ImportError: + ecdsa = None + + +def info(): + """ + Generate information for a bug report. + Based on the requests package help utility module. + """ + try: + platform_info = {"system": platform.system(), "release": platform.release()} + except IOError: + platform_info = {"system": "Unknown", "release": "Unknown"} + + implementation = platform.python_implementation() + + if implementation == "CPython": + implementation_version = platform.python_version() + elif implementation == "PyPy": + implementation_version = "%s.%s.%s" % ( + sys.pypy_version_info.major, + sys.pypy_version_info.minor, + sys.pypy_version_info.micro, + ) + if sys.pypy_version_info.releaselevel != "final": + implementation_version = "".join( + [implementation_version, sys.pypy_version_info.releaselevel] + ) + else: + implementation_version = "Unknown" + + return { + "platform": platform_info, + "implementation": {"name": implementation, "version": implementation_version}, + "cryptography": {"version": getattr(cryptography, "__version__", "")}, + "pyjwt": {"version": pyjwt_version}, + } + + +def main(): + """Pretty-print the bug information as JSON.""" + print(json.dumps(info(), sort_keys=True, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/contrib/python/PyJWT/py2/jwt/utils.py b/contrib/python/PyJWT/py2/jwt/utils.py new file mode 100644 index 00000000000..b33c7a2d455 --- /dev/null +++ b/contrib/python/PyJWT/py2/jwt/utils.py @@ -0,0 +1,113 @@ +import base64 +import binascii +import struct + +from .compat import binary_type, bytes_from_int, text_type + +try: + from cryptography.hazmat.primitives.asymmetric.utils import ( + decode_dss_signature, encode_dss_signature + ) +except ImportError: + pass + + +def force_unicode(value): + if isinstance(value, binary_type): + return value.decode('utf-8') + elif isinstance(value, text_type): + return value + else: + raise TypeError('Expected a string value') + + +def force_bytes(value): + if isinstance(value, text_type): + return value.encode('utf-8') + elif isinstance(value, binary_type): + return value + else: + raise TypeError('Expected a string value') + + +def base64url_decode(input): + if isinstance(input, text_type): + input = input.encode('ascii') + + rem = len(input) % 4 + + if rem > 0: + input += b'=' * (4 - rem) + + return base64.urlsafe_b64decode(input) + + +def base64url_encode(input): + return base64.urlsafe_b64encode(input).replace(b'=', b'') + + +def to_base64url_uint(val): + if val < 0: + raise ValueError('Must be a positive integer') + + int_bytes = bytes_from_int(val) + + if len(int_bytes) == 0: + int_bytes = b'\x00' + + return base64url_encode(int_bytes) + + +def from_base64url_uint(val): + if isinstance(val, text_type): + val = val.encode('ascii') + + data = base64url_decode(val) + + buf = struct.unpack('%sB' % len(data), data) + return int(''.join(["%02x" % byte for byte in buf]), 16) + + +def merge_dict(original, updates): + if not updates: + return original + + try: + merged_options = original.copy() + merged_options.update(updates) + except (AttributeError, ValueError) as e: + raise TypeError('original and updates must be a dictionary: %s' % e) + + return merged_options + + +def number_to_bytes(num, num_bytes): + padded_hex = '%0*x' % (2 * num_bytes, num) + big_endian = binascii.a2b_hex(padded_hex.encode('ascii')) + return big_endian + + +def bytes_to_number(string): + return int(binascii.b2a_hex(string), 16) + + +def der_to_raw_signature(der_sig, curve): + num_bits = curve.key_size + num_bytes = (num_bits + 7) // 8 + + r, s = decode_dss_signature(der_sig) + + return number_to_bytes(r, num_bytes) + number_to_bytes(s, num_bytes) + + +def raw_to_der_signature(raw_sig, curve): + num_bits = curve.key_size + num_bytes = (num_bits + 7) // 8 + + if len(raw_sig) != 2 * num_bytes: + raise ValueError('Invalid signature') + + r = bytes_to_number(raw_sig[:num_bytes]) + s = bytes_to_number(raw_sig[num_bytes:]) + + return encode_dss_signature(r, s) diff --git a/contrib/python/PyJWT/py2/ya.make b/contrib/python/PyJWT/py2/ya.make new file mode 100644 index 00000000000..57a9352fba1 --- /dev/null +++ b/contrib/python/PyJWT/py2/ya.make @@ -0,0 +1,43 @@ +# Generated by devtools/yamaker (pypi). + +PY2_LIBRARY() + +VERSION(1.7.1) + +LICENSE(MIT) + +PEERDIR( + contrib/python/cryptography +) + +NO_LINT() + +NO_CHECK_IMPORTS( + jwt.contrib.* +) + +PY_SRCS( + TOP_LEVEL + jwt/__init__.py + jwt/__main__.py + jwt/algorithms.py + jwt/api_jws.py + jwt/api_jwt.py + jwt/compat.py + jwt/contrib/__init__.py + jwt/contrib/algorithms/__init__.py + jwt/contrib/algorithms/py_ecdsa.py + jwt/contrib/algorithms/pycrypto.py + jwt/exceptions.py + jwt/help.py + jwt/utils.py +) + +RESOURCE_FILES( + PREFIX contrib/python/PyJWT/py2/ + .dist-info/METADATA + .dist-info/entry_points.txt + .dist-info/top_level.txt +) + +END() diff --git a/contrib/python/cffi/py2/.dist-info/METADATA b/contrib/python/cffi/py2/.dist-info/METADATA index 538e679147a..abe2ac79bb3 100644 --- a/contrib/python/cffi/py2/.dist-info/METADATA +++ b/contrib/python/cffi/py2/.dist-info/METADATA @@ -6,6 +6,7 @@ Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski Author-email: [email protected] License: MIT +Platform: UNKNOWN Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2 Classifier: Programming Language :: Python :: 2.7 @@ -18,10 +19,8 @@ Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: License :: OSI Approved :: MIT License -License-File: LICENSE Requires-Dist: pycparser - CFFI ==== @@ -32,3 +31,5 @@ Contact ------- `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_ + + diff --git a/contrib/python/cffi/py2/.dist-info/entry_points.txt b/contrib/python/cffi/py2/.dist-info/entry_points.txt index 4b0274f2333..eee7e0fb1f4 100644 --- a/contrib/python/cffi/py2/.dist-info/entry_points.txt +++ b/contrib/python/cffi/py2/.dist-info/entry_points.txt @@ -1,2 +1,3 @@ [distutils.setup_keywords] cffi_modules = cffi.setuptools_ext:cffi_modules + diff --git a/contrib/python/future/py2/future/backports/http/cookies.py b/contrib/python/future/py2/future/backports/http/cookies.py index 934fa56a41f..8bb61e22c4b 100644 --- a/contrib/python/future/py2/future/backports/http/cookies.py +++ b/contrib/python/future/py2/future/backports/http/cookies.py @@ -461,7 +461,7 @@ _CookiePattern = re.compile(r""" )? # End of optional value group \s* # Any number of spaces. (\s+|;|$) # Ending either at space, semicolon, or EOS. - """, re.ASCII | re.VERBOSE) # May be removed if safe. + """, re.ASCII) # May be removed if safe. # At long last, here is the cookie class. Using this class is almost just like diff --git a/contrib/python/future/py2/future/backports/xmlrpc/client.py b/contrib/python/future/py2/future/backports/xmlrpc/client.py index 3f0cae9b00f..b0b8f5e19ef 100644 --- a/contrib/python/future/py2/future/backports/xmlrpc/client.py +++ b/contrib/python/future/py2/future/backports/xmlrpc/client.py @@ -133,12 +133,11 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) from future.builtins import bytes, dict, int, range, str -import sys import base64 -if sys.version_info[0] < 3: - # Py2.7 compatibility hack - base64.encodebytes = base64.encodestring - base64.decodebytes = base64.decodestring +# Py2.7 compatibility hack +base64.encodebytes = base64.encodestring +base64.decodebytes = base64.decodestring +import sys import time from datetime import datetime from future.backports.http import client as http_client diff --git a/contrib/python/future/py2/future/moves/_dummy_thread.py b/contrib/python/future/py2/future/moves/_dummy_thread.py index e5dca348fbd..688d249bbe7 100644 --- a/contrib/python/future/py2/future/moves/_dummy_thread.py +++ b/contrib/python/future/py2/future/moves/_dummy_thread.py @@ -2,10 +2,7 @@ from __future__ import absolute_import from future.utils import PY3 if PY3: - try: - from _dummy_thread import * - except ImportError: - from _thread import * + from _dummy_thread import * else: __future_module__ = True from dummy_thread import * diff --git a/contrib/python/future/py2/future/standard_library/__init__.py b/contrib/python/future/py2/future/standard_library/__init__.py index 41c4f36df25..cff02f95943 100644 --- a/contrib/python/future/py2/future/standard_library/__init__.py +++ b/contrib/python/future/py2/future/standard_library/__init__.py @@ -125,7 +125,7 @@ RENAMES = { # 'Tkinter': 'tkinter', '_winreg': 'winreg', 'thread': '_thread', - 'dummy_thread': '_dummy_thread' if sys.version_info < (3, 9) else '_thread', + 'dummy_thread': '_dummy_thread', # 'anydbm': 'dbm', # causes infinite import loop # 'whichdb': 'dbm', # causes infinite import loop # anydbm and whichdb are handled by fix_imports2 diff --git a/contrib/python/google-auth/py3/.dist-info/METADATA b/contrib/python/google-auth/py3/.dist-info/METADATA index b4012b87e36..e0785656a45 100644 --- a/contrib/python/google-auth/py3/.dist-info/METADATA +++ b/contrib/python/google-auth/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: google-auth -Version: 2.28.2 +Version: 2.29.0 Summary: Google Authentication Library Home-page: https://github.com/googleapis/google-auth-library-python Author: Google Cloud Platform diff --git a/contrib/python/google-auth/py3/google/auth/aws.py b/contrib/python/google-auth/py3/google/auth/aws.py index 6e0e4e864f1..28c065d3c75 100644 --- a/contrib/python/google-auth/py3/google/auth/aws.py +++ b/contrib/python/google-auth/py3/google/auth/aws.py @@ -21,10 +21,11 @@ of long-live service account private keys. AWS Credentials are initialized using external_account arguments which are typically loaded from the external credentials JSON file. -Unlike other Credentials that can be initialized with a list of explicit -arguments, secrets or credentials, external account clients use the -environment and hints/guidelines provided by the external_account JSON -file to retrieve credentials and exchange them for Google access tokens. + +This module also provides a definition for an abstract AWS security credentials supplier. +This supplier can be implemented to return valid AWS security credentials and an AWS region +and used to create AWS credentials. The credentials will then call the +supplier instead of using pre-defined methods such as calling the EC2 metadata endpoints. This module also provides a basic implementation of the `AWS Signature Version 4`_ request signing algorithm. @@ -37,6 +38,8 @@ via the GCP STS endpoint. .. _AWS STS GetCallerIdentity: https://docs.aws.amazon.com/STS/latest/APIReference/API_GetCallerIdentity.html """ +import abc +from dataclasses import dataclass import hashlib import hmac import http.client as http_client @@ -44,6 +47,7 @@ import json import os import posixpath import re +from typing import Optional import urllib from urllib.parse import urljoin @@ -61,6 +65,12 @@ _AWS_REQUEST_TYPE = "aws4_request" _AWS_SECURITY_TOKEN_HEADER = "x-amz-security-token" # The AWS authorization header name for the auto-generated date. _AWS_DATE_HEADER = "x-amz-date" +# The default AWS regional credential verification URL. +_DEFAULT_AWS_REGIONAL_CREDENTIAL_VERIFICATION_URL = ( + "https://sts.{region}.amazonaws.com?Action=GetCallerIdentity&Version=2011-06-15" +) +# IMDSV2 session token lifetime. This is set to a low value because the session token is used immediately. +_IMDSV2_SESSION_TOKEN_TTL_SECONDS = "300" class RequestSigner(object): @@ -92,8 +102,7 @@ class RequestSigner(object): https://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html Args: - aws_security_credentials (Mapping[str, str]): A dictionary containing - the AWS security credentials. + aws_security_credentials (AWSSecurityCredentials): The AWS security credentials. url (str): The AWS service URL containing the canonical URI and query string. method (str): The HTTP method used to call this API. @@ -105,10 +114,6 @@ class RequestSigner(object): Returns: Mapping[str, str]: The AWS signed request dictionary object. """ - # Get AWS credentials. - access_key = aws_security_credentials.get("access_key_id") - secret_key = aws_security_credentials.get("secret_access_key") - security_token = aws_security_credentials.get("security_token") additional_headers = additional_headers or {} @@ -129,9 +134,7 @@ class RequestSigner(object): canonical_querystring=_get_canonical_querystring(uri.query), method=method, region=self._region_name, - access_key=access_key, - secret_key=secret_key, - security_token=security_token, + aws_security_credentials=aws_security_credentials, request_payload=request_payload, additional_headers=additional_headers, ) @@ -147,8 +150,8 @@ class RequestSigner(object): headers[key] = additional_headers[key] # Add session token if available. - if security_token is not None: - headers[_AWS_SECURITY_TOKEN_HEADER] = security_token + if aws_security_credentials.session_token is not None: + headers[_AWS_SECURITY_TOKEN_HEADER] = aws_security_credentials.session_token signed_request = {"url": url, "method": method, "headers": headers} if request_payload: @@ -233,9 +236,7 @@ def _generate_authentication_header_map( canonical_querystring, method, region, - access_key, - secret_key, - security_token, + aws_security_credentials, request_payload="", additional_headers={}, ): @@ -248,10 +249,7 @@ def _generate_authentication_header_map( canonical_querystring (str): The AWS service URL query string. method (str): The HTTP method used to call this API. region (str): The AWS region. - access_key (str): The AWS access key ID. - secret_key (str): The AWS secret access key. - security_token (Optional[str]): The AWS security session token. This is - available for temporary sessions. + aws_security_credentials (AWSSecurityCredentials): The AWS security credentials. request_payload (Optional[str]): The optional request payload if available. additional_headers (Optional[Mapping[str, str]]): The optional @@ -274,8 +272,10 @@ def _generate_authentication_header_map( for key in additional_headers: full_headers[key.lower()] = additional_headers[key] # Add AWS session token if available. - if security_token is not None: - full_headers[_AWS_SECURITY_TOKEN_HEADER] = security_token + if aws_security_credentials.session_token is not None: + full_headers[ + _AWS_SECURITY_TOKEN_HEADER + ] = aws_security_credentials.session_token # Required headers full_headers["host"] = host @@ -321,14 +321,20 @@ def _generate_authentication_header_map( ) # https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html - signing_key = _get_signing_key(secret_key, date_stamp, region, service_name) + signing_key = _get_signing_key( + aws_security_credentials.secret_access_key, date_stamp, region, service_name + ) signature = hmac.new( signing_key, string_to_sign.encode("utf-8"), hashlib.sha256 ).hexdigest() # https://docs.aws.amazon.com/general/latest/gr/sigv4-add-signature-to-request.html authorization_header = "{} Credential={}/{}, SignedHeaders={}, Signature={}".format( - _AWS_ALGORITHM, access_key, credential_scope, signed_headers, signature + _AWS_ALGORITHM, + aws_security_credentials.access_key_id, + credential_scope, + signed_headers, + signature, ) authentication_header = {"authorization_header": authorization_header} @@ -338,211 +344,112 @@ def _generate_authentication_header_map( return authentication_header -class Credentials(external_account.Credentials): - """AWS external account credentials. - This is used to exchange serialized AWS signature v4 signed requests to - AWS STS GetCallerIdentity service for Google access tokens. - """ +@dataclass +class AwsSecurityCredentials: + """A class that models AWS security credentials with an optional session token. - def __init__( - self, - audience, - subject_token_type, - token_url, - credential_source=None, - *args, - **kwargs - ): - """Instantiates an AWS workload external account credentials object. - - Args: - audience (str): The STS audience field. - subject_token_type (str): The subject token type. - token_url (str): The STS endpoint URL. - credential_source (Mapping): The credential source dictionary used - to provide instructions on how to retrieve external credential - to be exchanged for Google access tokens. - args (List): Optional positional arguments passed into the underlying :meth:`~external_account.Credentials.__init__` method. - kwargs (Mapping): Optional keyword arguments passed into the underlying :meth:`~external_account.Credentials.__init__` method. - - Raises: - google.auth.exceptions.RefreshError: If an error is encountered during - access token retrieval logic. - ValueError: For invalid parameters. - - .. note:: Typically one of the helper constructors - :meth:`from_file` or - :meth:`from_info` are used instead of calling the constructor directly. - """ - super(Credentials, self).__init__( - audience=audience, - subject_token_type=subject_token_type, - token_url=token_url, - credential_source=credential_source, - *args, - **kwargs - ) - credential_source = credential_source or {} - self._environment_id = credential_source.get("environment_id") or "" - self._region_url = credential_source.get("region_url") - self._security_credentials_url = credential_source.get("url") - self._cred_verification_url = credential_source.get( - "regional_cred_verification_url" - ) - self._imdsv2_session_token_url = credential_source.get( - "imdsv2_session_token_url" - ) - self._region = None - self._request_signer = None - self._target_resource = audience - - # Get the environment ID. Currently, only one version supported (v1). - matches = re.match(r"^(aws)([\d]+)$", self._environment_id) - if matches: - env_id, env_version = matches.groups() - else: - env_id, env_version = (None, None) + Attributes: + access_key_id (str): The AWS security credentials access key id. + secret_access_key (str): The AWS security credentials secret access key. + session_token (Optional[str]): The optional AWS security credentials session token. This should be set when using temporary credentials. + """ - if env_id != "aws" or self._cred_verification_url is None: - raise exceptions.InvalidResource( - "No valid AWS 'credential_source' provided" - ) - elif int(env_version or "") != 1: - raise exceptions.InvalidValue( - "aws version '{}' is not supported in the current build.".format( - env_version - ) - ) + access_key_id: str + secret_access_key: str + session_token: Optional[str] = None - def retrieve_subject_token(self, request): - """Retrieves the subject token using the credential_source object. - The subject token is a serialized `AWS GetCallerIdentity signed request`_. - The logic is summarized as: +class AwsSecurityCredentialsSupplier(metaclass=abc.ABCMeta): + """Base class for AWS security credential suppliers. This can be implemented with custom logic to retrieve + AWS security credentials to exchange for a Google Cloud access token. The AWS external account credential does + not cache the AWS security credentials, so caching logic should be added in the implementation. + """ - Retrieve the AWS region from the AWS_REGION or AWS_DEFAULT_REGION - environment variable or from the AWS metadata server availability-zone - if not found in the environment variable. + @abc.abstractmethod + def get_aws_security_credentials(self, context, request): + """Returns the AWS security credentials for the requested context. - Check AWS credentials in environment variables. If not found, retrieve - from the AWS metadata server security-credentials endpoint. + .. warning: This is not cached by the calling Google credential, so caching logic should be implemented in the supplier. - When retrieving AWS credentials from the metadata server - security-credentials endpoint, the AWS role needs to be determined by - calling the security-credentials endpoint without any argument. Then the - credentials can be retrieved via: security-credentials/role_name + Args: + context (google.auth.externalaccount.SupplierContext): The context object + containing information about the requested audience and subject token type. + request (google.auth.transport.Request): The object used to make + HTTP requests. - Generate the signed request to AWS STS GetCallerIdentity action. + Raises: + google.auth.exceptions.RefreshError: If an error is encountered during + security credential retrieval logic. - Inject x-goog-cloud-target-resource into header and serialize the - signed request. This will be the subject-token to pass to GCP STS. + Returns: + AwsSecurityCredentials: The requested AWS security credentials. + """ + raise NotImplementedError("") - .. _AWS GetCallerIdentity signed request: - https://cloud.google.com/iam/docs/access-resources-aws#exchange-token + @abc.abstractmethod + def get_aws_region(self, context, request): + """Returns the AWS region for the requested context. Args: - request (google.auth.transport.Request): A callable used to make + context (google.auth.externalaccount.SupplierContext): The context object + containing information about the requested audience and subject token type. + request (google.auth.transport.Request): The object used to make HTTP requests. - Returns: - str: The retrieved subject token. - """ - # Fetch the session token required to make meta data endpoint calls to aws. - if ( - request is not None - and self._imdsv2_session_token_url is not None - and self._should_use_metadata_server() - ): - headers = {"X-aws-ec2-metadata-token-ttl-seconds": "300"} - imdsv2_session_token_response = request( - url=self._imdsv2_session_token_url, method="PUT", headers=headers - ) + Raises: + google.auth.exceptions.RefreshError: If an error is encountered during + region retrieval logic. - if imdsv2_session_token_response.status != 200: - raise exceptions.RefreshError( - "Unable to retrieve AWS Session Token", - imdsv2_session_token_response.data, - ) + Returns: + str: The AWS region. + """ + raise NotImplementedError("") - imdsv2_session_token = imdsv2_session_token_response.data - else: - imdsv2_session_token = None - # Initialize the request signer if not yet initialized after determining - # the current AWS region. - if self._request_signer is None: - self._region = self._get_region( - request, self._region_url, imdsv2_session_token - ) - self._request_signer = RequestSigner(self._region) +class _DefaultAwsSecurityCredentialsSupplier(AwsSecurityCredentialsSupplier): + """Default implementation of AWS security credentials supplier. Supports retrieving + credentials and region via EC2 metadata endpoints and environment variables. + """ - # Retrieve the AWS security credentials needed to generate the signed - # request. - aws_security_credentials = self._get_security_credentials( - request, imdsv2_session_token - ) - # Generate the signed request to AWS STS GetCallerIdentity API. - # Use the required regional endpoint. Otherwise, the request will fail. - request_options = self._request_signer.get_request_options( - aws_security_credentials, - self._cred_verification_url.replace("{region}", self._region), - "POST", + def __init__(self, credential_source): + self._region_url = credential_source.get("region_url") + self._security_credentials_url = credential_source.get("url") + self._imdsv2_session_token_url = credential_source.get( + "imdsv2_session_token_url" ) - # The GCP STS endpoint expects the headers to be formatted as: - # [ - # {key: 'x-amz-date', value: '...'}, - # {key: 'Authorization', value: '...'}, - # ... - # ] - # And then serialized as: - # quote(json.dumps({ - # url: '...', - # method: 'POST', - # headers: [{key: 'x-amz-date', value: '...'}, ...] - # })) - request_headers = request_options.get("headers") - # The full, canonical resource name of the workload identity pool - # provider, with or without the HTTPS prefix. - # Including this header as part of the signature is recommended to - # ensure data integrity. - request_headers["x-goog-cloud-target-resource"] = self._target_resource - # Serialize AWS signed request. - # Keeping inner keys in sorted order makes testing easier for Python - # versions <=3.5 as the stringified JSON string would have a predictable - # key order. - aws_signed_req = {} - aws_signed_req["url"] = request_options.get("url") - aws_signed_req["method"] = request_options.get("method") - aws_signed_req["headers"] = [] - # Reformat header to GCP STS expected format. - for key in sorted(request_headers.keys()): - aws_signed_req["headers"].append( - {"key": key, "value": request_headers[key]} - ) + @_helpers.copy_docstring(AwsSecurityCredentialsSupplier) + def get_aws_security_credentials(self, context, request): - return urllib.parse.quote( - json.dumps(aws_signed_req, separators=(",", ":"), sort_keys=True) + # Check environment variables for permanent credentials first. + # https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html + env_aws_access_key_id = os.environ.get(environment_vars.AWS_ACCESS_KEY_ID) + env_aws_secret_access_key = os.environ.get( + environment_vars.AWS_SECRET_ACCESS_KEY ) + # This is normally not available for permanent credentials. + env_aws_session_token = os.environ.get(environment_vars.AWS_SESSION_TOKEN) + if env_aws_access_key_id and env_aws_secret_access_key: + return AwsSecurityCredentials( + env_aws_access_key_id, env_aws_secret_access_key, env_aws_session_token + ) - def _get_region(self, request, url, imdsv2_session_token): - """Retrieves the current AWS region from either the AWS_REGION or - AWS_DEFAULT_REGION environment variable or from the AWS metadata server. + imdsv2_session_token = self._get_imdsv2_session_token(request) + role_name = self._get_metadata_role_name(request, imdsv2_session_token) - Args: - request (google.auth.transport.Request): A callable used to make - HTTP requests. - url (str): The AWS metadata server region URL. - imdsv2_session_token (str): The AWS IMDSv2 session token to be added as a - header in the requests to AWS metadata endpoint. + # Get security credentials. + credentials = self._get_metadata_security_credentials( + request, role_name, imdsv2_session_token + ) - Returns: - str: The current AWS region. + return AwsSecurityCredentials( + credentials.get("AccessKeyId"), + credentials.get("SecretAccessKey"), + credentials.get("Token"), + ) - Raises: - google.auth.exceptions.RefreshError: If an error occurs while - retrieving the AWS region. - """ + @_helpers.copy_docstring(AwsSecurityCredentialsSupplier) + def get_aws_region(self, context, request): # The AWS metadata server is not available in some AWS environments # such as AWS lambda. Instead, it is available via environment # variable. @@ -558,6 +465,7 @@ class Credentials(external_account.Credentials): raise exceptions.RefreshError("Unable to determine AWS region") headers = None + imdsv2_session_token = self._get_imdsv2_session_token(request) if imdsv2_session_token is not None: headers = {"X-aws-ec2-metadata-token": imdsv2_session_token} @@ -570,62 +478,35 @@ class Credentials(external_account.Credentials): else response.data ) - if response.status != 200: + if response.status != http_client.OK: raise exceptions.RefreshError( - "Unable to retrieve AWS region", response_body + "Unable to retrieve AWS region: {}".format(response_body) ) # This endpoint will return the region in format: us-east-2b. # Only the us-east-2 part should be used. return response_body[:-1] - def _get_security_credentials(self, request, imdsv2_session_token): - """Retrieves the AWS security credentials required for signing AWS - requests from either the AWS security credentials environment variables - or from the AWS metadata server. - - Args: - request (google.auth.transport.Request): A callable used to make - HTTP requests. - imdsv2_session_token (str): The AWS IMDSv2 session token to be added as a - header in the requests to AWS metadata endpoint. - - Returns: - Mapping[str, str]: The AWS security credentials dictionary object. - - Raises: - google.auth.exceptions.RefreshError: If an error occurs while - retrieving the AWS security credentials. - """ - - # Check environment variables for permanent credentials first. - # https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html - env_aws_access_key_id = os.environ.get(environment_vars.AWS_ACCESS_KEY_ID) - env_aws_secret_access_key = os.environ.get( - environment_vars.AWS_SECRET_ACCESS_KEY - ) - # This is normally not available for permanent credentials. - env_aws_session_token = os.environ.get(environment_vars.AWS_SESSION_TOKEN) - if env_aws_access_key_id and env_aws_secret_access_key: - return { - "access_key_id": env_aws_access_key_id, - "secret_access_key": env_aws_secret_access_key, - "security_token": env_aws_session_token, + def _get_imdsv2_session_token(self, request): + if request is not None and self._imdsv2_session_token_url is not None: + headers = { + "X-aws-ec2-metadata-token-ttl-seconds": _IMDSV2_SESSION_TOKEN_TTL_SECONDS } - # Get role name. - role_name = self._get_metadata_role_name(request, imdsv2_session_token) + imdsv2_session_token_response = request( + url=self._imdsv2_session_token_url, method="PUT", headers=headers + ) - # Get security credentials. - credentials = self._get_metadata_security_credentials( - request, role_name, imdsv2_session_token - ) + if imdsv2_session_token_response.status != http_client.OK: + raise exceptions.RefreshError( + "Unable to retrieve AWS Session Token: {}".format( + imdsv2_session_token_response.data + ) + ) - return { - "access_key_id": credentials.get("AccessKeyId"), - "secret_access_key": credentials.get("SecretAccessKey"), - "security_token": credentials.get("Token"), - } + return imdsv2_session_token_response.data + else: + return None def _get_metadata_security_credentials( self, request, role_name, imdsv2_session_token @@ -669,7 +550,7 @@ class Credentials(external_account.Credentials): if response.status != http_client.OK: raise exceptions.RefreshError( - "Unable to retrieve AWS security credentials", response_body + "Unable to retrieve AWS security credentials: {}".format(response_body) ) credentials_response = json.loads(response_body) @@ -717,35 +598,232 @@ class Credentials(external_account.Credentials): if response.status != http_client.OK: raise exceptions.RefreshError( - "Unable to retrieve AWS role name", response_body + "Unable to retrieve AWS role name {}".format(response_body) ) return response_body - def _should_use_metadata_server(self): - # The AWS region can be provided through AWS_REGION or AWS_DEFAULT_REGION. - # The metadata server should be used if it cannot be retrieved from one of - # these environment variables. - if not os.environ.get(environment_vars.AWS_REGION) and not os.environ.get( - environment_vars.AWS_DEFAULT_REGION - ): - return True - # AWS security credentials can be retrieved from the AWS_ACCESS_KEY_ID - # and AWS_SECRET_ACCESS_KEY environment variables. The metadata server - # should be used if either of these are not available. - if not os.environ.get(environment_vars.AWS_ACCESS_KEY_ID) or not os.environ.get( - environment_vars.AWS_SECRET_ACCESS_KEY +class Credentials(external_account.Credentials): + """AWS external account credentials. + This is used to exchange serialized AWS signature v4 signed requests to + AWS STS GetCallerIdentity service for Google access tokens. + """ + + def __init__( + self, + audience, + subject_token_type, + token_url=external_account._DEFAULT_TOKEN_URL, + credential_source=None, + aws_security_credentials_supplier=None, + *args, + **kwargs + ): + """Instantiates an AWS workload external account credentials object. + + Args: + audience (str): The STS audience field. + subject_token_type (str): The subject token type based on the Oauth2.0 token exchange spec. + Expected values include:: + + “urn:ietf:params:aws:token-type:aws4_request” + + token_url (Optional [str]): The STS endpoint URL. If not provided, will default to "https://sts.googleapis.com/v1/token". + credential_source (Optional [Mapping]): The credential source dictionary used + to provide instructions on how to retrieve external credential to be exchanged for Google access tokens. + Either a credential source or an AWS security credentials supplier must be provided. + + Example credential_source for AWS credential:: + + { + "environment_id": "aws1", + "regional_cred_verification_url": "https://sts.{region}.amazonaws.com?Action=GetCallerIdentity&Version=2011-06-15", + "region_url": "http://169.254.169.254/latest/meta-data/placement/availability-zone", + "url": "http://169.254.169.254/latest/meta-data/iam/security-credentials", + imdsv2_session_token_url": "http://169.254.169.254/latest/api/token" + } + + aws_security_credentials_supplier (Optional [AwsSecurityCredentialsSupplier]): Optional AWS security credentials supplier. + This will be called to supply valid AWS security credentails which will then + be exchanged for Google access tokens. Either an AWS security credentials supplier + or a credential source must be provided. + args (List): Optional positional arguments passed into the underlying :meth:`~external_account.Credentials.__init__` method. + kwargs (Mapping): Optional keyword arguments passed into the underlying :meth:`~external_account.Credentials.__init__` method. + + Raises: + google.auth.exceptions.RefreshError: If an error is encountered during + access token retrieval logic. + ValueError: For invalid parameters. + + .. note:: Typically one of the helper constructors + :meth:`from_file` or + :meth:`from_info` are used instead of calling the constructor directly. + """ + super(Credentials, self).__init__( + audience=audience, + subject_token_type=subject_token_type, + token_url=token_url, + credential_source=credential_source, + *args, + **kwargs + ) + if credential_source is None and aws_security_credentials_supplier is None: + raise exceptions.InvalidValue( + "A valid credential source or AWS security credentials supplier must be provided." + ) + if ( + credential_source is not None + and aws_security_credentials_supplier is not None ): - return True + raise exceptions.InvalidValue( + "AWS credential cannot have both a credential source and an AWS security credentials supplier." + ) + + if aws_security_credentials_supplier: + self._aws_security_credentials_supplier = aws_security_credentials_supplier + # The regional cred verification URL would normally be provided through the credential source. So set it to the default one here. + self._cred_verification_url = ( + _DEFAULT_AWS_REGIONAL_CREDENTIAL_VERIFICATION_URL + ) + else: + environment_id = credential_source.get("environment_id") or "" + self._aws_security_credentials_supplier = _DefaultAwsSecurityCredentialsSupplier( + credential_source + ) + self._cred_verification_url = credential_source.get( + "regional_cred_verification_url" + ) - return False + # Get the environment ID, i.e. "aws1". Currently, only one version supported (1). + matches = re.match(r"^(aws)([\d]+)$", environment_id) + if matches: + env_id, env_version = matches.groups() + else: + env_id, env_version = (None, None) + + if env_id != "aws" or self._cred_verification_url is None: + raise exceptions.InvalidResource( + "No valid AWS 'credential_source' provided" + ) + elif env_version is None or int(env_version) != 1: + raise exceptions.InvalidValue( + "aws version '{}' is not supported in the current build.".format( + env_version + ) + ) + + self._target_resource = audience + self._request_signer = None + + def retrieve_subject_token(self, request): + """Retrieves the subject token using the credential_source object. + The subject token is a serialized `AWS GetCallerIdentity signed request`_. + + The logic is summarized as: + + Retrieve the AWS region from the AWS_REGION or AWS_DEFAULT_REGION + environment variable or from the AWS metadata server availability-zone + if not found in the environment variable. + + Check AWS credentials in environment variables. If not found, retrieve + from the AWS metadata server security-credentials endpoint. + + When retrieving AWS credentials from the metadata server + security-credentials endpoint, the AWS role needs to be determined by + calling the security-credentials endpoint without any argument. Then the + credentials can be retrieved via: security-credentials/role_name + + Generate the signed request to AWS STS GetCallerIdentity action. + + Inject x-goog-cloud-target-resource into header and serialize the + signed request. This will be the subject-token to pass to GCP STS. + + .. _AWS GetCallerIdentity signed request: + https://cloud.google.com/iam/docs/access-resources-aws#exchange-token + + Args: + request (google.auth.transport.Request): A callable used to make + HTTP requests. + Returns: + str: The retrieved subject token. + """ + + # Initialize the request signer if not yet initialized after determining + # the current AWS region. + if self._request_signer is None: + self._region = self._aws_security_credentials_supplier.get_aws_region( + self._supplier_context, request + ) + self._request_signer = RequestSigner(self._region) + + # Retrieve the AWS security credentials needed to generate the signed + # request. + aws_security_credentials = self._aws_security_credentials_supplier.get_aws_security_credentials( + self._supplier_context, request + ) + # Generate the signed request to AWS STS GetCallerIdentity API. + # Use the required regional endpoint. Otherwise, the request will fail. + request_options = self._request_signer.get_request_options( + aws_security_credentials, + self._cred_verification_url.replace("{region}", self._region), + "POST", + ) + # The GCP STS endpoint expects the headers to be formatted as: + # [ + # {key: 'x-amz-date', value: '...'}, + # {key: 'Authorization', value: '...'}, + # ... + # ] + # And then serialized as: + # quote(json.dumps({ + # url: '...', + # method: 'POST', + # headers: [{key: 'x-amz-date', value: '...'}, ...] + # })) + request_headers = request_options.get("headers") + # The full, canonical resource name of the workload identity pool + # provider, with or without the HTTPS prefix. + # Including this header as part of the signature is recommended to + # ensure data integrity. + request_headers["x-goog-cloud-target-resource"] = self._target_resource + + # Serialize AWS signed request. + aws_signed_req = {} + aws_signed_req["url"] = request_options.get("url") + aws_signed_req["method"] = request_options.get("method") + aws_signed_req["headers"] = [] + # Reformat header to GCP STS expected format. + for key in request_headers.keys(): + aws_signed_req["headers"].append( + {"key": key, "value": request_headers[key]} + ) + + return urllib.parse.quote( + json.dumps(aws_signed_req, separators=(",", ":"), sort_keys=True) + ) def _create_default_metrics_options(self): metrics_options = super(Credentials, self)._create_default_metrics_options() metrics_options["source"] = "aws" + if self._has_custom_supplier(): + metrics_options["source"] = "programmatic" return metrics_options + def _has_custom_supplier(self): + return self._credential_source is None + + def _constructor_args(self): + args = super(Credentials, self)._constructor_args() + # If a custom supplier was used, append it to the args dict. + if self._has_custom_supplier(): + args.update( + { + "aws_security_credentials_supplier": self._aws_security_credentials_supplier + } + ) + return args + @classmethod def from_info(cls, info, **kwargs): """Creates an AWS Credentials instance from parsed external account info. @@ -761,6 +839,12 @@ class Credentials(external_account.Credentials): Raises: ValueError: For invalid parameters. """ + aws_security_credentials_supplier = info.get( + "aws_security_credentials_supplier" + ) + kwargs.update( + {"aws_security_credentials_supplier": aws_security_credentials_supplier} + ) return super(Credentials, cls).from_info(info, **kwargs) @classmethod diff --git a/contrib/python/google-auth/py3/google/auth/external_account.py b/contrib/python/google-auth/py3/google/auth/external_account.py index 0420883f86b..c14001bc2b1 100644 --- a/contrib/python/google-auth/py3/google/auth/external_account.py +++ b/contrib/python/google-auth/py3/google/auth/external_account.py @@ -29,6 +29,7 @@ token exchange endpoint following the `OAuth 2.0 Token Exchange`_ spec. import abc import copy +from dataclasses import dataclass import datetime import io import json @@ -50,6 +51,29 @@ _STS_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange" _STS_REQUESTED_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token" # Cloud resource manager URL used to retrieve project information. _CLOUD_RESOURCE_MANAGER = "https://cloudresourcemanager.googleapis.com/v1/projects/" +# Default Google sts token url. +_DEFAULT_TOKEN_URL = "https://sts.googleapis.com/v1/token" + + +@dataclass +class SupplierContext: + """A context class that contains information about the requested third party credential that is passed + to AWS security credential and subject token suppliers. + + Attributes: + subject_token_type (str): The requested subject token type based on the Oauth2.0 token exchange spec. + Expected values include:: + + “urn:ietf:params:oauth:token-type:jwt” + “urn:ietf:params:oauth:token-type:id-token” + “urn:ietf:params:oauth:token-type:saml2” + “urn:ietf:params:aws:token-type:aws4_request” + + audience (str): The requested audience for the subject token. + """ + + subject_token_type: str + audience: str class Credentials( @@ -88,7 +112,14 @@ class Credentials( Args: audience (str): The STS audience field. - subject_token_type (str): The subject token type. + subject_token_type (str): The subject token type based on the Oauth2.0 token exchange spec. + Expected values include:: + + “urn:ietf:params:oauth:token-type:jwt” + “urn:ietf:params:oauth:token-type:id-token” + “urn:ietf:params:oauth:token-type:saml2” + “urn:ietf:params:aws:token-type:aws4_request” + token_url (str): The STS endpoint URL. credential_source (Mapping): The credential source dictionary. service_account_impersonation_url (Optional[str]): The optional service account @@ -145,11 +176,11 @@ class Credentials( self._metrics_options = self._create_default_metrics_options() - if self._service_account_impersonation_url: - self._impersonated_credentials = self._initialize_impersonated_credentials() - else: - self._impersonated_credentials = None + self._impersonated_credentials = None self._project_id = None + self._supplier_context = SupplierContext( + self._subject_token_type, self._audience + ) if not self.is_workforce_pool and self._workforce_pool_user_project: # Workload identity pools do not support workforce pool user projects. @@ -358,6 +389,10 @@ class Credentials( @_helpers.copy_docstring(credentials.Credentials) def refresh(self, request): scopes = self._scopes if self._scopes is not None else self._default_scopes + + if self._should_initialize_impersonated_credentials(): + self._impersonated_credentials = self._initialize_impersonated_credentials() + if self._impersonated_credentials: self._impersonated_credentials.refresh(request) self.token = self._impersonated_credentials.token @@ -421,6 +456,12 @@ class Credentials( new_cred._metrics_options = self._metrics_options return new_cred + def _should_initialize_impersonated_credentials(self): + return ( + self._service_account_impersonation_url is not None + and self._impersonated_credentials is None + ) + def _initialize_impersonated_credentials(self): """Generates an impersonated credentials. diff --git a/contrib/python/google-auth/py3/google/auth/identity_pool.py b/contrib/python/google-auth/py3/google/auth/identity_pool.py index a515353c376..a9ec5773346 100644 --- a/contrib/python/google-auth/py3/google/auth/identity_pool.py +++ b/contrib/python/google-auth/py3/google/auth/identity_pool.py @@ -26,11 +26,13 @@ long-live service account private keys. Identity Pool Credentials are initialized using external_account arguments which are typically loaded from an external credentials file or -an external credentials URL. Unlike other Credentials that can be initialized -with a list of explicit arguments, secrets or credentials, external account -clients use the environment and hints/guidelines provided by the -external_account JSON file to retrieve credentials and exchange them for Google -access tokens. +an external credentials URL. + +This module also provides a definition for an abstract subject token supplier. +This supplier can be implemented to return a valid OIDC or SAML2.0 subject token +and used to create Identity Pool credentials. The credentials will then call the +supplier instead of using pre-defined methods such as reading a local file or +calling a URL. """ try: @@ -38,15 +40,129 @@ try: # Python 2.7 compatibility except ImportError: # pragma: NO COVER from collections import Mapping -import io +import abc import json import os +from typing import NamedTuple from google.auth import _helpers from google.auth import exceptions from google.auth import external_account +class SubjectTokenSupplier(metaclass=abc.ABCMeta): + """Base class for subject token suppliers. This can be implemented with custom logic to retrieve + a subject token to exchange for a Google Cloud access token when using Workload or + Workforce Identity Federation. The identity pool credential does not cache the subject token, + so caching logic should be added in the implementation. + """ + + @abc.abstractmethod + def get_subject_token(self, context, request): + """Returns the requested subject token. The subject token must be valid. + + .. warning: This is not cached by the calling Google credential, so caching logic should be implemented in the supplier. + + Args: + context (google.auth.externalaccount.SupplierContext): The context object + containing information about the requested audience and subject token type. + request (google.auth.transport.Request): The object used to make + HTTP requests. + + Raises: + google.auth.exceptions.RefreshError: If an error is encountered during + subject token retrieval logic. + + Returns: + str: The requested subject token string. + """ + raise NotImplementedError("") + + +class _TokenContent(NamedTuple): + """Models the token content response from file and url internal suppliers. + Attributes: + content (str): The string content of the file or URL response. + location (str): The location the content was retrieved from. This will either be a file location or a URL. + """ + + content: str + location: str + + +class _FileSupplier(SubjectTokenSupplier): + """ Internal implementation of subject token supplier which supports reading a subject token from a file.""" + + def __init__(self, path, format_type, subject_token_field_name): + self._path = path + self._format_type = format_type + self._subject_token_field_name = subject_token_field_name + + @_helpers.copy_docstring(SubjectTokenSupplier) + def get_subject_token(self, context, request): + if not os.path.exists(self._path): + raise exceptions.RefreshError("File '{}' was not found.".format(self._path)) + + with open(self._path, "r", encoding="utf-8") as file_obj: + token_content = _TokenContent(file_obj.read(), self._path) + + return _parse_token_data( + token_content, self._format_type, self._subject_token_field_name + ) + + +class _UrlSupplier(SubjectTokenSupplier): + """ Internal implementation of subject token supplier which supports retrieving a subject token by calling a URL endpoint.""" + + def __init__(self, url, format_type, subject_token_field_name, headers): + self._url = url + self._format_type = format_type + self._subject_token_field_name = subject_token_field_name + self._headers = headers + + @_helpers.copy_docstring(SubjectTokenSupplier) + def get_subject_token(self, context, request): + response = request(url=self._url, method="GET", headers=self._headers) + + # support both string and bytes type response.data + response_body = ( + response.data.decode("utf-8") + if hasattr(response.data, "decode") + else response.data + ) + + if response.status != 200: + raise exceptions.RefreshError( + "Unable to retrieve Identity Pool subject token", response_body + ) + token_content = _TokenContent(response_body, self._url) + return _parse_token_data( + token_content, self._format_type, self._subject_token_field_name + ) + + +def _parse_token_data(token_content, format_type="text", subject_token_field_name=None): + if format_type == "text": + token = token_content.content + else: + try: + # Parse file content as JSON. + response_data = json.loads(token_content.content) + # Get the subject_token. + token = response_data[subject_token_field_name] + except (KeyError, ValueError): + raise exceptions.RefreshError( + "Unable to parse subject_token from JSON file '{}' using key '{}'".format( + token_content.location, subject_token_field_name + ) + ) + if not token: + raise exceptions.RefreshError( + "Missing subject_token in the credential_source file" + ) + return token + + class Credentials(external_account.Credentials): """External account credentials sourced from files and URLs.""" @@ -54,8 +170,9 @@ class Credentials(external_account.Credentials): self, audience, subject_token_type, - token_url, - credential_source, + token_url=external_account._DEFAULT_TOKEN_URL, + credential_source=None, + subject_token_supplier=None, *args, **kwargs ): @@ -63,11 +180,18 @@ class Credentials(external_account.Credentials): Args: audience (str): The STS audience field. - subject_token_type (str): The subject token type. - token_url (str): The STS endpoint URL. - credential_source (Mapping): The credential source dictionary used to + subject_token_type (str): The subject token type based on the Oauth2.0 token exchange spec. + Expected values include:: + + “urn:ietf:params:oauth:token-type:jwt” + “urn:ietf:params:oauth:token-type:id-token” + “urn:ietf:params:oauth:token-type:saml2” + + token_url (Optional [str]): The STS endpoint URL. If not provided, will default to "https://sts.googleapis.com/v1/token". + credential_source (Optional [Mapping]): The credential source dictionary used to provide instructions on how to retrieve external credential to be - exchanged for Google access tokens. + exchanged for Google access tokens. Either a credential source or + a subject token supplier must be provided. Example credential_source for url-sourced credential:: @@ -85,6 +209,10 @@ class Credentials(external_account.Credentials): { "file": "/path/to/token/file.txt" } + subject_token_supplier (Optional [SubjectTokenSupplier]): Optional subject token supplier. + This will be called to supply a valid subject token which will then + be exchanged for Google access tokens. Either a subject token supplier + or a credential source must be provided. args (List): Optional positional arguments passed into the underlying :meth:`~external_account.Credentials.__init__` method. kwargs (Mapping): Optional keyword arguments passed into the underlying :meth:`~external_account.Credentials.__init__` method. @@ -106,10 +234,25 @@ class Credentials(external_account.Credentials): *args, **kwargs ) - if not isinstance(credential_source, Mapping): + if credential_source is None and subject_token_supplier is None: + raise exceptions.InvalidValue( + "A valid credential source or a subject token supplier must be provided." + ) + if credential_source is not None and subject_token_supplier is not None: + raise exceptions.InvalidValue( + "Identity pool credential cannot have both a credential source and a subject token supplier." + ) + + if subject_token_supplier is not None: + self._subject_token_supplier = subject_token_supplier self._credential_source_file = None self._credential_source_url = None else: + if not isinstance(credential_source, Mapping): + self._credential_source_executable = None + raise exceptions.MalformedError( + "Invalid credential_source. The credential_source is not a dict." + ) self._credential_source_file = credential_source.get("file") self._credential_source_url = credential_source.get("url") self._credential_source_headers = credential_source.get("headers") @@ -143,79 +286,35 @@ class Credentials(external_account.Credentials): else: self._credential_source_field_name = None - if self._credential_source_file and self._credential_source_url: - raise exceptions.MalformedError( - "Ambiguous credential_source. 'file' is mutually exclusive with 'url'." - ) - if not self._credential_source_file and not self._credential_source_url: - raise exceptions.MalformedError( - "Missing credential_source. A 'file' or 'url' must be provided." - ) + if self._credential_source_file and self._credential_source_url: + raise exceptions.MalformedError( + "Ambiguous credential_source. 'file' is mutually exclusive with 'url'." + ) + if not self._credential_source_file and not self._credential_source_url: + raise exceptions.MalformedError( + "Missing credential_source. A 'file' or 'url' must be provided." + ) + + if self._credential_source_file: + self._subject_token_supplier = _FileSupplier( + self._credential_source_file, + self._credential_source_format_type, + self._credential_source_field_name, + ) + else: + self._subject_token_supplier = _UrlSupplier( + self._credential_source_url, + self._credential_source_format_type, + self._credential_source_field_name, + self._credential_source_headers, + ) @_helpers.copy_docstring(external_account.Credentials) def retrieve_subject_token(self, request): - return self._parse_token_data( - self._get_token_data(request), - self._credential_source_format_type, - self._credential_source_field_name, - ) - - def _get_token_data(self, request): - if self._credential_source_file: - return self._get_file_data(self._credential_source_file) - else: - return self._get_url_data( - request, self._credential_source_url, self._credential_source_headers - ) - - def _get_file_data(self, filename): - if not os.path.exists(filename): - raise exceptions.RefreshError("File '{}' was not found.".format(filename)) - - with io.open(filename, "r", encoding="utf-8") as file_obj: - return file_obj.read(), filename - - def _get_url_data(self, request, url, headers): - response = request(url=url, method="GET", headers=headers) - - # support both string and bytes type response.data - response_body = ( - response.data.decode("utf-8") - if hasattr(response.data, "decode") - else response.data + return self._subject_token_supplier.get_subject_token( + self._supplier_context, request ) - if response.status != 200: - raise exceptions.RefreshError( - "Unable to retrieve Identity Pool subject token", response_body - ) - - return response_body, url - - def _parse_token_data( - self, token_content, format_type="text", subject_token_field_name=None - ): - content, filename = token_content - if format_type == "text": - token = content - else: - try: - # Parse file content as JSON. - response_data = json.loads(content) - # Get the subject_token. - token = response_data[subject_token_field_name] - except (KeyError, ValueError): - raise exceptions.RefreshError( - "Unable to parse subject_token from JSON file '{}' using key '{}'".format( - filename, subject_token_field_name - ) - ) - if not token: - raise exceptions.RefreshError( - "Missing subject_token in the credential_source file" - ) - return token - def _create_default_metrics_options(self): metrics_options = super(Credentials, self)._create_default_metrics_options() # Check that credential source is a dict before checking for file vs url. This check needs to be done @@ -226,8 +325,20 @@ class Credentials(external_account.Credentials): metrics_options["source"] = "file" else: metrics_options["source"] = "url" + else: + metrics_options["source"] = "programmatic" return metrics_options + def _has_custom_supplier(self): + return self._credential_source is None + + def _constructor_args(self): + args = super(Credentials, self)._constructor_args() + # If a custom supplier was used, append it to the args dict. + if self._has_custom_supplier(): + args.update({"subject_token_supplier": self._subject_token_supplier}) + return args + @classmethod def from_info(cls, info, **kwargs): """Creates an Identity Pool Credentials instance from parsed external account info. @@ -244,6 +355,8 @@ class Credentials(external_account.Credentials): Raises: ValueError: For invalid parameters. """ + subject_token_supplier = info.get("subject_token_supplier") + kwargs.update({"subject_token_supplier": subject_token_supplier}) return super(Credentials, cls).from_info(info, **kwargs) @classmethod diff --git a/contrib/python/google-auth/py3/google/auth/version.py b/contrib/python/google-auth/py3/google/auth/version.py index 0959c754195..f0dd919dca6 100644 --- a/contrib/python/google-auth/py3/google/auth/version.py +++ b/contrib/python/google-auth/py3/google/auth/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.28.2" +__version__ = "2.29.0" diff --git a/contrib/python/google-auth/py3/tests/test_aws.py b/contrib/python/google-auth/py3/tests/test_aws.py index 3f358d52b09..56148203128 100644 --- a/contrib/python/google-auth/py3/tests/test_aws.py +++ b/contrib/python/google-auth/py3/tests/test_aws.py @@ -21,7 +21,7 @@ import urllib.parse import mock import pytest # type: ignore -from google.auth import _helpers +from google.auth import _helpers, external_account from google.auth import aws from google.auth import environment_vars from google.auth import exceptions @@ -616,8 +616,13 @@ class TestRequestSigner(object): ): utcnow.return_value = datetime.datetime.strptime(time, "%Y-%m-%dT%H:%M:%SZ") request_signer = aws.RequestSigner(region) + credentials_object = aws.AwsSecurityCredentials( + credentials.get("access_key_id"), + credentials.get("secret_access_key"), + credentials.get("security_token"), + ) actual_signed_request = request_signer.get_request_options( - credentials, + credentials_object, original_request.get("url"), original_request.get("method"), original_request.get("data"), @@ -631,10 +636,7 @@ class TestRequestSigner(object): with pytest.raises(ValueError) as excinfo: request_signer.get_request_options( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - }, + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY), "invalid", "POST", ) @@ -646,10 +648,7 @@ class TestRequestSigner(object): with pytest.raises(ValueError) as excinfo: request_signer.get_request_options( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - }, + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY), "http://invalid", "POST", ) @@ -661,10 +660,7 @@ class TestRequestSigner(object): with pytest.raises(ValueError) as excinfo: request_signer.get_request_options( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - }, + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY), "https://", "POST", ) @@ -672,6 +668,36 @@ class TestRequestSigner(object): assert excinfo.match(r"Invalid AWS service URL") +class TestAwsSecurityCredentialsSupplier(aws.AwsSecurityCredentialsSupplier): + def __init__( + self, + security_credentials=None, + region=None, + credentials_exception=None, + region_exception=None, + expected_context=None, + ): + self._security_credentials = security_credentials + self._region = region + self._credentials_exception = credentials_exception + self._region_exception = region_exception + self._expected_context = expected_context + + def get_aws_security_credentials(self, context, request): + if self._expected_context is not None: + assert self._expected_context == context + if self._credentials_exception is not None: + raise self._credentials_exception + return self._security_credentials + + def get_aws_region(self, context, request): + if self._expected_context is not None: + assert self._expected_context == context + if self._region_exception is not None: + raise self._region_exception + return self._region + + class TestCredentials(object): AWS_REGION = "us-east-2" AWS_ROLE = "gcp-aws-role" @@ -734,7 +760,7 @@ class TestCredentials(object): ], } # Include security token if available. - if "security_token" in aws_security_credentials: + if aws_security_credentials.session_token is not None: reformatted_signed_request.get("headers").append( { "key": "x-amz-security-token", @@ -773,16 +799,17 @@ class TestCredentials(object): in an AWS environment. """ responses = [] - if imdsv2_session_token_status: - # AWS session token request - imdsv2_session_response = mock.create_autospec( - transport.Response, instance=True - ) - imdsv2_session_response.status = imdsv2_session_token_status - imdsv2_session_response.data = imdsv2_session_token_data - responses.append(imdsv2_session_response) if region_status: + if imdsv2_session_token_status: + # AWS session token request + imdsv2_session_response = mock.create_autospec( + transport.Response, instance=True + ) + imdsv2_session_response.status = imdsv2_session_token_status + imdsv2_session_response.data = imdsv2_session_token_data + responses.append(imdsv2_session_response) + # AWS region request. region_response = mock.create_autospec(transport.Response, instance=True) region_response.status = region_status @@ -790,6 +817,15 @@ class TestCredentials(object): region_response.data = "{}b".format(region_name).encode("utf-8") responses.append(region_response) + if imdsv2_session_token_status: + # AWS session token request + imdsv2_session_response = mock.create_autospec( + transport.Response, instance=True + ) + imdsv2_session_response.status = imdsv2_session_token_status + imdsv2_session_response.data = imdsv2_session_token_data + responses.append(imdsv2_session_response) + if role_status: # AWS role name request. role_response = mock.create_autospec(transport.Response, instance=True) @@ -834,7 +870,8 @@ class TestCredentials(object): @classmethod def make_credentials( cls, - credential_source, + credential_source=None, + aws_security_credentials_supplier=None, token_url=TOKEN_URL, token_info_url=TOKEN_INFO_URL, client_id=None, @@ -851,6 +888,7 @@ class TestCredentials(object): token_info_url=token_info_url, service_account_impersonation_url=service_account_impersonation_url, credential_source=credential_source, + aws_security_credentials_supplier=aws_security_credentials_supplier, client_id=client_id, client_secret=client_secret, quota_project_id=quota_project_id, @@ -929,6 +967,7 @@ class TestCredentials(object): client_id=CLIENT_ID, client_secret=CLIENT_SECRET, credential_source=self.CREDENTIAL_SOURCE, + aws_security_credentials_supplier=None, quota_project_id=QUOTA_PROJECT_ID, workforce_pool_user_project=None, universe_domain=DEFAULT_UNIVERSE_DOMAIN, @@ -957,6 +996,38 @@ class TestCredentials(object): client_id=None, client_secret=None, credential_source=self.CREDENTIAL_SOURCE, + aws_security_credentials_supplier=None, + quota_project_id=None, + workforce_pool_user_project=None, + universe_domain=DEFAULT_UNIVERSE_DOMAIN, + ) + + @mock.patch.object(aws.Credentials, "__init__", return_value=None) + def test_from_info_supplier(self, mock_init): + supplier = TestAwsSecurityCredentialsSupplier() + + credentials = aws.Credentials.from_info( + { + "audience": AUDIENCE, + "subject_token_type": SUBJECT_TOKEN_TYPE, + "token_url": TOKEN_URL, + "aws_security_credentials_supplier": supplier, + } + ) + + # Confirm aws.Credentials instance initialized with the expected parameters. + assert isinstance(credentials, aws.Credentials) + mock_init.assert_called_once_with( + audience=AUDIENCE, + subject_token_type=SUBJECT_TOKEN_TYPE, + token_url=TOKEN_URL, + token_info_url=None, + service_account_impersonation_url=None, + service_account_impersonation_options={}, + client_id=None, + client_secret=None, + credential_source=None, + aws_security_credentials_supplier=supplier, quota_project_id=None, workforce_pool_user_project=None, universe_domain=DEFAULT_UNIVERSE_DOMAIN, @@ -993,6 +1064,7 @@ class TestCredentials(object): client_id=CLIENT_ID, client_secret=CLIENT_SECRET, credential_source=self.CREDENTIAL_SOURCE, + aws_security_credentials_supplier=None, quota_project_id=QUOTA_PROJECT_ID, workforce_pool_user_project=None, universe_domain=DEFAULT_UNIVERSE_DOMAIN, @@ -1022,6 +1094,7 @@ class TestCredentials(object): client_id=None, client_secret=None, credential_source=self.CREDENTIAL_SOURCE, + aws_security_credentials_supplier=None, quota_project_id=None, workforce_pool_user_project=None, universe_domain=DEFAULT_UNIVERSE_DOMAIN, @@ -1036,6 +1109,27 @@ class TestCredentials(object): assert excinfo.match(r"No valid AWS 'credential_source' provided") + def test_constructor_invalid_credential_source_and_supplier(self): + # Provide both a credential source and supplier. + with pytest.raises(ValueError) as excinfo: + self.make_credentials( + credential_source=self.CREDENTIAL_SOURCE, + aws_security_credentials_supplier="test", + ) + + assert excinfo.match( + r"AWS credential cannot have both a credential source and an AWS security credentials supplier." + ) + + def test_constructor_invalid_no_credential_source_or_supplier(self): + # Provide no credential source or supplier. + with pytest.raises(ValueError) as excinfo: + self.make_credentials() + + assert excinfo.match( + r"A valid credential source or AWS security credentials supplier must be provided." + ) + def test_constructor_invalid_environment_id(self): # Provide invalid environment_id. credential_source = self.CREDENTIAL_SOURCE.copy() @@ -1158,11 +1252,7 @@ class TestCredentials(object): subject_token = credentials.retrieve_subject_token(request) assert subject_token == self.make_serialized_aws_signed_request( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - "security_token": TOKEN, - } + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) ) # Assert region request. self.assert_aws_metadata_request_kwargs( @@ -1231,11 +1321,7 @@ class TestCredentials(object): subject_token = credentials.retrieve_subject_token(request) assert subject_token == self.make_serialized_aws_signed_request( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - "security_token": TOKEN, - } + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) ) # Assert session token request self.assert_aws_metadata_request_kwargs( @@ -1250,15 +1336,22 @@ class TestCredentials(object): REGION_URL, {"X-aws-ec2-metadata-token": self.AWS_IMDSV2_SESSION_TOKEN}, ) - # Assert role request. + # Assert session token request self.assert_aws_metadata_request_kwargs( request.call_args_list[2][1], + IMDSV2_SESSION_TOKEN_URL, + {"X-aws-ec2-metadata-token-ttl-seconds": "300"}, + "PUT", + ) + # Assert role request. + self.assert_aws_metadata_request_kwargs( + request.call_args_list[3][1], SECURITY_CREDS_URL, {"X-aws-ec2-metadata-token": self.AWS_IMDSV2_SESSION_TOKEN}, ) # Assert security credentials request. self.assert_aws_metadata_request_kwargs( - request.call_args_list[3][1], + request.call_args_list[4][1], "{}/{}".format(SECURITY_CREDS_URL, self.AWS_ROLE), { "Content-Type": "application/json", @@ -1335,11 +1428,7 @@ class TestCredentials(object): subject_token = credentials.retrieve_subject_token(request) assert subject_token == self.make_serialized_aws_signed_request( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - "security_token": TOKEN, - } + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) ) # Assert session token request. self.assert_aws_metadata_request_kwargs( @@ -1396,11 +1485,7 @@ class TestCredentials(object): subject_token = credentials.retrieve_subject_token(request) assert subject_token == self.make_serialized_aws_signed_request( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - "security_token": TOKEN, - } + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) ) # Assert session token request. self.assert_aws_metadata_request_kwargs( @@ -1451,11 +1536,7 @@ class TestCredentials(object): subject_token = credentials.retrieve_subject_token(request) assert subject_token == self.make_serialized_aws_signed_request( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - "security_token": TOKEN, - } + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) ) # Assert session token request. self.assert_aws_metadata_request_kwargs( @@ -1530,11 +1611,7 @@ class TestCredentials(object): subject_token = credentials.retrieve_subject_token(request) assert subject_token == self.make_serialized_aws_signed_request( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - "security_token": TOKEN, - } + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) ) # Assert session token request. self.assert_aws_metadata_request_kwargs( @@ -1549,15 +1626,22 @@ class TestCredentials(object): REGION_URL_IPV6, {"X-aws-ec2-metadata-token": self.AWS_IMDSV2_SESSION_TOKEN}, ) - # Assert role request. + # Assert session token request. self.assert_aws_metadata_request_kwargs( request.call_args_list[2][1], + IMDSV2_SESSION_TOKEN_URL_IPV6, + {"X-aws-ec2-metadata-token-ttl-seconds": "300"}, + "PUT", + ) + # Assert role request. + self.assert_aws_metadata_request_kwargs( + request.call_args_list[3][1], SECURITY_CREDS_URL_IPV6, {"X-aws-ec2-metadata-token": self.AWS_IMDSV2_SESSION_TOKEN}, ) # Assert security credentials request. self.assert_aws_metadata_request_kwargs( - request.call_args_list[3][1], + request.call_args_list[4][1], "{}/{}".format(SECURITY_CREDS_URL_IPV6, self.AWS_ROLE), { "Content-Type": "application/json", @@ -1619,7 +1703,7 @@ class TestCredentials(object): subject_token = credentials.retrieve_subject_token(request) assert subject_token == self.make_serialized_aws_signed_request( - {"access_key_id": ACCESS_KEY_ID, "secret_access_key": SECRET_ACCESS_KEY} + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY) ) @mock.patch("google.auth._helpers.utcnow") @@ -1636,11 +1720,7 @@ class TestCredentials(object): subject_token = credentials.retrieve_subject_token(None) assert subject_token == self.make_serialized_aws_signed_request( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - "security_token": TOKEN, - } + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) ) @mock.patch("google.auth._helpers.utcnow") @@ -1659,11 +1739,7 @@ class TestCredentials(object): subject_token = credentials.retrieve_subject_token(None) assert subject_token == self.make_serialized_aws_signed_request( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - "security_token": TOKEN, - } + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) ) @mock.patch("google.auth._helpers.utcnow") @@ -1686,11 +1762,7 @@ class TestCredentials(object): subject_token = credentials.retrieve_subject_token(None) assert subject_token == self.make_serialized_aws_signed_request( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - "security_token": TOKEN, - } + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) ) @mock.patch("google.auth._helpers.utcnow") @@ -1708,7 +1780,7 @@ class TestCredentials(object): subject_token = credentials.retrieve_subject_token(None) assert subject_token == self.make_serialized_aws_signed_request( - {"access_key_id": ACCESS_KEY_ID, "secret_access_key": SECRET_ACCESS_KEY} + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY) ) @mock.patch("google.auth._helpers.utcnow") @@ -1730,11 +1802,7 @@ class TestCredentials(object): subject_token = credentials.retrieve_subject_token(request) assert subject_token == self.make_serialized_aws_signed_request( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - "security_token": TOKEN, - } + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) ) def test_retrieve_subject_token_error_determining_aws_region(self): @@ -1806,11 +1874,7 @@ class TestCredentials(object): self.AWS_SIGNATURE_TIME, "%Y-%m-%dT%H:%M:%SZ" ) expected_subject_token = self.make_serialized_aws_signed_request( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - "security_token": TOKEN, - } + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) ) token_headers = { "Content-Type": "application/x-www-form-urlencoded", @@ -1869,11 +1933,7 @@ class TestCredentials(object): self.AWS_SIGNATURE_TIME, "%Y-%m-%dT%H:%M:%SZ" ) expected_subject_token = self.make_serialized_aws_signed_request( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - "security_token": TOKEN, - } + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) ) token_headers = { "Content-Type": "application/x-www-form-urlencoded", @@ -1939,11 +1999,7 @@ class TestCredentials(object): _helpers.utcnow().replace(microsecond=0) + datetime.timedelta(seconds=3600) ).isoformat("T") + "Z" expected_subject_token = self.make_serialized_aws_signed_request( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - "security_token": TOKEN, - } + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) ) token_headers = { "Content-Type": "application/x-www-form-urlencoded", @@ -2036,11 +2092,7 @@ class TestCredentials(object): _helpers.utcnow().replace(microsecond=0) + datetime.timedelta(seconds=3600) ).isoformat("T") + "Z" expected_subject_token = self.make_serialized_aws_signed_request( - { - "access_key_id": ACCESS_KEY_ID, - "secret_access_key": SECRET_ACCESS_KEY, - "security_token": TOKEN, - } + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) ) token_headers = { "Content-Type": "application/x-www-form-urlencoded", @@ -2122,3 +2174,249 @@ class TestCredentials(object): credentials.refresh(request) assert excinfo.match(r"Unable to retrieve AWS region") + + @mock.patch("google.auth._helpers.utcnow") + def test_retrieve_subject_token_success_with_supplier(self, utcnow): + utcnow.return_value = datetime.datetime.strptime( + self.AWS_SIGNATURE_TIME, "%Y-%m-%dT%H:%M:%SZ" + ) + request = self.make_mock_request() + + security_credentials = aws.AwsSecurityCredentials( + ACCESS_KEY_ID, SECRET_ACCESS_KEY + ) + supplier = TestAwsSecurityCredentialsSupplier( + security_credentials=security_credentials, region=self.AWS_REGION + ) + + credentials = self.make_credentials(aws_security_credentials_supplier=supplier) + + subject_token = credentials.retrieve_subject_token(request) + assert subject_token == self.make_serialized_aws_signed_request( + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY) + ) + + @mock.patch("google.auth._helpers.utcnow") + def test_retrieve_subject_token_success_with_supplier_session_token(self, utcnow): + utcnow.return_value = datetime.datetime.strptime( + self.AWS_SIGNATURE_TIME, "%Y-%m-%dT%H:%M:%SZ" + ) + request = self.make_mock_request() + + security_credentials = aws.AwsSecurityCredentials( + ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN + ) + supplier = TestAwsSecurityCredentialsSupplier( + security_credentials=security_credentials, region=self.AWS_REGION + ) + + credentials = self.make_credentials(aws_security_credentials_supplier=supplier) + + subject_token = credentials.retrieve_subject_token(request) + assert subject_token == self.make_serialized_aws_signed_request( + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) + ) + + @mock.patch("google.auth._helpers.utcnow") + def test_retrieve_subject_token_success_with_supplier_correct_context(self, utcnow): + utcnow.return_value = datetime.datetime.strptime( + self.AWS_SIGNATURE_TIME, "%Y-%m-%dT%H:%M:%SZ" + ) + request = self.make_mock_request() + expected_context = external_account.SupplierContext( + SUBJECT_TOKEN_TYPE, AUDIENCE + ) + + security_credentials = aws.AwsSecurityCredentials( + ACCESS_KEY_ID, SECRET_ACCESS_KEY + ) + supplier = TestAwsSecurityCredentialsSupplier( + security_credentials=security_credentials, + region=self.AWS_REGION, + expected_context=expected_context, + ) + + credentials = self.make_credentials(aws_security_credentials_supplier=supplier) + + credentials.retrieve_subject_token(request) + + def test_retrieve_subject_token_error_with_supplier(self): + request = self.make_mock_request() + expected_exception = exceptions.RefreshError("Test error") + supplier = TestAwsSecurityCredentialsSupplier( + region=self.AWS_REGION, credentials_exception=expected_exception + ) + + credentials = self.make_credentials(aws_security_credentials_supplier=supplier) + + with pytest.raises(exceptions.RefreshError) as excinfo: + credentials.refresh(request) + + assert excinfo.match(r"Test error") + + def test_retrieve_subject_token_error_with_supplier_region(self): + request = self.make_mock_request() + expected_exception = exceptions.RefreshError("Test error") + security_credentials = aws.AwsSecurityCredentials( + ACCESS_KEY_ID, SECRET_ACCESS_KEY + ) + supplier = TestAwsSecurityCredentialsSupplier( + security_credentials=security_credentials, + region_exception=expected_exception, + ) + + credentials = self.make_credentials(aws_security_credentials_supplier=supplier) + + with pytest.raises(exceptions.RefreshError) as excinfo: + credentials.refresh(request) + + assert excinfo.match(r"Test error") + + @mock.patch( + "google.auth.metrics.python_and_auth_lib_version", + return_value=LANG_LIBRARY_METRICS_HEADER_VALUE, + ) + @mock.patch("google.auth._helpers.utcnow") + def test_refresh_success_with_supplier_with_impersonation( + self, utcnow, mock_auth_lib_value + ): + utcnow.return_value = datetime.datetime.strptime( + self.AWS_SIGNATURE_TIME, "%Y-%m-%dT%H:%M:%SZ" + ) + expire_time = ( + _helpers.utcnow().replace(microsecond=0) + datetime.timedelta(seconds=3600) + ).isoformat("T") + "Z" + expected_subject_token = self.make_serialized_aws_signed_request( + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) + ) + token_headers = { + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": "Basic " + BASIC_AUTH_ENCODING, + "x-goog-api-client": "gl-python/3.7 auth/1.1 google-byoid-sdk sa-impersonation/true config-lifetime/false source/programmatic", + } + token_request_data = { + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "audience": AUDIENCE, + "requested_token_type": "urn:ietf:params:oauth:token-type:access_token", + "scope": "https://www.googleapis.com/auth/iam", + "subject_token": expected_subject_token, + "subject_token_type": SUBJECT_TOKEN_TYPE, + } + # Service account impersonation request/response. + impersonation_response = { + "accessToken": "SA_ACCESS_TOKEN", + "expireTime": expire_time, + } + impersonation_headers = { + "Content-Type": "application/json", + "authorization": "Bearer {}".format(self.SUCCESS_RESPONSE["access_token"]), + "x-goog-user-project": QUOTA_PROJECT_ID, + "x-goog-api-client": IMPERSONATE_ACCESS_TOKEN_REQUEST_METRICS_HEADER_VALUE, + "x-allowed-locations": "0x0", + } + impersonation_request_data = { + "delegates": None, + "scope": SCOPES, + "lifetime": "3600s", + } + request = self.make_mock_request( + token_status=http_client.OK, + token_data=self.SUCCESS_RESPONSE, + impersonation_status=http_client.OK, + impersonation_data=impersonation_response, + ) + + supplier = TestAwsSecurityCredentialsSupplier( + security_credentials=aws.AwsSecurityCredentials( + ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN + ), + region=self.AWS_REGION, + ) + + credentials = self.make_credentials( + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + aws_security_credentials_supplier=supplier, + service_account_impersonation_url=SERVICE_ACCOUNT_IMPERSONATION_URL, + quota_project_id=QUOTA_PROJECT_ID, + scopes=SCOPES, + # Default scopes should be ignored. + default_scopes=["ignored"], + ) + + credentials.refresh(request) + + assert len(request.call_args_list) == 2 + # First request should be sent to GCP STS endpoint. + self.assert_token_request_kwargs( + request.call_args_list[0][1], token_headers, token_request_data + ) + # Second request should be sent to iamcredentials endpoint for service + # account impersonation. + self.assert_impersonation_request_kwargs( + request.call_args_list[1][1], + impersonation_headers, + impersonation_request_data, + ) + assert credentials.token == impersonation_response["accessToken"] + assert credentials.quota_project_id == QUOTA_PROJECT_ID + assert credentials.scopes == SCOPES + assert credentials.default_scopes == ["ignored"] + + @mock.patch( + "google.auth.metrics.python_and_auth_lib_version", + return_value=LANG_LIBRARY_METRICS_HEADER_VALUE, + ) + @mock.patch("google.auth._helpers.utcnow") + def test_refresh_success_with_supplier(self, utcnow, mock_auth_lib_value): + utcnow.return_value = datetime.datetime.strptime( + self.AWS_SIGNATURE_TIME, "%Y-%m-%dT%H:%M:%SZ" + ) + expected_subject_token = self.make_serialized_aws_signed_request( + aws.AwsSecurityCredentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN) + ) + token_headers = { + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": "Basic " + BASIC_AUTH_ENCODING, + "x-goog-api-client": "gl-python/3.7 auth/1.1 google-byoid-sdk sa-impersonation/false config-lifetime/false source/programmatic", + } + token_request_data = { + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "audience": AUDIENCE, + "requested_token_type": "urn:ietf:params:oauth:token-type:access_token", + "scope": " ".join(SCOPES), + "subject_token": expected_subject_token, + "subject_token_type": SUBJECT_TOKEN_TYPE, + } + request = self.make_mock_request( + token_status=http_client.OK, token_data=self.SUCCESS_RESPONSE + ) + + supplier = TestAwsSecurityCredentialsSupplier( + security_credentials=aws.AwsSecurityCredentials( + ACCESS_KEY_ID, SECRET_ACCESS_KEY, TOKEN + ), + region=self.AWS_REGION, + ) + + credentials = self.make_credentials( + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + aws_security_credentials_supplier=supplier, + quota_project_id=QUOTA_PROJECT_ID, + scopes=SCOPES, + # Default scopes should be ignored. + default_scopes=["ignored"], + ) + + credentials.refresh(request) + + assert len(request.call_args_list) == 1 + # First request should be sent to GCP STS endpoint. + self.assert_token_request_kwargs( + request.call_args_list[0][1], token_headers, token_request_data + ) + assert credentials.token == self.SUCCESS_RESPONSE["access_token"] + assert credentials.quota_project_id == QUOTA_PROJECT_ID + assert credentials.scopes == SCOPES + assert credentials.default_scopes == ["ignored"] diff --git a/contrib/python/google-auth/py3/tests/test_external_account.py b/contrib/python/google-auth/py3/tests/test_external_account.py index 03a5014ce5b..c458b21b643 100644 --- a/contrib/python/google-auth/py3/tests/test_external_account.py +++ b/contrib/python/google-auth/py3/tests/test_external_account.py @@ -477,16 +477,6 @@ class TestCredentials(object): universe_domain=DEFAULT_UNIVERSE_DOMAIN, ) - def test_with_invalid_impersonation_target_principal(self): - invalid_url = "https://iamcredentials.googleapis.com/v1/invalid" - - with pytest.raises(exceptions.RefreshError) as excinfo: - self.make_credentials(service_account_impersonation_url=invalid_url) - - assert excinfo.match( - r"Unable to determine target principal from service account impersonation URL." - ) - def test_info(self): credentials = self.make_credentials(universe_domain="dummy_universe.com") @@ -1069,6 +1059,21 @@ class TestCredentials(object): assert not credentials.expired assert credentials.token is None + def test_refresh_impersonation_invalid_impersonated_url_error(self): + credentials = self.make_credentials( + service_account_impersonation_url="https://iamcredentials.googleapis.com/v1/invalid", + scopes=self.SCOPES, + ) + + with pytest.raises(exceptions.RefreshError) as excinfo: + credentials.refresh(None) + + assert excinfo.match( + r"Unable to determine target principal from service account impersonation URL." + ) + assert not credentials.expired + assert credentials.token is None + @mock.patch( "google.auth.metrics.python_and_auth_lib_version", return_value=LANG_LIBRARY_METRICS_HEADER_VALUE, @@ -1913,3 +1918,10 @@ class TestCredentials(object): assert project_id is None # Only 2 requests to STS and cloud resource manager should be sent. assert len(request.call_args_list) == 2 + + +def test_supplier_context(): + context = external_account.SupplierContext("TestTokenType", "TestAudience") + + assert context.subject_token_type == "TestTokenType" + assert context.audience == "TestAudience" diff --git a/contrib/python/google-auth/py3/tests/test_identity_pool.py b/contrib/python/google-auth/py3/tests/test_identity_pool.py index 96be1d61c28..0de711832f6 100644 --- a/contrib/python/google-auth/py3/tests/test_identity_pool.py +++ b/contrib/python/google-auth/py3/tests/test_identity_pool.py @@ -21,7 +21,7 @@ import urllib import mock import pytest # type: ignore -from google.auth import _helpers +from google.auth import _helpers, external_account from google.auth import exceptions from google.auth import identity_pool from google.auth import metrics @@ -152,6 +152,22 @@ INVALID_SERVICE_ACCOUNT_IMPERSONATION_URLS = [ ] +class TestSubjectTokenSupplier(identity_pool.SubjectTokenSupplier): + def __init__( + self, subject_token=None, subject_token_exception=None, expected_context=None + ): + self._subject_token = subject_token + self._subject_token_exception = subject_token_exception + self._expected_context = expected_context + + def get_subject_token(self, context, request): + if self._expected_context is not None: + assert self._expected_context == context + if self._subject_token_exception is not None: + raise self._subject_token_exception + return self._subject_token + + class TestCredentials(object): CREDENTIAL_SOURCE_TEXT = {"file": SUBJECT_TOKEN_TEXT_FILE} CREDENTIAL_SOURCE_JSON = { @@ -274,10 +290,13 @@ class TestCredentials(object): else: metrics_options["sa-impersonation"] = "false" metrics_options["config-lifetime"] = "false" - if credentials._credential_source_file: - metrics_options["source"] = "file" + if credentials._credential_source: + if credentials._credential_source_file: + metrics_options["source"] = "file" + else: + metrics_options["source"] = "url" else: - metrics_options["source"] = "url" + metrics_options["source"] = "programmatic" token_headers["x-goog-api-client"] = metrics.byoid_metrics_header( metrics_options @@ -387,6 +406,7 @@ class TestCredentials(object): default_scopes=None, service_account_impersonation_url=None, credential_source=None, + subject_token_supplier=None, workforce_pool_user_project=None, ): return identity_pool.Credentials( @@ -396,6 +416,7 @@ class TestCredentials(object): token_info_url=token_info_url, service_account_impersonation_url=service_account_impersonation_url, credential_source=credential_source, + subject_token_supplier=subject_token_supplier, client_id=client_id, client_secret=client_secret, quota_project_id=quota_project_id, @@ -433,6 +454,7 @@ class TestCredentials(object): client_id=CLIENT_ID, client_secret=CLIENT_SECRET, credential_source=self.CREDENTIAL_SOURCE_TEXT, + subject_token_supplier=None, quota_project_id=QUOTA_PROJECT_ID, workforce_pool_user_project=None, universe_domain=DEFAULT_UNIVERSE_DOMAIN, @@ -461,6 +483,38 @@ class TestCredentials(object): client_id=None, client_secret=None, credential_source=self.CREDENTIAL_SOURCE_TEXT, + subject_token_supplier=None, + quota_project_id=None, + workforce_pool_user_project=None, + universe_domain=DEFAULT_UNIVERSE_DOMAIN, + ) + + @mock.patch.object(identity_pool.Credentials, "__init__", return_value=None) + def test_from_info_supplier(self, mock_init): + supplier = TestSubjectTokenSupplier() + + credentials = identity_pool.Credentials.from_info( + { + "audience": AUDIENCE, + "subject_token_type": SUBJECT_TOKEN_TYPE, + "token_url": TOKEN_URL, + "subject_token_supplier": supplier, + } + ) + + # Confirm identity_pool.Credentials instantiated with expected attributes. + assert isinstance(credentials, identity_pool.Credentials) + mock_init.assert_called_once_with( + audience=AUDIENCE, + subject_token_type=SUBJECT_TOKEN_TYPE, + token_url=TOKEN_URL, + token_info_url=None, + service_account_impersonation_url=None, + service_account_impersonation_options={}, + client_id=None, + client_secret=None, + credential_source=None, + subject_token_supplier=supplier, quota_project_id=None, workforce_pool_user_project=None, universe_domain=DEFAULT_UNIVERSE_DOMAIN, @@ -490,6 +544,7 @@ class TestCredentials(object): client_id=None, client_secret=None, credential_source=self.CREDENTIAL_SOURCE_TEXT, + subject_token_supplier=None, quota_project_id=None, workforce_pool_user_project=WORKFORCE_POOL_USER_PROJECT, universe_domain=DEFAULT_UNIVERSE_DOMAIN, @@ -525,6 +580,7 @@ class TestCredentials(object): client_id=CLIENT_ID, client_secret=CLIENT_SECRET, credential_source=self.CREDENTIAL_SOURCE_TEXT, + subject_token_supplier=None, quota_project_id=QUOTA_PROJECT_ID, workforce_pool_user_project=None, universe_domain=DEFAULT_UNIVERSE_DOMAIN, @@ -554,6 +610,7 @@ class TestCredentials(object): client_id=None, client_secret=None, credential_source=self.CREDENTIAL_SOURCE_TEXT, + subject_token_supplier=None, quota_project_id=None, workforce_pool_user_project=None, universe_domain=DEFAULT_UNIVERSE_DOMAIN, @@ -584,6 +641,7 @@ class TestCredentials(object): client_id=None, client_secret=None, credential_source=self.CREDENTIAL_SOURCE_TEXT, + subject_token_supplier=None, quota_project_id=None, workforce_pool_user_project=WORKFORCE_POOL_USER_PROJECT, universe_domain=DEFAULT_UNIVERSE_DOMAIN, @@ -634,7 +692,29 @@ class TestCredentials(object): with pytest.raises(ValueError) as excinfo: self.make_credentials(credential_source="non-dict") - assert excinfo.match(r"Missing credential_source") + assert excinfo.match( + r"Invalid credential_source. The credential_source is not a dict." + ) + + def test_constructor_invalid_no_credential_source_or_supplier(self): + with pytest.raises(ValueError) as excinfo: + self.make_credentials() + + assert excinfo.match( + r"A valid credential source or a subject token supplier must be provided." + ) + + def test_constructor_invalid_both_credential_source_and_supplier(self): + supplier = TestSubjectTokenSupplier() + with pytest.raises(ValueError) as excinfo: + self.make_credentials( + credential_source=self.CREDENTIAL_SOURCE_TEXT, + subject_token_supplier=supplier, + ) + + assert excinfo.match( + r"Identity pool credential cannot have both a credential source and a subject token supplier." + ) def test_constructor_invalid_credential_source_format_type(self): credential_source = {"format": {"type": "xml"}} @@ -1298,3 +1378,78 @@ class TestCredentials(object): self.CREDENTIAL_URL, "not_found" ) ) + + def test_retrieve_subject_token_supplier(self): + supplier = TestSubjectTokenSupplier(subject_token=JSON_FILE_SUBJECT_TOKEN) + + credentials = self.make_credentials(subject_token_supplier=supplier) + + subject_token = credentials.retrieve_subject_token(None) + + assert subject_token == JSON_FILE_SUBJECT_TOKEN + + def test_retrieve_subject_token_supplier_correct_context(self): + supplier = TestSubjectTokenSupplier( + subject_token=JSON_FILE_SUBJECT_TOKEN, + expected_context=external_account.SupplierContext( + SUBJECT_TOKEN_TYPE, AUDIENCE + ), + ) + + credentials = self.make_credentials(subject_token_supplier=supplier) + + credentials.retrieve_subject_token(None) + + def test_retrieve_subject_token_supplier_error(self): + expected_exception = exceptions.RefreshError("test error") + supplier = TestSubjectTokenSupplier(subject_token_exception=expected_exception) + + credentials = self.make_credentials(subject_token_supplier=supplier) + + with pytest.raises(exceptions.RefreshError) as excinfo: + credentials.refresh(self.make_mock_request(token_data=JSON_FILE_CONTENT)) + + assert excinfo.match("test error") + + def test_refresh_success_supplier_with_impersonation_url(self): + # Initialize credentials with service account impersonation and a supplier. + supplier = TestSubjectTokenSupplier(subject_token=JSON_FILE_SUBJECT_TOKEN) + credentials = self.make_credentials( + subject_token_supplier=supplier, + service_account_impersonation_url=SERVICE_ACCOUNT_IMPERSONATION_URL, + scopes=SCOPES, + ) + + self.assert_underlying_credentials_refresh( + credentials=credentials, + audience=AUDIENCE, + subject_token=TEXT_FILE_SUBJECT_TOKEN, + subject_token_type=SUBJECT_TOKEN_TYPE, + token_url=TOKEN_URL, + service_account_impersonation_url=SERVICE_ACCOUNT_IMPERSONATION_URL, + basic_auth_encoding=None, + quota_project_id=None, + used_scopes=SCOPES, + scopes=SCOPES, + default_scopes=None, + ) + + def test_refresh_success_supplier_without_impersonation_url(self): + # Initialize supplier credentials without service account impersonation. + supplier = TestSubjectTokenSupplier(subject_token=JSON_FILE_SUBJECT_TOKEN) + credentials = self.make_credentials( + subject_token_supplier=supplier, scopes=SCOPES + ) + + self.assert_underlying_credentials_refresh( + credentials=credentials, + audience=AUDIENCE, + subject_token=TEXT_FILE_SUBJECT_TOKEN, + subject_token_type=SUBJECT_TOKEN_TYPE, + token_url=TOKEN_URL, + basic_auth_encoding=None, + quota_project_id=None, + used_scopes=SCOPES, + scopes=SCOPES, + default_scopes=None, + ) diff --git a/contrib/python/google-auth/py3/ya.make b/contrib/python/google-auth/py3/ya.make index 6b3decfc4ec..952d1ebdd3a 100644 --- a/contrib/python/google-auth/py3/ya.make +++ b/contrib/python/google-auth/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(2.28.2) +VERSION(2.29.0) LICENSE(Apache-2.0) diff --git a/contrib/python/hypothesis/py3/.dist-info/METADATA b/contrib/python/hypothesis/py3/.dist-info/METADATA index c61fb5cb387..17a8050e565 100644 --- a/contrib/python/hypothesis/py3/.dist-info/METADATA +++ b/contrib/python/hypothesis/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: hypothesis -Version: 6.99.9 +Version: 6.99.12 Summary: A library for property-based testing Home-page: https://hypothesis.works Author: David R. MacIver and Zac Hatfield-Dodds diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py index c35b533ecb4..3f15a974efc 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/data.py @@ -56,6 +56,7 @@ from hypothesis.internal.floats import ( SIGNALING_NAN, SMALLEST_SUBNORMAL, float_to_int, + int_to_float, make_float_clamper, next_down, next_up, @@ -998,13 +999,29 @@ class IRNode: return self.value == shrink_towards if self.ir_type == "float": - # floats shrink "like integers" (for now, anyway), except shrink_towards - # is not configurable and is always 0. + min_value = self.kwargs["min_value"] + max_value = self.kwargs["max_value"] shrink_towards = 0 - shrink_towards = max(self.kwargs["min_value"], shrink_towards) - shrink_towards = min(self.kwargs["max_value"], shrink_towards) - return ir_value_equal("float", self.value, shrink_towards) + if min_value == -math.inf and max_value == math.inf: + return ir_value_equal("float", self.value, shrink_towards) + + if ( + not math.isinf(min_value) + and not math.isinf(max_value) + and math.ceil(min_value) <= math.floor(max_value) + ): + # the interval contains an integer. the simplest integer is the + # one closest to shrink_towards + shrink_towards = max(math.ceil(min_value), shrink_towards) + shrink_towards = min(math.floor(max_value), shrink_towards) + return ir_value_equal("float", self.value, shrink_towards) + + # the real answer here is "the value in [min_value, max_value] with + # the lowest denominator when represented as a fraction". + # It would be good to compute this correctly in the future, but it's + # also not incorrect to be conservative here. + return False if self.ir_type == "boolean": return self.value is False if self.ir_type == "string": @@ -1481,6 +1498,27 @@ class HypothesisProvider(PrimitiveProvider): result = clamped else: result = nasty_floats[i - 1] + # nan values generated via int_to_float break list membership: + # + # >>> n = 18444492273895866368 + # >>> assert math.isnan(int_to_float(n)) + # >>> assert int_to_float(n) not in [int_to_float(n)] + # + # because int_to_float nans are not equal in the sense of either + # `a == b` or `a is b`. + # + # This can lead to flaky errors when collections require unique + # floats. I think what is happening is that in some places we + # provide math.nan, and in others we provide + # int_to_float(float_to_int(math.nan)), and which one gets used + # is not deterministic across test iterations. + # + # As a (temporary?) fix, we'll *always* generate nan values which + # are not equal in the identity sense. + # + # see also https://github.com/HypothesisWorks/hypothesis/issues/3926. + if math.isnan(result): + result = int_to_float(float_to_int(result)) self._draw_float(forced=result, fake_forced=fake_forced) diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py index ce232c67f1f..ae28fcd7419 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/conjecture/shrinker.py @@ -1194,7 +1194,6 @@ class Shrinker: block, lambda b: self.try_shrinking_blocks(targets, b), random=self.random, - full=False, ) @defines_shrink_pass() @@ -1217,7 +1216,7 @@ class Shrinker: node = chooser.choose( self.nodes, - lambda node: node.ir_type == "float" and not node.was_forced + lambda node: node.ir_type == "float" and not node.trivial # avoid shrinking integer-valued floats. In our current ordering, these # are already simpler than all other floats, so it's better to shrink # them in other passes. @@ -1364,7 +1363,6 @@ class Shrinker: self.shrink_target.buffer[u:v], lambda b: self.try_shrinking_blocks((i,), b), random=self.random, - full=False, ) if self.shrink_target is not initial: diff --git a/contrib/python/hypothesis/py3/hypothesis/internal/reflection.py b/contrib/python/hypothesis/py3/hypothesis/internal/reflection.py index a829f097be5..c9ffa6eb2da 100644 --- a/contrib/python/hypothesis/py3/hypothesis/internal/reflection.py +++ b/contrib/python/hypothesis/py3/hypothesis/internal/reflection.py @@ -14,6 +14,7 @@ to really unreasonable lengths to produce pretty output.""" import ast import hashlib import inspect +import linecache import os import re import sys @@ -315,6 +316,10 @@ def extract_lambda_source(f): sig = inspect.signature(f) assert sig.return_annotation in (inspect.Parameter.empty, None), sig + # Using pytest-xdist on Python 3.13, there's an entry in the linecache for + # file "<string>", which then returns nonsense to getsource. Discard it. + linecache.cache.pop("<string>", None) + if sig.parameters: if_confused = f"lambda {str(sig)[1:-1]}: <unknown>" else: @@ -329,7 +334,7 @@ def extract_lambda_source(f): source = source.strip() if "lambda" not in source and sys.platform == "emscripten": # pragma: no cover return if_confused # work around Pyodide bug in inspect.getsource() - assert "lambda" in source + assert "lambda" in source, source tree = None diff --git a/contrib/python/hypothesis/py3/hypothesis/version.py b/contrib/python/hypothesis/py3/hypothesis/version.py index 670380b6195..14d8902f651 100644 --- a/contrib/python/hypothesis/py3/hypothesis/version.py +++ b/contrib/python/hypothesis/py3/hypothesis/version.py @@ -8,5 +8,5 @@ # v. 2.0. If a copy of the MPL was not distributed with this file, You can # obtain one at https://mozilla.org/MPL/2.0/. -__version_info__ = (6, 99, 9) +__version_info__ = (6, 99, 12) __version__ = ".".join(map(str, __version_info__)) diff --git a/contrib/python/hypothesis/py3/ya.make b/contrib/python/hypothesis/py3/ya.make index f6d2f61d06c..604d175b405 100644 --- a/contrib/python/hypothesis/py3/ya.make +++ b/contrib/python/hypothesis/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(6.99.9) +VERSION(6.99.12) LICENSE(MPL-2.0) diff --git a/contrib/python/importlib-metadata/py3/.dist-info/METADATA b/contrib/python/importlib-metadata/py3/.dist-info/METADATA index 38ffc3a5f08..ec8a77c1163 100644 --- a/contrib/python/importlib-metadata/py3/.dist-info/METADATA +++ b/contrib/python/importlib-metadata/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: importlib_metadata -Version: 7.0.2 +Version: 7.1.0 Summary: Read metadata from Python packages Home-page: https://github.com/python/importlib_metadata Author: Jason R. Coombs @@ -32,6 +32,7 @@ Requires-Dist: packaging ; extra == 'testing' Requires-Dist: pyfakefs ; extra == 'testing' Requires-Dist: flufl.flake8 ; extra == 'testing' Requires-Dist: pytest-perf >=0.9.2 ; extra == 'testing' +Requires-Dist: jaraco.test >=5.4 ; extra == 'testing' Requires-Dist: pytest-mypy ; (platform_python_implementation != "PyPy") and extra == 'testing' Requires-Dist: importlib-resources >=1.3 ; (python_version < "3.9") and extra == 'testing' diff --git a/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py b/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py index a069efe55f4..0ed3e8dfb5b 100644 --- a/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py +++ b/contrib/python/importlib-metadata/py3/importlib_metadata/__init__.py @@ -3,7 +3,6 @@ from __future__ import annotations import os import re import abc -import csv import sys import json import email @@ -18,7 +17,8 @@ import itertools import posixpath import collections -from . import _adapters, _meta, _py39compat +from . import _adapters, _meta +from .compat import py39 from ._collections import FreezableDefaultDict, Pair from ._compat import ( NullFinder, @@ -285,7 +285,7 @@ class EntryPoints(tuple): Select entry points from self that match the given parameters (typically group and/or name). """ - return EntryPoints(ep for ep in self if _py39compat.ep_matches(ep, **params)) + return EntryPoints(ep for ep in self if py39.ep_matches(ep, **params)) @property def names(self) -> Set[str]: @@ -527,6 +527,10 @@ class Distribution(DeprecatedNonAbstract): @pass_none def make_files(lines): + # Delay csv import, since Distribution.files is not as widely used + # as other parts of importlib.metadata + import csv + return starmap(make_file, csv.reader(lines)) @pass_none @@ -878,8 +882,9 @@ class MetadataPathFinder(NullFinder, DistributionFinder): of Python that do not have a PathFinder find_distributions(). """ + @classmethod def find_distributions( - self, context=DistributionFinder.Context() + cls, context=DistributionFinder.Context() ) -> Iterable[PathDistribution]: """ Find distributions. @@ -889,7 +894,7 @@ class MetadataPathFinder(NullFinder, DistributionFinder): (or all names if ``None`` indicated) along the paths in the list of directories ``context.path``. """ - found = self._search_paths(context.name, context.path) + found = cls._search_paths(context.name, context.path) return map(PathDistribution, found) @classmethod @@ -900,6 +905,7 @@ class MetadataPathFinder(NullFinder, DistributionFinder): path.search(prepared) for path in map(FastPath, paths) ) + @classmethod def invalidate_caches(cls) -> None: FastPath.__new__.cache_clear() @@ -1054,7 +1060,7 @@ def version(distribution_name: str) -> str: _unique = functools.partial( unique_everseen, - key=_py39compat.normalized_name, + key=py39.normalized_name, ) """ Wrapper for ``distributions`` to return unique distributions by name. diff --git a/contrib/python/importlib-metadata/py3/importlib_metadata/compat/__init__.py b/contrib/python/importlib-metadata/py3/importlib_metadata/compat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/contrib/python/importlib-metadata/py3/importlib_metadata/compat/__init__.py diff --git a/contrib/python/importlib-metadata/py3/importlib_metadata/_py39compat.py b/contrib/python/importlib-metadata/py3/importlib_metadata/compat/py39.py index fc6b82216f8..1f15bd97e6a 100644 --- a/contrib/python/importlib-metadata/py3/importlib_metadata/_py39compat.py +++ b/contrib/python/importlib-metadata/py3/importlib_metadata/compat/py39.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Optional if TYPE_CHECKING: # pragma: no cover # Prevent circular imports on runtime. - from . import Distribution, EntryPoint + from .. import Distribution, EntryPoint else: Distribution = EntryPoint = Any @@ -18,7 +18,7 @@ def normalized_name(dist: Distribution) -> Optional[str]: try: return dist._normalized_name except AttributeError: - from . import Prepared # -> delay to prevent circular imports. + from .. import Prepared # -> delay to prevent circular imports. return Prepared.normalize(getattr(dist, "name", None) or dist.metadata['Name']) @@ -30,7 +30,7 @@ def ep_matches(ep: EntryPoint, **params) -> bool: try: return ep.matches(**params) except AttributeError: - from . import EntryPoint # -> delay to prevent circular imports. + from .. import EntryPoint # -> delay to prevent circular imports. # Reconstruct the EntryPoint object to make sure it is compatible. return EntryPoint(ep.name, ep.value, ep.group).matches(**params) diff --git a/contrib/python/importlib-metadata/py3/ya.make b/contrib/python/importlib-metadata/py3/ya.make index 962408f997b..00d69e1c375 100644 --- a/contrib/python/importlib-metadata/py3/ya.make +++ b/contrib/python/importlib-metadata/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(7.0.2) +VERSION(7.1.0) LICENSE(Apache-2.0) @@ -21,8 +21,9 @@ PY_SRCS( importlib_metadata/_functools.py importlib_metadata/_itertools.py importlib_metadata/_meta.py - importlib_metadata/_py39compat.py importlib_metadata/_text.py + importlib_metadata/compat/__init__.py + importlib_metadata/compat/py39.py importlib_metadata/diagnose.py ) diff --git a/contrib/python/lz4/py2/.dist-info/METADATA b/contrib/python/lz4/py2/.dist-info/METADATA index a896017502e..1f76df926c9 100644 --- a/contrib/python/lz4/py2/.dist-info/METADATA +++ b/contrib/python/lz4/py2/.dist-info/METADATA @@ -18,9 +18,10 @@ Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 Classifier: Programming Language :: Python :: 3.7 Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Provides-Extra: docs Provides-Extra: tests Provides-Extra: flake8 -Provides-Extra: docs +Requires-Dist: future Provides-Extra: docs Requires-Dist: sphinx (>=1.6.0); extra == 'docs' Requires-Dist: sphinx-bootstrap-theme; extra == 'docs' diff --git a/contrib/python/numpy/py3/numpy/__init__.py b/contrib/python/numpy/py3/numpy/__init__.py index a2fbd368beb..91da496a952 100644 --- a/contrib/python/numpy/py3/numpy/__init__.py +++ b/contrib/python/numpy/py3/numpy/__init__.py @@ -390,7 +390,7 @@ else: if sys.platform == "darwin": from . import exceptions with warnings.catch_warnings(record=True) as w: - #_mac_os_check() + _mac_os_check() # Throw runtime error, if the test failed Check for warning and error_message if len(w) > 0: for _wn in w: diff --git a/contrib/python/pytest-mock/py3/.dist-info/METADATA b/contrib/python/pytest-mock/py3/.dist-info/METADATA index 2d35e954469..1ffa498c292 100644 --- a/contrib/python/pytest-mock/py3/.dist-info/METADATA +++ b/contrib/python/pytest-mock/py3/.dist-info/METADATA @@ -1,38 +1,36 @@ Metadata-Version: 2.1 Name: pytest-mock -Version: 3.12.0 +Version: 3.14.0 Summary: Thin-wrapper around the mock package for easier use with pytest -Home-page: https://github.com/pytest-dev/pytest-mock/ -Author: Bruno Oliveira -Author-email: [email protected] +Author-email: Bruno Oliveira <[email protected]> License: MIT +Project-URL: Homepage, https://github.com/pytest-dev/pytest-mock/ Project-URL: Documentation, https://pytest-mock.readthedocs.io/en/latest/ Project-URL: Changelog, https://pytest-mock.readthedocs.io/en/latest/changelog.html Project-URL: Source, https://github.com/pytest-dev/pytest-mock/ Project-URL: Tracker, https://github.com/pytest-dev/pytest-mock/issues -Keywords: pytest mock -Platform: any +Keywords: pytest,mock Classifier: Development Status :: 5 - Production/Stable Classifier: Framework :: Pytest Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: MIT License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3 :: Only Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3 :: Only Classifier: Topic :: Software Development :: Testing Requires-Python: >=3.8 Description-Content-Type: text/x-rst License-File: LICENSE -Requires-Dist: pytest >=5.0 +Requires-Dist: pytest >=6.2.5 Provides-Extra: dev Requires-Dist: pre-commit ; extra == 'dev' -Requires-Dist: tox ; extra == 'dev' Requires-Dist: pytest-asyncio ; extra == 'dev' +Requires-Dist: tox ; extra == 'dev' =========== pytest-mock diff --git a/contrib/python/pytest-mock/py3/pytest_mock/__init__.py b/contrib/python/pytest-mock/py3/pytest_mock/__init__.py index 0afa47c0c17..75fd27afdeb 100644 --- a/contrib/python/pytest-mock/py3/pytest_mock/__init__.py +++ b/contrib/python/pytest-mock/py3/pytest_mock/__init__.py @@ -1,18 +1,22 @@ +from pytest_mock.plugin import AsyncMockType +from pytest_mock.plugin import MockerFixture +from pytest_mock.plugin import MockType +from pytest_mock.plugin import PytestMockWarning from pytest_mock.plugin import class_mocker from pytest_mock.plugin import mocker -from pytest_mock.plugin import MockerFixture from pytest_mock.plugin import module_mocker from pytest_mock.plugin import package_mocker from pytest_mock.plugin import pytest_addoption from pytest_mock.plugin import pytest_configure -from pytest_mock.plugin import PytestMockWarning from pytest_mock.plugin import session_mocker MockFixture = MockerFixture # backward-compatibility only (#204) __all__ = [ + "AsyncMockType", "MockerFixture", "MockFixture", + "MockType", "PytestMockWarning", "pytest_addoption", "pytest_configure", diff --git a/contrib/python/pytest-mock/py3/pytest_mock/_version.py b/contrib/python/pytest-mock/py3/pytest_mock/_version.py index 9e1a89958d8..431bd6d8195 100644 --- a/contrib/python/pytest-mock/py3/pytest_mock/_version.py +++ b/contrib/python/pytest-mock/py3/pytest_mock/_version.py @@ -12,5 +12,5 @@ __version__: str __version_tuple__: VERSION_TUPLE version_tuple: VERSION_TUPLE -__version__ = version = '3.12.0' -__version_tuple__ = version_tuple = (3, 12, 0) +__version__ = version = '3.14.0' +__version_tuple__ = version_tuple = (3, 14, 0) diff --git a/contrib/python/pytest-mock/py3/pytest_mock/plugin.py b/contrib/python/pytest-mock/py3/pytest_mock/plugin.py index 673f98bffb4..1e0a0b20255 100644 --- a/contrib/python/pytest-mock/py3/pytest_mock/plugin.py +++ b/contrib/python/pytest-mock/py3/pytest_mock/plugin.py @@ -2,23 +2,25 @@ import asyncio import builtins import functools import inspect -import sys import unittest.mock import warnings +from dataclasses import dataclass +from dataclasses import field from typing import Any from typing import Callable -from typing import cast from typing import Dict from typing import Generator from typing import Iterable +from typing import Iterator from typing import List from typing import Mapping from typing import Optional -from typing import overload from typing import Tuple from typing import Type from typing import TypeVar from typing import Union +from typing import cast +from typing import overload import pytest @@ -27,22 +29,58 @@ from ._util import parse_ini_boolean _T = TypeVar("_T") -if sys.version_info >= (3, 8): - AsyncMockType = unittest.mock.AsyncMock - MockType = Union[ - unittest.mock.MagicMock, - unittest.mock.AsyncMock, - unittest.mock.NonCallableMagicMock, - ] -else: - AsyncMockType = Any - MockType = Union[unittest.mock.MagicMock, unittest.mock.NonCallableMagicMock] +AsyncMockType = unittest.mock.AsyncMock +MockType = Union[ + unittest.mock.MagicMock, + unittest.mock.AsyncMock, + unittest.mock.NonCallableMagicMock, +] class PytestMockWarning(UserWarning): """Base class for all warnings emitted by pytest-mock.""" +@dataclass +class MockCacheItem: + mock: MockType + patch: Optional[Any] = None + + +@dataclass +class MockCache: + """ + Cache MagicMock and Patcher instances so we can undo them later. + """ + + cache: List[MockCacheItem] = field(default_factory=list) + + def _find(self, mock: MockType) -> MockCacheItem: + for mock_item in self.cache: + if mock_item.mock is mock: + return mock_item + raise ValueError("This mock object is not registered") + + def add(self, mock: MockType, **kwargs: Any) -> MockCacheItem: + self.cache.append(MockCacheItem(mock=mock, **kwargs)) + return self.cache[-1] + + def remove(self, mock: MockType) -> None: + mock_item = self._find(mock) + if mock_item.patch: + mock_item.patch.stop() + self.cache.remove(mock_item) + + def clear(self) -> None: + for mock_item in reversed(self.cache): + if mock_item.patch is not None: + mock_item.patch.stop() + self.cache.clear() + + def __iter__(self) -> Iterator[MockCacheItem]: + return iter(self.cache) + + class MockerFixture: """ Fixture that provides the same interface to functions in the mock module, @@ -50,11 +88,9 @@ class MockerFixture: """ def __init__(self, config: Any) -> None: - self._patches_and_mocks: List[Tuple[Any, unittest.mock.MagicMock]] = [] + self._mock_cache: MockCache = MockCache() self.mock_module = mock_module = get_mock_module(config) - self.patch = self._Patcher( - self._patches_and_mocks, mock_module - ) # type: MockerFixture._Patcher + self.patch = self._Patcher(self._mock_cache, mock_module) # type: MockerFixture._Patcher # aliases for convenience self.Mock = mock_module.Mock self.MagicMock = mock_module.MagicMock @@ -77,7 +113,7 @@ class MockerFixture: m: MockType = self.mock_module.create_autospec( spec, spec_set, instance, **kwargs ) - self._patches_and_mocks.append((None, m)) + self._mock_cache.add(m) return m def resetall( @@ -95,37 +131,33 @@ class MockerFixture: else: supports_reset_mock_with_args = (self.Mock,) - for p, m in self._patches_and_mocks: + for mock_item in self._mock_cache: # See issue #237. - if not hasattr(m, "reset_mock"): + if not hasattr(mock_item.mock, "reset_mock"): continue - if isinstance(m, supports_reset_mock_with_args): - m.reset_mock(return_value=return_value, side_effect=side_effect) + # NOTE: The mock may be a dictionary + if hasattr(mock_item.mock, "spy_return_list"): + mock_item.mock.spy_return_list = [] + if isinstance(mock_item.mock, supports_reset_mock_with_args): + mock_item.mock.reset_mock( + return_value=return_value, side_effect=side_effect + ) else: - m.reset_mock() + mock_item.mock.reset_mock() def stopall(self) -> None: """ Stop all patchers started by this fixture. Can be safely called multiple times. """ - for p, m in reversed(self._patches_and_mocks): - if p is not None: - p.stop() - self._patches_and_mocks.clear() + self._mock_cache.clear() def stop(self, mock: unittest.mock.MagicMock) -> None: """ Stops a previous patch or spy call by passing the ``MagicMock`` object returned by it. """ - for index, (p, m) in enumerate(self._patches_and_mocks): - if mock is m: - p.stop() - del self._patches_and_mocks[index] - break - else: - raise ValueError("This mock object is not registered") + self._mock_cache.remove(mock) def spy(self, obj: object, name: str) -> MockType: """ @@ -137,14 +169,6 @@ class MockerFixture: :return: Spy object. """ method = getattr(obj, name) - if inspect.isclass(obj) and isinstance( - inspect.getattr_static(obj, name), (classmethod, staticmethod) - ): - # Can't use autospec classmethod or staticmethod objects before 3.7 - # see: https://bugs.python.org/issue23078 - autospec = False - else: - autospec = inspect.ismethod(method) or inspect.isfunction(method) def wrapper(*args, **kwargs): spy_obj.spy_return = None @@ -156,6 +180,7 @@ class MockerFixture: raise else: spy_obj.spy_return = r + spy_obj.spy_return_list.append(r) return r async def async_wrapper(*args, **kwargs): @@ -168,6 +193,7 @@ class MockerFixture: raise else: spy_obj.spy_return = r + spy_obj.spy_return_list.append(r) return r if asyncio.iscoroutinefunction(method): @@ -175,8 +201,11 @@ class MockerFixture: else: wrapped = functools.update_wrapper(wrapper, method) + autospec = inspect.ismethod(method) or inspect.isfunction(method) + spy_obj = self.patch.object(obj, name, side_effect=wrapped, autospec=autospec) spy_obj.spy_return = None + spy_obj.spy_return_list = [] spy_obj.spy_exception = None return spy_obj @@ -214,8 +243,8 @@ class MockerFixture: DEFAULT = object() - def __init__(self, patches_and_mocks, mock_module): - self.__patches_and_mocks = patches_and_mocks + def __init__(self, mock_cache, mock_module): + self.__mock_cache = mock_cache self.mock_module = mock_module def _start_patch( @@ -227,22 +256,18 @@ class MockerFixture: """ p = mock_func(*args, **kwargs) mocked: MockType = p.start() - self.__patches_and_mocks.append((p, mocked)) + self.__mock_cache.add(mock=mocked, patch=p) if hasattr(mocked, "reset_mock"): # check if `mocked` is actually a mock object, as depending on autospec or target # parameters `mocked` can be anything if hasattr(mocked, "__enter__") and warn_on_mock_enter: - if sys.version_info >= (3, 8): - depth = 5 - else: - depth = 4 mocked.__enter__.side_effect = lambda: warnings.warn( "Mocks returned by pytest-mock do not need to be used as context managers. " "The mocker fixture automatically undoes mocking at the end of a test. " "This warning can be ignored if it was triggered by mocking a context manager. " "https://pytest-mock.readthedocs.io/en/latest/remarks.html#usage-as-context-manager", PytestMockWarning, - stacklevel=depth, + stacklevel=5, ) return mocked @@ -256,7 +281,7 @@ class MockerFixture: spec_set: Optional[object] = None, autospec: Optional[object] = None, new_callable: object = None, - **kwargs: Any + **kwargs: Any, ) -> MockType: """API to mock.patch.object""" if new is self.DEFAULT: @@ -272,7 +297,7 @@ class MockerFixture: spec_set=spec_set, autospec=autospec, new_callable=new_callable, - **kwargs + **kwargs, ) def context_manager( @@ -285,7 +310,7 @@ class MockerFixture: spec_set: Optional[builtins.object] = None, autospec: Optional[builtins.object] = None, new_callable: builtins.object = None, - **kwargs: Any + **kwargs: Any, ) -> MockType: """This is equivalent to mock.patch.object except that the returned mock does not issue a warning when used as a context manager.""" @@ -302,7 +327,7 @@ class MockerFixture: spec_set=spec_set, autospec=autospec, new_callable=new_callable, - **kwargs + **kwargs, ) def multiple( @@ -313,7 +338,7 @@ class MockerFixture: spec_set: Optional[builtins.object] = None, autospec: Optional[builtins.object] = None, new_callable: Optional[builtins.object] = None, - **kwargs: Any + **kwargs: Any, ) -> Dict[str, MockType]: """API to mock.patch.multiple""" return self._start_patch( @@ -325,7 +350,7 @@ class MockerFixture: spec_set=spec_set, autospec=autospec, new_callable=new_callable, - **kwargs + **kwargs, ) def dict( @@ -333,7 +358,7 @@ class MockerFixture: in_dict: Union[Mapping[Any, Any], str], values: Union[Mapping[Any, Any], Iterable[Tuple[Any, Any]]] = (), clear: bool = False, - **kwargs: Any + **kwargs: Any, ) -> Any: """API to mock.patch.dict""" return self._start_patch( @@ -342,7 +367,7 @@ class MockerFixture: in_dict, values=values, clear=clear, - **kwargs + **kwargs, ) @overload @@ -355,9 +380,8 @@ class MockerFixture: spec_set: Optional[builtins.object] = ..., autospec: Optional[builtins.object] = ..., new_callable: None = ..., - **kwargs: Any - ) -> MockType: - ... + **kwargs: Any, + ) -> MockType: ... @overload def __call__( @@ -369,9 +393,8 @@ class MockerFixture: spec_set: Optional[builtins.object] = ..., autospec: Optional[builtins.object] = ..., new_callable: None = ..., - **kwargs: Any - ) -> _T: - ... + **kwargs: Any, + ) -> _T: ... @overload def __call__( @@ -383,9 +406,8 @@ class MockerFixture: spec_set: Optional[builtins.object], autospec: Optional[builtins.object], new_callable: Callable[[], _T], - **kwargs: Any - ) -> _T: - ... + **kwargs: Any, + ) -> _T: ... @overload def __call__( @@ -398,9 +420,8 @@ class MockerFixture: autospec: Optional[builtins.object] = ..., *, new_callable: Callable[[], _T], - **kwargs: Any - ) -> _T: - ... + **kwargs: Any, + ) -> _T: ... def __call__( self, @@ -411,7 +432,7 @@ class MockerFixture: spec_set: Optional[builtins.object] = None, autospec: Optional[builtins.object] = None, new_callable: Optional[Callable[[], Any]] = None, - **kwargs: Any + **kwargs: Any, ) -> Any: """API to mock.patch""" if new is self.DEFAULT: @@ -426,7 +447,7 @@ class MockerFixture: spec_set=spec_set, autospec=autospec, new_callable=new_callable, - **kwargs + **kwargs, ) diff --git a/contrib/python/pytest-mock/py3/ya.make b/contrib/python/pytest-mock/py3/ya.make index 40f864e45da..7b9bcc9b14f 100644 --- a/contrib/python/pytest-mock/py3/ya.make +++ b/contrib/python/pytest-mock/py3/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(3.12.0) +VERSION(3.14.0) LICENSE(MIT) diff --git a/contrib/python/requests-oauthlib/.dist-info/METADATA b/contrib/python/requests-oauthlib/.dist-info/METADATA index 1029e2ff65a..0a62eb5b6c3 100644 --- a/contrib/python/requests-oauthlib/.dist-info/METADATA +++ b/contrib/python/requests-oauthlib/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: requests-oauthlib -Version: 1.4.0 +Version: 2.0.0 Summary: OAuthlib authentication support for Requests. Home-page: https://github.com/requests/requests-oauthlib Author: Kenneth Reitz @@ -11,8 +11,6 @@ Classifier: Intended Audience :: Developers Classifier: Natural Language :: English Classifier: License :: OSI Approved :: BSD License Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 @@ -25,7 +23,7 @@ Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy -Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +Requires-Python: >=3.4 Description-Content-Type: text/x-rst License-File: LICENSE Requires-Dist: oauthlib >=3.0.0 @@ -96,7 +94,7 @@ To install requests and requests_oauthlib you can use pip: History ------- -v1.4.0 (27 Feb 2024) +v2.0.0 (22 March 2024) ++++++++++++++++++++++++ Full set of changes are in [github](https://github.com/requests/requests-oauthlib/milestone/4?closed=1). @@ -117,6 +115,11 @@ Additions & changes: - Updated dependencies - Cleanup some docs and examples +v1.4.0 (27 Feb 2024) +++++++++++++++++++++++++ + +- Version 2.0.0 published initially as 1.4.0, it was yanked eventually. + v1.3.1 (21 January 2022) ++++++++++++++++++++++++ diff --git a/contrib/python/requests-oauthlib/requests_oauthlib/__init__.py b/contrib/python/requests-oauthlib/requests_oauthlib/__init__.py index a71064cca08..865d72fb768 100644 --- a/contrib/python/requests-oauthlib/requests_oauthlib/__init__.py +++ b/contrib/python/requests-oauthlib/requests_oauthlib/__init__.py @@ -6,7 +6,7 @@ from .oauth1_session import OAuth1Session from .oauth2_auth import OAuth2 from .oauth2_session import OAuth2Session, TokenUpdated -__version__ = "1.4.0" +__version__ = "2.0.0" import requests diff --git a/contrib/python/requests-oauthlib/tests/examples/__init__.py b/contrib/python/requests-oauthlib/tests/examples/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/contrib/python/requests-oauthlib/tests/examples/__init__.py diff --git a/contrib/python/requests-oauthlib/ya.make b/contrib/python/requests-oauthlib/ya.make index d1a60dc1009..da3d8536ed6 100644 --- a/contrib/python/requests-oauthlib/ya.make +++ b/contrib/python/requests-oauthlib/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(1.4.0) +VERSION(2.0.0) LICENSE(ISC) diff --git a/contrib/python/ruamel.yaml.clib/py2/.dist-info/METADATA b/contrib/python/ruamel.yaml.clib/py2/.dist-info/METADATA index 6c952ac9d6b..df2e2797e24 100644 --- a/contrib/python/ruamel.yaml.clib/py2/.dist-info/METADATA +++ b/contrib/python/ruamel.yaml.clib/py2/.dist-info/METADATA @@ -22,7 +22,6 @@ Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Topic :: Software Development :: Libraries :: Python Modules Description-Content-Type: text/x-rst - ruamel.yaml.clib ================ diff --git a/contrib/python/simplejson/py2/.dist-info/METADATA b/contrib/python/simplejson/py2/.dist-info/METADATA index 030823c2a19..498829fb7e7 100644 --- a/contrib/python/simplejson/py2/.dist-info/METADATA +++ b/contrib/python/simplejson/py2/.dist-info/METADATA @@ -30,7 +30,6 @@ Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Software Development :: Libraries :: Python Modules Requires-Python: >=2.5, !=3.0.*, !=3.1.*, !=3.2.* -License-File: LICENSE.txt simplejson ---------- @@ -66,3 +65,5 @@ Python 2.2. This is based on a very old version of simplejson, is not maintained, and should only be used as a last resort. .. _python2.2: https://github.com/simplejson/simplejson/tree/python2.2 + + diff --git a/contrib/python/tzlocal/py2/.dist-info/METADATA b/contrib/python/tzlocal/py2/.dist-info/METADATA deleted file mode 100644 index 7bafd427d5c..00000000000 --- a/contrib/python/tzlocal/py2/.dist-info/METADATA +++ /dev/null @@ -1,326 +0,0 @@ -Metadata-Version: 2.1 -Name: tzlocal -Version: 2.1 -Summary: tzinfo object for the local timezone -Home-page: https://github.com/regebro/tzlocal -Author: Lennart Regebro -Author-email: [email protected] -License: MIT -Keywords: timezone pytz -Platform: UNKNOWN -Classifier: Development Status :: 5 - Production/Stable -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: Microsoft :: Windows -Classifier: Operating System :: Unix -Classifier: Operating System :: MacOS :: MacOS X -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Requires-Dist: pytz - -tzlocal -======= - -This Python module returns a ``tzinfo`` object with the local timezone information under Unix and Win-32. -It requires ``pytz``, and returns ``pytz`` ``tzinfo`` objects. - -This module attempts to fix a glaring hole in ``pytz``, that there is no way to -get the local timezone information, unless you know the zoneinfo name, and -under several Linux distros that's hard or impossible to figure out. - -Also, with Windows different timezone system using pytz isn't of much use -unless you separately configure the zoneinfo timezone name. - -With ``tzlocal`` you only need to call ``get_localzone()`` and you will get a -``tzinfo`` object with the local time zone info. On some Unices you will still -not get to know what the timezone name is, but you don't need that when you -have the tzinfo file. However, if the timezone name is readily available it -will be used. - - -Supported systems ------------------ - -These are the systems that are in theory supported: - - * Windows 2000 and later - - * Any unix-like system with a ``/etc/localtime`` or ``/usr/local/etc/localtime`` - -If you have one of the above systems and it does not work, it's a bug. -Please report it. - -Please note that if you getting a time zone called ``local``, this is not a bug, it's -actually the main feature of ``tzlocal``, that even if your system does NOT have a configuration file -with the zoneinfo name of your time zone, it will still work. - -You can also use ``tzlocal`` to get the name of your local timezone, but only if your system is -configured to make that possible. ``tzlocal`` looks for the timezone name in ``/etc/timezone``, ``/var/db/zoneinfo``, -``/etc/sysconfig/clock`` and ``/etc/conf.d/clock``. If your ``/etc/localtime`` is a symlink it can also extract the -name from that symlink. - -If you need the name of your local time zone, then please make sure your system is properly configured to allow that. -If it isn't configured, tzlocal will default to UTC. - -Usage ------ - -Load the local timezone: - - >>> from tzlocal import get_localzone - >>> tz = get_localzone() - >>> tz - <DstTzInfo 'Europe/Warsaw' WMT+1:24:00 STD> - -Create a local datetime: - - >>> from datetime import datetime - >>> dt = tz.localize(datetime(2015, 4, 10, 7, 22)) - >>> dt - datetime.datetime(2015, 4, 10, 7, 22, tzinfo=<DstTzInfo 'Europe/Warsaw' CEST+2:00:00 DST>) - -Lookup another timezone with `pytz`: - - >>> import pytz - >>> eastern = pytz.timezone('US/Eastern') - -Convert the datetime: - - >>> dt.astimezone(eastern) - datetime.datetime(2015, 4, 10, 1, 22, tzinfo=<DstTzInfo 'US/Eastern' EDT-1 day, 20:00:00 DST>) - - -Maintainer ----------- - -* Lennart Regebro, [email protected] - -Contributors ------------- - -* Marc Van Olmen -* Benjamen Meyer -* Manuel Ebert -* Xiaokun Zhu -* Cameris -* Edward Betts -* McK KIM -* Cris Ewing -* Ayala Shachar -* Lev Maximov -* Jakub Wilk -* John Quarles -* Preston Landers -* Victor Torres -* Jean Jordaan -* Zackary Welch -* Mickaël Schoentgen -* Gabriel Corona - -(Sorry if I forgot someone) - -License -------- - -* MIT https://opensource.org/licenses/MIT - - -Changes -======= - -2.1 (2020-05-08) ----------------- - -- No changes. - - -2.1b1 (2020-02-08) ------------------- - -- The is_dst flag is wrong for Europe/Dublin on some Unix releases. - I changed to another way of determining if DST is in effect or not. - -- Added support for Python 3.7 and 3.8. Dropped 3.5 although it still works. - - -2.0.0 (2019-07-23) ------------------- - -- No differences since 2.0.0b3 - -Major differences since 1.5.1 -............................. - -- When no time zone configuration can be find, tzlocal now return UTC. - This is a major difference from 1.x, where an exception would be raised. - This change is because Docker images often have no configuration at all, - and the unix utilities will then default to UTC, so we follow that. - -- If tzlocal on Unix finds a timezone name in a /etc config file, then - tzlocal now verifies that the timezone it fouds has the same offset as - the local computer is configured with. If it doesn't, something is - configured incorrectly. (Victor Torres, regebro) - -- Get timezone via Termux `getprop` wrapper on Android. It's not officially - supported because we can't test it, but at least we make an effort. - (Jean Jordaan) - -Minor differences and bug fixes -............................... - -- Skip comment lines when parsing /etc/timezone. (Edward Betts) - -- Don't load timezone from current directory. (Gabriel Corona) - -- Now verifies that the config files actually contain something before - reading them. (Zackary Welch, regebro) - -- Got rid of a BytesWarning (Mickaël Schoentgen) - -- Now handles if config file paths exists, but are directories. - -- Moved tests out from distributions - -- Support wheels - - -1.5.1 (2017-12-01) ------------------- - -- 1.5 had a bug that slipped through testing, fixed that, - increased test coverage. - - -1.5 (2017-11-30) ----------------- - -- No longer treats macOS as special, but as a unix. - -- get_windows_info.py is renamed to update_windows_mappings.py - -- Windows mappings now also contain mappings from deprecated zoneinfo names. - (Preston-Landers, regebro) - - -1.4 (2017-04-18) ----------------- - -- I use MIT on my other projects, so relicensing. - - -1.4b1 (2017-04-14) ------------------- - -- Dropping support for Python versions nobody uses (2.5, 3.1, 3.2), adding 3.6 - Python 3.1 and 3.2 still works, 2.5 has been broken for some time. - -- Ayalash's OS X fix didn't work on Python 2.7, fixed that. - - -1.3.2 (2017-04-12) ------------------- - -- Ensure closing of subprocess on OS X (ayalash) - -- Removed unused imports (jwilk) - -- Closes stdout and stderr to get rid of ResourceWarnings (johnwquarles) - -- Updated Windows timezones (axil) - - -1.3 (2016-10-15) ----------------- - -- #34: Added support for /var/db/zoneinfo - - -1.2.2 (2016-03-02) ------------------- - -- #30: Fixed a bug on OS X. - - -1.2.1 (2016-02-28) ------------------- - -- Tests failed if TZ was set in the environment. (EdwardBetts) - -- Replaces os.popen() with subprocess.Popen() for OS X to - handle when systemsetup doesn't exist. (mckabi, cewing) - - -1.2 (2015-06-14) ----------------- - -- Systemd stores no time zone name, forcing us to look at the name of the file - that localtime symlinks to. (cameris) - - -1.1.2 (2014-10-18) ------------------- - -- Timezones that has 3 items did not work on Mac OS X. - (Marc Van Olmen) - -- Now doesn't fail if the TZ environment variable isn't an Olsen time zone. - -- Some timezones on Windows can apparently be empty (perhaps the are deleted). - Now these are ignored. - (Xiaokun Zhu) - - -1.1.1 (2014-01-29) ------------------- - -- I forgot to add Etc/UTC as an alias for Etc/GMT. - - -1.1 (2014-01-28) ----------------- - -- Adding better support for OS X. - -- Added support to map from tzdata/Olsen names to Windows names. - (Thanks to Benjamen Meyer). - - -1.0 (2013-05-29) ----------------- - -- Fixed some more cases where spaces needs replacing with underscores. - -- Better handling of misconfigured /etc/timezone. - -- Better error message on Windows if we can't find a timezone at all. - - -0.3 (2012-09-13) ----------------- - -- Windows 7 support. - -- Python 2.5 supported; because it only needed a __future__ import. - -- Python 3.3 tested, it worked. - -- Got rid of relative imports, because I don't actually like them, - so I don't know why I used them in the first place. - -- For each Windows zone, use the default zoneinfo zone, not the last one. - - -0.2 (2012-09-12) ----------------- - -- Python 3 support. - - -0.1 (2012-09-11) ----------------- - -- Initial release. - - diff --git a/contrib/python/tzlocal/py2/.dist-info/top_level.txt b/contrib/python/tzlocal/py2/.dist-info/top_level.txt deleted file mode 100644 index cd5e9b12a4b..00000000000 --- a/contrib/python/tzlocal/py2/.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -tzlocal diff --git a/contrib/python/tzlocal/py2/LICENSE.txt b/contrib/python/tzlocal/py2/LICENSE.txt deleted file mode 100644 index 9be1d2fe595..00000000000 --- a/contrib/python/tzlocal/py2/LICENSE.txt +++ /dev/null @@ -1,19 +0,0 @@ -Copyright 2011-2017 Lennart Regebro - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/contrib/python/tzlocal/py2/README.rst b/contrib/python/tzlocal/py2/README.rst deleted file mode 100644 index eaa0bdd8132..00000000000 --- a/contrib/python/tzlocal/py2/README.rst +++ /dev/null @@ -1,105 +0,0 @@ -tzlocal -======= - -This Python module returns a ``tzinfo`` object with the local timezone information under Unix and Win-32. -It requires ``pytz``, and returns ``pytz`` ``tzinfo`` objects. - -This module attempts to fix a glaring hole in ``pytz``, that there is no way to -get the local timezone information, unless you know the zoneinfo name, and -under several Linux distros that's hard or impossible to figure out. - -Also, with Windows different timezone system using pytz isn't of much use -unless you separately configure the zoneinfo timezone name. - -With ``tzlocal`` you only need to call ``get_localzone()`` and you will get a -``tzinfo`` object with the local time zone info. On some Unices you will still -not get to know what the timezone name is, but you don't need that when you -have the tzinfo file. However, if the timezone name is readily available it -will be used. - - -Supported systems ------------------ - -These are the systems that are in theory supported: - - * Windows 2000 and later - - * Any unix-like system with a ``/etc/localtime`` or ``/usr/local/etc/localtime`` - -If you have one of the above systems and it does not work, it's a bug. -Please report it. - -Please note that if you getting a time zone called ``local``, this is not a bug, it's -actually the main feature of ``tzlocal``, that even if your system does NOT have a configuration file -with the zoneinfo name of your time zone, it will still work. - -You can also use ``tzlocal`` to get the name of your local timezone, but only if your system is -configured to make that possible. ``tzlocal`` looks for the timezone name in ``/etc/timezone``, ``/var/db/zoneinfo``, -``/etc/sysconfig/clock`` and ``/etc/conf.d/clock``. If your ``/etc/localtime`` is a symlink it can also extract the -name from that symlink. - -If you need the name of your local time zone, then please make sure your system is properly configured to allow that. -If it isn't configured, tzlocal will default to UTC. - -Usage ------ - -Load the local timezone: - - >>> from tzlocal import get_localzone - >>> tz = get_localzone() - >>> tz - <DstTzInfo 'Europe/Warsaw' WMT+1:24:00 STD> - -Create a local datetime: - - >>> from datetime import datetime - >>> dt = tz.localize(datetime(2015, 4, 10, 7, 22)) - >>> dt - datetime.datetime(2015, 4, 10, 7, 22, tzinfo=<DstTzInfo 'Europe/Warsaw' CEST+2:00:00 DST>) - -Lookup another timezone with `pytz`: - - >>> import pytz - >>> eastern = pytz.timezone('US/Eastern') - -Convert the datetime: - - >>> dt.astimezone(eastern) - datetime.datetime(2015, 4, 10, 1, 22, tzinfo=<DstTzInfo 'US/Eastern' EDT-1 day, 20:00:00 DST>) - - -Maintainer ----------- - -* Lennart Regebro, [email protected] - -Contributors ------------- - -* Marc Van Olmen -* Benjamen Meyer -* Manuel Ebert -* Xiaokun Zhu -* Cameris -* Edward Betts -* McK KIM -* Cris Ewing -* Ayala Shachar -* Lev Maximov -* Jakub Wilk -* John Quarles -* Preston Landers -* Victor Torres -* Jean Jordaan -* Zackary Welch -* Mickaël Schoentgen -* Gabriel Corona - -(Sorry if I forgot someone) - -License -------- - -* MIT https://opensource.org/licenses/MIT diff --git a/contrib/python/tzlocal/py2/tzlocal/__init__.py b/contrib/python/tzlocal/py2/tzlocal/__init__.py deleted file mode 100644 index c8196d66d9e..00000000000 --- a/contrib/python/tzlocal/py2/tzlocal/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -import sys -if sys.platform == 'win32': - from tzlocal.win32 import get_localzone, reload_localzone -else: - from tzlocal.unix import get_localzone, reload_localzone diff --git a/contrib/python/tzlocal/py2/tzlocal/unix.py b/contrib/python/tzlocal/py2/tzlocal/unix.py deleted file mode 100644 index 8574965a5a9..00000000000 --- a/contrib/python/tzlocal/py2/tzlocal/unix.py +++ /dev/null @@ -1,174 +0,0 @@ -import os -import pytz -import re -import warnings - -from tzlocal import utils - -_cache_tz = None - - -def _tz_from_env(tzenv): - if tzenv[0] == ':': - tzenv = tzenv[1:] - - # TZ specifies a file - if os.path.isabs(tzenv) and os.path.exists(tzenv): - with open(tzenv, 'rb') as tzfile: - return pytz.tzfile.build_tzinfo('local', tzfile) - - # TZ specifies a zoneinfo zone. - try: - tz = pytz.timezone(tzenv) - # That worked, so we return this: - return tz - except pytz.UnknownTimeZoneError: - raise pytz.UnknownTimeZoneError( - "tzlocal() does not support non-zoneinfo timezones like %s. \n" - "Please use a timezone in the form of Continent/City") - - -def _try_tz_from_env(): - tzenv = os.environ.get('TZ') - if tzenv: - try: - return _tz_from_env(tzenv) - except pytz.UnknownTimeZoneError: - pass - - -def _get_localzone(_root='/'): - """Tries to find the local timezone configuration. - - This method prefers finding the timezone name and passing that to pytz, - over passing in the localtime file, as in the later case the zoneinfo - name is unknown. - - The parameter _root makes the function look for files like /etc/localtime - beneath the _root directory. This is primarily used by the tests. - In normal usage you call the function without parameters.""" - - tzenv = _try_tz_from_env() - if tzenv: - return tzenv - - # Are we under Termux on Android? - if os.path.exists('/system/bin/getprop'): - import subprocess - androidtz = subprocess.check_output(['getprop', 'persist.sys.timezone']).strip().decode() - return pytz.timezone(androidtz) - - # Now look for distribution specific configuration files - # that contain the timezone name. - for configfile in ('etc/timezone', 'var/db/zoneinfo'): - tzpath = os.path.join(_root, configfile) - try: - with open(tzpath, 'rb') as tzfile: - data = tzfile.read() - - # Issue #3 was that /etc/timezone was a zoneinfo file. - # That's a misconfiguration, but we need to handle it gracefully: - if data[:5] == b'TZif2': - continue - - etctz = data.strip().decode() - if not etctz: - # Empty file, skip - continue - for etctz in data.decode().splitlines(): - # Get rid of host definitions and comments: - if ' ' in etctz: - etctz, dummy = etctz.split(' ', 1) - if '#' in etctz: - etctz, dummy = etctz.split('#', 1) - if not etctz: - continue - tz = pytz.timezone(etctz.replace(' ', '_')) - if _root == '/': - # We are using a file in etc to name the timezone. - # Verify that the timezone specified there is actually used: - utils.assert_tz_offset(tz) - return tz - - except IOError: - # File doesn't exist or is a directory - continue - - # CentOS has a ZONE setting in /etc/sysconfig/clock, - # OpenSUSE has a TIMEZONE setting in /etc/sysconfig/clock and - # Gentoo has a TIMEZONE setting in /etc/conf.d/clock - # We look through these files for a timezone: - - zone_re = re.compile(r'\s*ZONE\s*=\s*\"') - timezone_re = re.compile(r'\s*TIMEZONE\s*=\s*\"') - end_re = re.compile('\"') - - for filename in ('etc/sysconfig/clock', 'etc/conf.d/clock'): - tzpath = os.path.join(_root, filename) - try: - with open(tzpath, 'rt') as tzfile: - data = tzfile.readlines() - - for line in data: - # Look for the ZONE= setting. - match = zone_re.match(line) - if match is None: - # No ZONE= setting. Look for the TIMEZONE= setting. - match = timezone_re.match(line) - if match is not None: - # Some setting existed - line = line[match.end():] - etctz = line[:end_re.search(line).start()] - - # We found a timezone - tz = pytz.timezone(etctz.replace(' ', '_')) - if _root == '/': - # We are using a file in etc to name the timezone. - # Verify that the timezone specified there is actually used: - utils.assert_tz_offset(tz) - return tz - - except IOError: - # File doesn't exist or is a directory - continue - - # systemd distributions use symlinks that include the zone name, - # see manpage of localtime(5) and timedatectl(1) - tzpath = os.path.join(_root, 'etc/localtime') - if os.path.exists(tzpath) and os.path.islink(tzpath): - tzpath = os.path.realpath(tzpath) - start = tzpath.find("/")+1 - while start != 0: - tzpath = tzpath[start:] - try: - return pytz.timezone(tzpath) - except pytz.UnknownTimeZoneError: - pass - start = tzpath.find("/")+1 - - # No explicit setting existed. Use localtime - for filename in ('etc/localtime', 'usr/local/etc/localtime'): - tzpath = os.path.join(_root, filename) - - if not os.path.exists(tzpath): - continue - with open(tzpath, 'rb') as tzfile: - return pytz.tzfile.build_tzinfo('local', tzfile) - - warnings.warn('Can not find any timezone configuration, defaulting to UTC.') - return pytz.utc - -def get_localzone(): - """Get the computers configured local timezone, if any.""" - global _cache_tz - if _cache_tz is None: - _cache_tz = _get_localzone() - - return _cache_tz - - -def reload_localzone(): - """Reload the cached localzone. You need to call this if the timezone has changed.""" - global _cache_tz - _cache_tz = _get_localzone() - return _cache_tz diff --git a/contrib/python/tzlocal/py2/tzlocal/utils.py b/contrib/python/tzlocal/py2/tzlocal/utils.py deleted file mode 100644 index 5a6779903de..00000000000 --- a/contrib/python/tzlocal/py2/tzlocal/utils.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- -import time -import datetime -import calendar - - -def get_system_offset(): - """Get system's timezone offset using built-in library time. - - For the Timezone constants (altzone, daylight, timezone, and tzname), the - value is determined by the timezone rules in effect at module load time or - the last time tzset() is called and may be incorrect for times in the past. - - To keep compatibility with Windows, we're always importing time module here. - """ - - localtime = calendar.timegm(time.localtime()) - gmtime = calendar.timegm(time.gmtime()) - offset = gmtime - localtime - # We could get the localtime and gmtime on either side of a second switch - # so we check that the difference is less than one minute, because nobody - # has that small DST differences. - if abs(offset - time.altzone) < 60: - return -time.altzone - else: - return -time.timezone - - -def get_tz_offset(tz): - """Get timezone's offset using built-in function datetime.utcoffset().""" - return int(datetime.datetime.now(tz).utcoffset().total_seconds()) - - -def assert_tz_offset(tz): - """Assert that system's timezone offset equals to the timezone offset found. - - If they don't match, we probably have a misconfiguration, for example, an - incorrect timezone set in /etc/timezone file in systemd distributions.""" - tz_offset = get_tz_offset(tz) - system_offset = get_system_offset() - if tz_offset != system_offset: - msg = ('Timezone offset does not match system offset: {0} != {1}. ' - 'Please, check your config files.').format( - tz_offset, system_offset - ) - raise ValueError(msg) diff --git a/contrib/python/tzlocal/py2/tzlocal/win32.py b/contrib/python/tzlocal/py2/tzlocal/win32.py deleted file mode 100644 index fcc42a23f3f..00000000000 --- a/contrib/python/tzlocal/py2/tzlocal/win32.py +++ /dev/null @@ -1,104 +0,0 @@ -try: - import _winreg as winreg -except ImportError: - import winreg - -import pytz - -from tzlocal.windows_tz import win_tz -from tzlocal import utils - -_cache_tz = None - - -def valuestodict(key): - """Convert a registry key's values to a dictionary.""" - dict = {} - size = winreg.QueryInfoKey(key)[1] - for i in range(size): - data = winreg.EnumValue(key, i) - dict[data[0]] = data[1] - return dict - - -def get_localzone_name(): - # Windows is special. It has unique time zone names (in several - # meanings of the word) available, but unfortunately, they can be - # translated to the language of the operating system, so we need to - # do a backwards lookup, by going through all time zones and see which - # one matches. - handle = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) - - TZLOCALKEYNAME = r"SYSTEM\CurrentControlSet\Control\TimeZoneInformation" - localtz = winreg.OpenKey(handle, TZLOCALKEYNAME) - keyvalues = valuestodict(localtz) - localtz.Close() - - if 'TimeZoneKeyName' in keyvalues: - # Windows 7 (and Vista?) - - # For some reason this returns a string with loads of NUL bytes at - # least on some systems. I don't know if this is a bug somewhere, I - # just work around it. - tzkeyname = keyvalues['TimeZoneKeyName'].split('\x00', 1)[0] - else: - # Windows 2000 or XP - - # This is the localized name: - tzwin = keyvalues['StandardName'] - - # Open the list of timezones to look up the real name: - TZKEYNAME = r"SOFTWARE\Microsoft\Windows NT\CurrentVersion\Time Zones" - tzkey = winreg.OpenKey(handle, TZKEYNAME) - - # Now, match this value to Time Zone information - tzkeyname = None - for i in range(winreg.QueryInfoKey(tzkey)[0]): - subkey = winreg.EnumKey(tzkey, i) - sub = winreg.OpenKey(tzkey, subkey) - data = valuestodict(sub) - sub.Close() - try: - if data['Std'] == tzwin: - tzkeyname = subkey - break - except KeyError: - # This timezone didn't have proper configuration. - # Ignore it. - pass - - tzkey.Close() - handle.Close() - - if tzkeyname is None: - raise LookupError('Can not find Windows timezone configuration') - - timezone = win_tz.get(tzkeyname) - if timezone is None: - # Nope, that didn't work. Try adding "Standard Time", - # it seems to work a lot of times: - timezone = win_tz.get(tzkeyname + " Standard Time") - - # Return what we have. - if timezone is None: - raise pytz.UnknownTimeZoneError('Can not find timezone ' + tzkeyname) - - return timezone - - -def get_localzone(): - """Returns the zoneinfo-based tzinfo object that matches the Windows-configured timezone.""" - global _cache_tz - if _cache_tz is None: - _cache_tz = pytz.timezone(get_localzone_name()) - - utils.assert_tz_offset(_cache_tz) - return _cache_tz - - -def reload_localzone(): - """Reload the cached localzone. You need to call this if the timezone has changed.""" - global _cache_tz - _cache_tz = pytz.timezone(get_localzone_name()) - utils.assert_tz_offset(_cache_tz) - return _cache_tz diff --git a/contrib/python/tzlocal/py2/tzlocal/windows_tz.py b/contrib/python/tzlocal/py2/tzlocal/windows_tz.py deleted file mode 100644 index 86ba807d060..00000000000 --- a/contrib/python/tzlocal/py2/tzlocal/windows_tz.py +++ /dev/null @@ -1,697 +0,0 @@ -# This file is autogenerated by the update_windows_mapping.py script -# Do not edit. -win_tz = {'AUS Central Standard Time': 'Australia/Darwin', - 'AUS Eastern Standard Time': 'Australia/Sydney', - 'Afghanistan Standard Time': 'Asia/Kabul', - 'Alaskan Standard Time': 'America/Anchorage', - 'Aleutian Standard Time': 'America/Adak', - 'Altai Standard Time': 'Asia/Barnaul', - 'Arab Standard Time': 'Asia/Riyadh', - 'Arabian Standard Time': 'Asia/Dubai', - 'Arabic Standard Time': 'Asia/Baghdad', - 'Argentina Standard Time': 'America/Buenos_Aires', - 'Astrakhan Standard Time': 'Europe/Astrakhan', - 'Atlantic Standard Time': 'America/Halifax', - 'Aus Central W. Standard Time': 'Australia/Eucla', - 'Azerbaijan Standard Time': 'Asia/Baku', - 'Azores Standard Time': 'Atlantic/Azores', - 'Bahia Standard Time': 'America/Bahia', - 'Bangladesh Standard Time': 'Asia/Dhaka', - 'Belarus Standard Time': 'Europe/Minsk', - 'Bougainville Standard Time': 'Pacific/Bougainville', - 'Canada Central Standard Time': 'America/Regina', - 'Cape Verde Standard Time': 'Atlantic/Cape_Verde', - 'Caucasus Standard Time': 'Asia/Yerevan', - 'Cen. Australia Standard Time': 'Australia/Adelaide', - 'Central America Standard Time': 'America/Guatemala', - 'Central Asia Standard Time': 'Asia/Almaty', - 'Central Brazilian Standard Time': 'America/Cuiaba', - 'Central Europe Standard Time': 'Europe/Budapest', - 'Central European Standard Time': 'Europe/Warsaw', - 'Central Pacific Standard Time': 'Pacific/Guadalcanal', - 'Central Standard Time': 'America/Chicago', - 'Central Standard Time (Mexico)': 'America/Mexico_City', - 'Chatham Islands Standard Time': 'Pacific/Chatham', - 'China Standard Time': 'Asia/Shanghai', - 'Cuba Standard Time': 'America/Havana', - 'Dateline Standard Time': 'Etc/GMT+12', - 'E. Africa Standard Time': 'Africa/Nairobi', - 'E. Australia Standard Time': 'Australia/Brisbane', - 'E. Europe Standard Time': 'Europe/Chisinau', - 'E. South America Standard Time': 'America/Sao_Paulo', - 'Easter Island Standard Time': 'Pacific/Easter', - 'Eastern Standard Time': 'America/New_York', - 'Eastern Standard Time (Mexico)': 'America/Cancun', - 'Egypt Standard Time': 'Africa/Cairo', - 'Ekaterinburg Standard Time': 'Asia/Yekaterinburg', - 'FLE Standard Time': 'Europe/Kiev', - 'Fiji Standard Time': 'Pacific/Fiji', - 'GMT Standard Time': 'Europe/London', - 'GTB Standard Time': 'Europe/Bucharest', - 'Georgian Standard Time': 'Asia/Tbilisi', - 'Greenland Standard Time': 'America/Godthab', - 'Greenwich Standard Time': 'Atlantic/Reykjavik', - 'Haiti Standard Time': 'America/Port-au-Prince', - 'Hawaiian Standard Time': 'Pacific/Honolulu', - 'India Standard Time': 'Asia/Calcutta', - 'Iran Standard Time': 'Asia/Tehran', - 'Israel Standard Time': 'Asia/Jerusalem', - 'Jordan Standard Time': 'Asia/Amman', - 'Kaliningrad Standard Time': 'Europe/Kaliningrad', - 'Korea Standard Time': 'Asia/Seoul', - 'Libya Standard Time': 'Africa/Tripoli', - 'Line Islands Standard Time': 'Pacific/Kiritimati', - 'Lord Howe Standard Time': 'Australia/Lord_Howe', - 'Magadan Standard Time': 'Asia/Magadan', - 'Magallanes Standard Time': 'America/Punta_Arenas', - 'Marquesas Standard Time': 'Pacific/Marquesas', - 'Mauritius Standard Time': 'Indian/Mauritius', - 'Middle East Standard Time': 'Asia/Beirut', - 'Montevideo Standard Time': 'America/Montevideo', - 'Morocco Standard Time': 'Africa/Casablanca', - 'Mountain Standard Time': 'America/Denver', - 'Mountain Standard Time (Mexico)': 'America/Chihuahua', - 'Myanmar Standard Time': 'Asia/Rangoon', - 'N. Central Asia Standard Time': 'Asia/Novosibirsk', - 'Namibia Standard Time': 'Africa/Windhoek', - 'Nepal Standard Time': 'Asia/Katmandu', - 'New Zealand Standard Time': 'Pacific/Auckland', - 'Newfoundland Standard Time': 'America/St_Johns', - 'Norfolk Standard Time': 'Pacific/Norfolk', - 'North Asia East Standard Time': 'Asia/Irkutsk', - 'North Asia Standard Time': 'Asia/Krasnoyarsk', - 'North Korea Standard Time': 'Asia/Pyongyang', - 'Omsk Standard Time': 'Asia/Omsk', - 'Pacific SA Standard Time': 'America/Santiago', - 'Pacific Standard Time': 'America/Los_Angeles', - 'Pacific Standard Time (Mexico)': 'America/Tijuana', - 'Pakistan Standard Time': 'Asia/Karachi', - 'Paraguay Standard Time': 'America/Asuncion', - 'Qyzylorda Standard Time': 'Asia/Qyzylorda', - 'Romance Standard Time': 'Europe/Paris', - 'Russia Time Zone 10': 'Asia/Srednekolymsk', - 'Russia Time Zone 11': 'Asia/Kamchatka', - 'Russia Time Zone 3': 'Europe/Samara', - 'Russian Standard Time': 'Europe/Moscow', - 'SA Eastern Standard Time': 'America/Cayenne', - 'SA Pacific Standard Time': 'America/Bogota', - 'SA Western Standard Time': 'America/La_Paz', - 'SE Asia Standard Time': 'Asia/Bangkok', - 'Saint Pierre Standard Time': 'America/Miquelon', - 'Sakhalin Standard Time': 'Asia/Sakhalin', - 'Samoa Standard Time': 'Pacific/Apia', - 'Sao Tome Standard Time': 'Africa/Sao_Tome', - 'Saratov Standard Time': 'Europe/Saratov', - 'Singapore Standard Time': 'Asia/Singapore', - 'South Africa Standard Time': 'Africa/Johannesburg', - 'Sri Lanka Standard Time': 'Asia/Colombo', - 'Sudan Standard Time': 'Africa/Khartoum', - 'Syria Standard Time': 'Asia/Damascus', - 'Taipei Standard Time': 'Asia/Taipei', - 'Tasmania Standard Time': 'Australia/Hobart', - 'Tocantins Standard Time': 'America/Araguaina', - 'Tokyo Standard Time': 'Asia/Tokyo', - 'Tomsk Standard Time': 'Asia/Tomsk', - 'Tonga Standard Time': 'Pacific/Tongatapu', - 'Transbaikal Standard Time': 'Asia/Chita', - 'Turkey Standard Time': 'Europe/Istanbul', - 'Turks And Caicos Standard Time': 'America/Grand_Turk', - 'US Eastern Standard Time': 'America/Indianapolis', - 'US Mountain Standard Time': 'America/Phoenix', - 'UTC': 'Etc/GMT', - 'UTC+12': 'Etc/GMT-12', - 'UTC+13': 'Etc/GMT-13', - 'UTC-02': 'Etc/GMT+2', - 'UTC-08': 'Etc/GMT+8', - 'UTC-09': 'Etc/GMT+9', - 'UTC-11': 'Etc/GMT+11', - 'Ulaanbaatar Standard Time': 'Asia/Ulaanbaatar', - 'Venezuela Standard Time': 'America/Caracas', - 'Vladivostok Standard Time': 'Asia/Vladivostok', - 'Volgograd Standard Time': 'Europe/Volgograd', - 'W. Australia Standard Time': 'Australia/Perth', - 'W. Central Africa Standard Time': 'Africa/Lagos', - 'W. Europe Standard Time': 'Europe/Berlin', - 'W. Mongolia Standard Time': 'Asia/Hovd', - 'West Asia Standard Time': 'Asia/Tashkent', - 'West Bank Standard Time': 'Asia/Hebron', - 'West Pacific Standard Time': 'Pacific/Port_Moresby', - 'Yakutsk Standard Time': 'Asia/Yakutsk'} - -# Old name for the win_tz variable: -tz_names = win_tz - -tz_win = {'Africa/Abidjan': 'Greenwich Standard Time', - 'Africa/Accra': 'Greenwich Standard Time', - 'Africa/Addis_Ababa': 'E. Africa Standard Time', - 'Africa/Algiers': 'W. Central Africa Standard Time', - 'Africa/Asmera': 'E. Africa Standard Time', - 'Africa/Bamako': 'Greenwich Standard Time', - 'Africa/Bangui': 'W. Central Africa Standard Time', - 'Africa/Banjul': 'Greenwich Standard Time', - 'Africa/Bissau': 'Greenwich Standard Time', - 'Africa/Blantyre': 'South Africa Standard Time', - 'Africa/Brazzaville': 'W. Central Africa Standard Time', - 'Africa/Bujumbura': 'South Africa Standard Time', - 'Africa/Cairo': 'Egypt Standard Time', - 'Africa/Casablanca': 'Morocco Standard Time', - 'Africa/Ceuta': 'Romance Standard Time', - 'Africa/Conakry': 'Greenwich Standard Time', - 'Africa/Dakar': 'Greenwich Standard Time', - 'Africa/Dar_es_Salaam': 'E. Africa Standard Time', - 'Africa/Djibouti': 'E. Africa Standard Time', - 'Africa/Douala': 'W. Central Africa Standard Time', - 'Africa/El_Aaiun': 'Morocco Standard Time', - 'Africa/Freetown': 'Greenwich Standard Time', - 'Africa/Gaborone': 'South Africa Standard Time', - 'Africa/Harare': 'South Africa Standard Time', - 'Africa/Johannesburg': 'South Africa Standard Time', - 'Africa/Juba': 'E. Africa Standard Time', - 'Africa/Kampala': 'E. Africa Standard Time', - 'Africa/Khartoum': 'Sudan Standard Time', - 'Africa/Kigali': 'South Africa Standard Time', - 'Africa/Kinshasa': 'W. Central Africa Standard Time', - 'Africa/Lagos': 'W. Central Africa Standard Time', - 'Africa/Libreville': 'W. Central Africa Standard Time', - 'Africa/Lome': 'Greenwich Standard Time', - 'Africa/Luanda': 'W. Central Africa Standard Time', - 'Africa/Lubumbashi': 'South Africa Standard Time', - 'Africa/Lusaka': 'South Africa Standard Time', - 'Africa/Malabo': 'W. Central Africa Standard Time', - 'Africa/Maputo': 'South Africa Standard Time', - 'Africa/Maseru': 'South Africa Standard Time', - 'Africa/Mbabane': 'South Africa Standard Time', - 'Africa/Mogadishu': 'E. Africa Standard Time', - 'Africa/Monrovia': 'Greenwich Standard Time', - 'Africa/Nairobi': 'E. Africa Standard Time', - 'Africa/Ndjamena': 'W. Central Africa Standard Time', - 'Africa/Niamey': 'W. Central Africa Standard Time', - 'Africa/Nouakchott': 'Greenwich Standard Time', - 'Africa/Ouagadougou': 'Greenwich Standard Time', - 'Africa/Porto-Novo': 'W. Central Africa Standard Time', - 'Africa/Sao_Tome': 'Sao Tome Standard Time', - 'Africa/Timbuktu': 'Greenwich Standard Time', - 'Africa/Tripoli': 'Libya Standard Time', - 'Africa/Tunis': 'W. Central Africa Standard Time', - 'Africa/Windhoek': 'Namibia Standard Time', - 'America/Adak': 'Aleutian Standard Time', - 'America/Anchorage': 'Alaskan Standard Time', - 'America/Anguilla': 'SA Western Standard Time', - 'America/Antigua': 'SA Western Standard Time', - 'America/Araguaina': 'Tocantins Standard Time', - 'America/Argentina/La_Rioja': 'Argentina Standard Time', - 'America/Argentina/Rio_Gallegos': 'Argentina Standard Time', - 'America/Argentina/Salta': 'Argentina Standard Time', - 'America/Argentina/San_Juan': 'Argentina Standard Time', - 'America/Argentina/San_Luis': 'Argentina Standard Time', - 'America/Argentina/Tucuman': 'Argentina Standard Time', - 'America/Argentina/Ushuaia': 'Argentina Standard Time', - 'America/Aruba': 'SA Western Standard Time', - 'America/Asuncion': 'Paraguay Standard Time', - 'America/Atka': 'Aleutian Standard Time', - 'America/Bahia': 'Bahia Standard Time', - 'America/Bahia_Banderas': 'Central Standard Time (Mexico)', - 'America/Barbados': 'SA Western Standard Time', - 'America/Belem': 'SA Eastern Standard Time', - 'America/Belize': 'Central America Standard Time', - 'America/Blanc-Sablon': 'SA Western Standard Time', - 'America/Boa_Vista': 'SA Western Standard Time', - 'America/Bogota': 'SA Pacific Standard Time', - 'America/Boise': 'Mountain Standard Time', - 'America/Buenos_Aires': 'Argentina Standard Time', - 'America/Cambridge_Bay': 'Mountain Standard Time', - 'America/Campo_Grande': 'Central Brazilian Standard Time', - 'America/Cancun': 'Eastern Standard Time (Mexico)', - 'America/Caracas': 'Venezuela Standard Time', - 'America/Catamarca': 'Argentina Standard Time', - 'America/Cayenne': 'SA Eastern Standard Time', - 'America/Cayman': 'SA Pacific Standard Time', - 'America/Chicago': 'Central Standard Time', - 'America/Chihuahua': 'Mountain Standard Time (Mexico)', - 'America/Coral_Harbour': 'SA Pacific Standard Time', - 'America/Cordoba': 'Argentina Standard Time', - 'America/Costa_Rica': 'Central America Standard Time', - 'America/Creston': 'US Mountain Standard Time', - 'America/Cuiaba': 'Central Brazilian Standard Time', - 'America/Curacao': 'SA Western Standard Time', - 'America/Danmarkshavn': 'UTC', - 'America/Dawson': 'Pacific Standard Time', - 'America/Dawson_Creek': 'US Mountain Standard Time', - 'America/Denver': 'Mountain Standard Time', - 'America/Detroit': 'Eastern Standard Time', - 'America/Dominica': 'SA Western Standard Time', - 'America/Edmonton': 'Mountain Standard Time', - 'America/Eirunepe': 'SA Pacific Standard Time', - 'America/El_Salvador': 'Central America Standard Time', - 'America/Ensenada': 'Pacific Standard Time (Mexico)', - 'America/Fort_Nelson': 'US Mountain Standard Time', - 'America/Fortaleza': 'SA Eastern Standard Time', - 'America/Glace_Bay': 'Atlantic Standard Time', - 'America/Godthab': 'Greenland Standard Time', - 'America/Goose_Bay': 'Atlantic Standard Time', - 'America/Grand_Turk': 'Turks And Caicos Standard Time', - 'America/Grenada': 'SA Western Standard Time', - 'America/Guadeloupe': 'SA Western Standard Time', - 'America/Guatemala': 'Central America Standard Time', - 'America/Guayaquil': 'SA Pacific Standard Time', - 'America/Guyana': 'SA Western Standard Time', - 'America/Halifax': 'Atlantic Standard Time', - 'America/Havana': 'Cuba Standard Time', - 'America/Hermosillo': 'US Mountain Standard Time', - 'America/Indiana/Knox': 'Central Standard Time', - 'America/Indiana/Marengo': 'US Eastern Standard Time', - 'America/Indiana/Petersburg': 'Eastern Standard Time', - 'America/Indiana/Tell_City': 'Central Standard Time', - 'America/Indiana/Vevay': 'US Eastern Standard Time', - 'America/Indiana/Vincennes': 'Eastern Standard Time', - 'America/Indiana/Winamac': 'Eastern Standard Time', - 'America/Indianapolis': 'US Eastern Standard Time', - 'America/Inuvik': 'Mountain Standard Time', - 'America/Iqaluit': 'Eastern Standard Time', - 'America/Jamaica': 'SA Pacific Standard Time', - 'America/Jujuy': 'Argentina Standard Time', - 'America/Juneau': 'Alaskan Standard Time', - 'America/Kentucky/Monticello': 'Eastern Standard Time', - 'America/Knox_IN': 'Central Standard Time', - 'America/Kralendijk': 'SA Western Standard Time', - 'America/La_Paz': 'SA Western Standard Time', - 'America/Lima': 'SA Pacific Standard Time', - 'America/Los_Angeles': 'Pacific Standard Time', - 'America/Louisville': 'Eastern Standard Time', - 'America/Lower_Princes': 'SA Western Standard Time', - 'America/Maceio': 'SA Eastern Standard Time', - 'America/Managua': 'Central America Standard Time', - 'America/Manaus': 'SA Western Standard Time', - 'America/Marigot': 'SA Western Standard Time', - 'America/Martinique': 'SA Western Standard Time', - 'America/Matamoros': 'Central Standard Time', - 'America/Mazatlan': 'Mountain Standard Time (Mexico)', - 'America/Mendoza': 'Argentina Standard Time', - 'America/Menominee': 'Central Standard Time', - 'America/Merida': 'Central Standard Time (Mexico)', - 'America/Metlakatla': 'Alaskan Standard Time', - 'America/Mexico_City': 'Central Standard Time (Mexico)', - 'America/Miquelon': 'Saint Pierre Standard Time', - 'America/Moncton': 'Atlantic Standard Time', - 'America/Monterrey': 'Central Standard Time (Mexico)', - 'America/Montevideo': 'Montevideo Standard Time', - 'America/Montreal': 'Eastern Standard Time', - 'America/Montserrat': 'SA Western Standard Time', - 'America/Nassau': 'Eastern Standard Time', - 'America/New_York': 'Eastern Standard Time', - 'America/Nipigon': 'Eastern Standard Time', - 'America/Nome': 'Alaskan Standard Time', - 'America/Noronha': 'UTC-02', - 'America/North_Dakota/Beulah': 'Central Standard Time', - 'America/North_Dakota/Center': 'Central Standard Time', - 'America/North_Dakota/New_Salem': 'Central Standard Time', - 'America/Ojinaga': 'Mountain Standard Time', - 'America/Panama': 'SA Pacific Standard Time', - 'America/Pangnirtung': 'Eastern Standard Time', - 'America/Paramaribo': 'SA Eastern Standard Time', - 'America/Phoenix': 'US Mountain Standard Time', - 'America/Port-au-Prince': 'Haiti Standard Time', - 'America/Port_of_Spain': 'SA Western Standard Time', - 'America/Porto_Acre': 'SA Pacific Standard Time', - 'America/Porto_Velho': 'SA Western Standard Time', - 'America/Puerto_Rico': 'SA Western Standard Time', - 'America/Punta_Arenas': 'Magallanes Standard Time', - 'America/Rainy_River': 'Central Standard Time', - 'America/Rankin_Inlet': 'Central Standard Time', - 'America/Recife': 'SA Eastern Standard Time', - 'America/Regina': 'Canada Central Standard Time', - 'America/Resolute': 'Central Standard Time', - 'America/Rio_Branco': 'SA Pacific Standard Time', - 'America/Santa_Isabel': 'Pacific Standard Time (Mexico)', - 'America/Santarem': 'SA Eastern Standard Time', - 'America/Santiago': 'Pacific SA Standard Time', - 'America/Santo_Domingo': 'SA Western Standard Time', - 'America/Sao_Paulo': 'E. South America Standard Time', - 'America/Scoresbysund': 'Azores Standard Time', - 'America/Shiprock': 'Mountain Standard Time', - 'America/Sitka': 'Alaskan Standard Time', - 'America/St_Barthelemy': 'SA Western Standard Time', - 'America/St_Johns': 'Newfoundland Standard Time', - 'America/St_Kitts': 'SA Western Standard Time', - 'America/St_Lucia': 'SA Western Standard Time', - 'America/St_Thomas': 'SA Western Standard Time', - 'America/St_Vincent': 'SA Western Standard Time', - 'America/Swift_Current': 'Canada Central Standard Time', - 'America/Tegucigalpa': 'Central America Standard Time', - 'America/Thule': 'Atlantic Standard Time', - 'America/Thunder_Bay': 'Eastern Standard Time', - 'America/Tijuana': 'Pacific Standard Time (Mexico)', - 'America/Toronto': 'Eastern Standard Time', - 'America/Tortola': 'SA Western Standard Time', - 'America/Vancouver': 'Pacific Standard Time', - 'America/Virgin': 'SA Western Standard Time', - 'America/Whitehorse': 'Pacific Standard Time', - 'America/Winnipeg': 'Central Standard Time', - 'America/Yakutat': 'Alaskan Standard Time', - 'America/Yellowknife': 'Mountain Standard Time', - 'Antarctica/Casey': 'Singapore Standard Time', - 'Antarctica/Davis': 'SE Asia Standard Time', - 'Antarctica/DumontDUrville': 'West Pacific Standard Time', - 'Antarctica/Macquarie': 'Central Pacific Standard Time', - 'Antarctica/Mawson': 'West Asia Standard Time', - 'Antarctica/McMurdo': 'New Zealand Standard Time', - 'Antarctica/Palmer': 'SA Eastern Standard Time', - 'Antarctica/Rothera': 'SA Eastern Standard Time', - 'Antarctica/South_Pole': 'New Zealand Standard Time', - 'Antarctica/Syowa': 'E. Africa Standard Time', - 'Antarctica/Vostok': 'Central Asia Standard Time', - 'Arctic/Longyearbyen': 'W. Europe Standard Time', - 'Asia/Aden': 'Arab Standard Time', - 'Asia/Almaty': 'Central Asia Standard Time', - 'Asia/Amman': 'Jordan Standard Time', - 'Asia/Anadyr': 'Russia Time Zone 11', - 'Asia/Aqtau': 'West Asia Standard Time', - 'Asia/Aqtobe': 'West Asia Standard Time', - 'Asia/Ashgabat': 'West Asia Standard Time', - 'Asia/Ashkhabad': 'West Asia Standard Time', - 'Asia/Atyrau': 'West Asia Standard Time', - 'Asia/Baghdad': 'Arabic Standard Time', - 'Asia/Bahrain': 'Arab Standard Time', - 'Asia/Baku': 'Azerbaijan Standard Time', - 'Asia/Bangkok': 'SE Asia Standard Time', - 'Asia/Barnaul': 'Altai Standard Time', - 'Asia/Beirut': 'Middle East Standard Time', - 'Asia/Bishkek': 'Central Asia Standard Time', - 'Asia/Brunei': 'Singapore Standard Time', - 'Asia/Calcutta': 'India Standard Time', - 'Asia/Chita': 'Transbaikal Standard Time', - 'Asia/Choibalsan': 'Ulaanbaatar Standard Time', - 'Asia/Chongqing': 'China Standard Time', - 'Asia/Chungking': 'China Standard Time', - 'Asia/Colombo': 'Sri Lanka Standard Time', - 'Asia/Dacca': 'Bangladesh Standard Time', - 'Asia/Damascus': 'Syria Standard Time', - 'Asia/Dhaka': 'Bangladesh Standard Time', - 'Asia/Dili': 'Tokyo Standard Time', - 'Asia/Dubai': 'Arabian Standard Time', - 'Asia/Dushanbe': 'West Asia Standard Time', - 'Asia/Famagusta': 'GTB Standard Time', - 'Asia/Gaza': 'West Bank Standard Time', - 'Asia/Harbin': 'China Standard Time', - 'Asia/Hebron': 'West Bank Standard Time', - 'Asia/Hong_Kong': 'China Standard Time', - 'Asia/Hovd': 'W. Mongolia Standard Time', - 'Asia/Irkutsk': 'North Asia East Standard Time', - 'Asia/Jakarta': 'SE Asia Standard Time', - 'Asia/Jayapura': 'Tokyo Standard Time', - 'Asia/Jerusalem': 'Israel Standard Time', - 'Asia/Kabul': 'Afghanistan Standard Time', - 'Asia/Kamchatka': 'Russia Time Zone 11', - 'Asia/Karachi': 'Pakistan Standard Time', - 'Asia/Kashgar': 'Central Asia Standard Time', - 'Asia/Katmandu': 'Nepal Standard Time', - 'Asia/Khandyga': 'Yakutsk Standard Time', - 'Asia/Krasnoyarsk': 'North Asia Standard Time', - 'Asia/Kuala_Lumpur': 'Singapore Standard Time', - 'Asia/Kuching': 'Singapore Standard Time', - 'Asia/Kuwait': 'Arab Standard Time', - 'Asia/Macao': 'China Standard Time', - 'Asia/Macau': 'China Standard Time', - 'Asia/Magadan': 'Magadan Standard Time', - 'Asia/Makassar': 'Singapore Standard Time', - 'Asia/Manila': 'Singapore Standard Time', - 'Asia/Muscat': 'Arabian Standard Time', - 'Asia/Nicosia': 'GTB Standard Time', - 'Asia/Novokuznetsk': 'North Asia Standard Time', - 'Asia/Novosibirsk': 'N. Central Asia Standard Time', - 'Asia/Omsk': 'Omsk Standard Time', - 'Asia/Oral': 'West Asia Standard Time', - 'Asia/Phnom_Penh': 'SE Asia Standard Time', - 'Asia/Pontianak': 'SE Asia Standard Time', - 'Asia/Pyongyang': 'North Korea Standard Time', - 'Asia/Qatar': 'Arab Standard Time', - 'Asia/Qostanay': 'Central Asia Standard Time', - 'Asia/Qyzylorda': 'Qyzylorda Standard Time', - 'Asia/Rangoon': 'Myanmar Standard Time', - 'Asia/Riyadh': 'Arab Standard Time', - 'Asia/Saigon': 'SE Asia Standard Time', - 'Asia/Sakhalin': 'Sakhalin Standard Time', - 'Asia/Samarkand': 'West Asia Standard Time', - 'Asia/Seoul': 'Korea Standard Time', - 'Asia/Shanghai': 'China Standard Time', - 'Asia/Singapore': 'Singapore Standard Time', - 'Asia/Srednekolymsk': 'Russia Time Zone 10', - 'Asia/Taipei': 'Taipei Standard Time', - 'Asia/Tashkent': 'West Asia Standard Time', - 'Asia/Tbilisi': 'Georgian Standard Time', - 'Asia/Tehran': 'Iran Standard Time', - 'Asia/Tel_Aviv': 'Israel Standard Time', - 'Asia/Thimbu': 'Bangladesh Standard Time', - 'Asia/Thimphu': 'Bangladesh Standard Time', - 'Asia/Tokyo': 'Tokyo Standard Time', - 'Asia/Tomsk': 'Tomsk Standard Time', - 'Asia/Ujung_Pandang': 'Singapore Standard Time', - 'Asia/Ulaanbaatar': 'Ulaanbaatar Standard Time', - 'Asia/Ulan_Bator': 'Ulaanbaatar Standard Time', - 'Asia/Urumqi': 'Central Asia Standard Time', - 'Asia/Ust-Nera': 'Vladivostok Standard Time', - 'Asia/Vientiane': 'SE Asia Standard Time', - 'Asia/Vladivostok': 'Vladivostok Standard Time', - 'Asia/Yakutsk': 'Yakutsk Standard Time', - 'Asia/Yekaterinburg': 'Ekaterinburg Standard Time', - 'Asia/Yerevan': 'Caucasus Standard Time', - 'Atlantic/Azores': 'Azores Standard Time', - 'Atlantic/Bermuda': 'Atlantic Standard Time', - 'Atlantic/Canary': 'GMT Standard Time', - 'Atlantic/Cape_Verde': 'Cape Verde Standard Time', - 'Atlantic/Faeroe': 'GMT Standard Time', - 'Atlantic/Jan_Mayen': 'W. Europe Standard Time', - 'Atlantic/Madeira': 'GMT Standard Time', - 'Atlantic/Reykjavik': 'Greenwich Standard Time', - 'Atlantic/South_Georgia': 'UTC-02', - 'Atlantic/St_Helena': 'Greenwich Standard Time', - 'Atlantic/Stanley': 'SA Eastern Standard Time', - 'Australia/ACT': 'AUS Eastern Standard Time', - 'Australia/Adelaide': 'Cen. Australia Standard Time', - 'Australia/Brisbane': 'E. Australia Standard Time', - 'Australia/Broken_Hill': 'Cen. Australia Standard Time', - 'Australia/Canberra': 'AUS Eastern Standard Time', - 'Australia/Currie': 'Tasmania Standard Time', - 'Australia/Darwin': 'AUS Central Standard Time', - 'Australia/Eucla': 'Aus Central W. Standard Time', - 'Australia/Hobart': 'Tasmania Standard Time', - 'Australia/LHI': 'Lord Howe Standard Time', - 'Australia/Lindeman': 'E. Australia Standard Time', - 'Australia/Lord_Howe': 'Lord Howe Standard Time', - 'Australia/Melbourne': 'AUS Eastern Standard Time', - 'Australia/NSW': 'AUS Eastern Standard Time', - 'Australia/North': 'AUS Central Standard Time', - 'Australia/Perth': 'W. Australia Standard Time', - 'Australia/Queensland': 'E. Australia Standard Time', - 'Australia/South': 'Cen. Australia Standard Time', - 'Australia/Sydney': 'AUS Eastern Standard Time', - 'Australia/Tasmania': 'Tasmania Standard Time', - 'Australia/Victoria': 'AUS Eastern Standard Time', - 'Australia/West': 'W. Australia Standard Time', - 'Australia/Yancowinna': 'Cen. Australia Standard Time', - 'Brazil/Acre': 'SA Pacific Standard Time', - 'Brazil/DeNoronha': 'UTC-02', - 'Brazil/East': 'E. South America Standard Time', - 'Brazil/West': 'SA Western Standard Time', - 'CST6CDT': 'Central Standard Time', - 'Canada/Atlantic': 'Atlantic Standard Time', - 'Canada/Central': 'Central Standard Time', - 'Canada/Eastern': 'Eastern Standard Time', - 'Canada/Mountain': 'Mountain Standard Time', - 'Canada/Newfoundland': 'Newfoundland Standard Time', - 'Canada/Pacific': 'Pacific Standard Time', - 'Canada/Saskatchewan': 'Canada Central Standard Time', - 'Canada/Yukon': 'Pacific Standard Time', - 'Chile/Continental': 'Pacific SA Standard Time', - 'Chile/EasterIsland': 'Easter Island Standard Time', - 'Cuba': 'Cuba Standard Time', - 'EST5EDT': 'Eastern Standard Time', - 'Egypt': 'Egypt Standard Time', - 'Eire': 'GMT Standard Time', - 'Etc/GMT': 'UTC', - 'Etc/GMT+1': 'Cape Verde Standard Time', - 'Etc/GMT+10': 'Hawaiian Standard Time', - 'Etc/GMT+11': 'UTC-11', - 'Etc/GMT+12': 'Dateline Standard Time', - 'Etc/GMT+2': 'UTC-02', - 'Etc/GMT+3': 'SA Eastern Standard Time', - 'Etc/GMT+4': 'SA Western Standard Time', - 'Etc/GMT+5': 'SA Pacific Standard Time', - 'Etc/GMT+6': 'Central America Standard Time', - 'Etc/GMT+7': 'US Mountain Standard Time', - 'Etc/GMT+8': 'UTC-08', - 'Etc/GMT+9': 'UTC-09', - 'Etc/GMT-1': 'W. Central Africa Standard Time', - 'Etc/GMT-10': 'West Pacific Standard Time', - 'Etc/GMT-11': 'Central Pacific Standard Time', - 'Etc/GMT-12': 'UTC+12', - 'Etc/GMT-13': 'UTC+13', - 'Etc/GMT-14': 'Line Islands Standard Time', - 'Etc/GMT-2': 'South Africa Standard Time', - 'Etc/GMT-3': 'E. Africa Standard Time', - 'Etc/GMT-4': 'Arabian Standard Time', - 'Etc/GMT-5': 'West Asia Standard Time', - 'Etc/GMT-6': 'Central Asia Standard Time', - 'Etc/GMT-7': 'SE Asia Standard Time', - 'Etc/GMT-8': 'Singapore Standard Time', - 'Etc/GMT-9': 'Tokyo Standard Time', - 'Etc/UCT': 'UTC', - 'Etc/UTC': 'UTC', - 'Europe/Amsterdam': 'W. Europe Standard Time', - 'Europe/Andorra': 'W. Europe Standard Time', - 'Europe/Astrakhan': 'Astrakhan Standard Time', - 'Europe/Athens': 'GTB Standard Time', - 'Europe/Belfast': 'GMT Standard Time', - 'Europe/Belgrade': 'Central Europe Standard Time', - 'Europe/Berlin': 'W. Europe Standard Time', - 'Europe/Bratislava': 'Central Europe Standard Time', - 'Europe/Brussels': 'Romance Standard Time', - 'Europe/Bucharest': 'GTB Standard Time', - 'Europe/Budapest': 'Central Europe Standard Time', - 'Europe/Busingen': 'W. Europe Standard Time', - 'Europe/Chisinau': 'E. Europe Standard Time', - 'Europe/Copenhagen': 'Romance Standard Time', - 'Europe/Dublin': 'GMT Standard Time', - 'Europe/Gibraltar': 'W. Europe Standard Time', - 'Europe/Guernsey': 'GMT Standard Time', - 'Europe/Helsinki': 'FLE Standard Time', - 'Europe/Isle_of_Man': 'GMT Standard Time', - 'Europe/Istanbul': 'Turkey Standard Time', - 'Europe/Jersey': 'GMT Standard Time', - 'Europe/Kaliningrad': 'Kaliningrad Standard Time', - 'Europe/Kiev': 'FLE Standard Time', - 'Europe/Kirov': 'Russian Standard Time', - 'Europe/Lisbon': 'GMT Standard Time', - 'Europe/Ljubljana': 'Central Europe Standard Time', - 'Europe/London': 'GMT Standard Time', - 'Europe/Luxembourg': 'W. Europe Standard Time', - 'Europe/Madrid': 'Romance Standard Time', - 'Europe/Malta': 'W. Europe Standard Time', - 'Europe/Mariehamn': 'FLE Standard Time', - 'Europe/Minsk': 'Belarus Standard Time', - 'Europe/Monaco': 'W. Europe Standard Time', - 'Europe/Moscow': 'Russian Standard Time', - 'Europe/Oslo': 'W. Europe Standard Time', - 'Europe/Paris': 'Romance Standard Time', - 'Europe/Podgorica': 'Central Europe Standard Time', - 'Europe/Prague': 'Central Europe Standard Time', - 'Europe/Riga': 'FLE Standard Time', - 'Europe/Rome': 'W. Europe Standard Time', - 'Europe/Samara': 'Russia Time Zone 3', - 'Europe/San_Marino': 'W. Europe Standard Time', - 'Europe/Sarajevo': 'Central European Standard Time', - 'Europe/Saratov': 'Saratov Standard Time', - 'Europe/Simferopol': 'Russian Standard Time', - 'Europe/Skopje': 'Central European Standard Time', - 'Europe/Sofia': 'FLE Standard Time', - 'Europe/Stockholm': 'W. Europe Standard Time', - 'Europe/Tallinn': 'FLE Standard Time', - 'Europe/Tirane': 'Central Europe Standard Time', - 'Europe/Tiraspol': 'E. Europe Standard Time', - 'Europe/Ulyanovsk': 'Astrakhan Standard Time', - 'Europe/Uzhgorod': 'FLE Standard Time', - 'Europe/Vaduz': 'W. Europe Standard Time', - 'Europe/Vatican': 'W. Europe Standard Time', - 'Europe/Vienna': 'W. Europe Standard Time', - 'Europe/Vilnius': 'FLE Standard Time', - 'Europe/Volgograd': 'Volgograd Standard Time', - 'Europe/Warsaw': 'Central European Standard Time', - 'Europe/Zagreb': 'Central European Standard Time', - 'Europe/Zaporozhye': 'FLE Standard Time', - 'Europe/Zurich': 'W. Europe Standard Time', - 'GB': 'GMT Standard Time', - 'GB-Eire': 'GMT Standard Time', - 'GMT+0': 'UTC', - 'GMT-0': 'UTC', - 'GMT0': 'UTC', - 'Greenwich': 'UTC', - 'Hongkong': 'China Standard Time', - 'Iceland': 'Greenwich Standard Time', - 'Indian/Antananarivo': 'E. Africa Standard Time', - 'Indian/Chagos': 'Central Asia Standard Time', - 'Indian/Christmas': 'SE Asia Standard Time', - 'Indian/Cocos': 'Myanmar Standard Time', - 'Indian/Comoro': 'E. Africa Standard Time', - 'Indian/Kerguelen': 'West Asia Standard Time', - 'Indian/Mahe': 'Mauritius Standard Time', - 'Indian/Maldives': 'West Asia Standard Time', - 'Indian/Mauritius': 'Mauritius Standard Time', - 'Indian/Mayotte': 'E. Africa Standard Time', - 'Indian/Reunion': 'Mauritius Standard Time', - 'Iran': 'Iran Standard Time', - 'Israel': 'Israel Standard Time', - 'Jamaica': 'SA Pacific Standard Time', - 'Japan': 'Tokyo Standard Time', - 'Kwajalein': 'UTC+12', - 'Libya': 'Libya Standard Time', - 'MST7MDT': 'Mountain Standard Time', - 'Mexico/BajaNorte': 'Pacific Standard Time (Mexico)', - 'Mexico/BajaSur': 'Mountain Standard Time (Mexico)', - 'Mexico/General': 'Central Standard Time (Mexico)', - 'NZ': 'New Zealand Standard Time', - 'NZ-CHAT': 'Chatham Islands Standard Time', - 'Navajo': 'Mountain Standard Time', - 'PRC': 'China Standard Time', - 'PST8PDT': 'Pacific Standard Time', - 'Pacific/Apia': 'Samoa Standard Time', - 'Pacific/Auckland': 'New Zealand Standard Time', - 'Pacific/Bougainville': 'Bougainville Standard Time', - 'Pacific/Chatham': 'Chatham Islands Standard Time', - 'Pacific/Easter': 'Easter Island Standard Time', - 'Pacific/Efate': 'Central Pacific Standard Time', - 'Pacific/Enderbury': 'UTC+13', - 'Pacific/Fakaofo': 'UTC+13', - 'Pacific/Fiji': 'Fiji Standard Time', - 'Pacific/Funafuti': 'UTC+12', - 'Pacific/Galapagos': 'Central America Standard Time', - 'Pacific/Gambier': 'UTC-09', - 'Pacific/Guadalcanal': 'Central Pacific Standard Time', - 'Pacific/Guam': 'West Pacific Standard Time', - 'Pacific/Honolulu': 'Hawaiian Standard Time', - 'Pacific/Johnston': 'Hawaiian Standard Time', - 'Pacific/Kiritimati': 'Line Islands Standard Time', - 'Pacific/Kosrae': 'Central Pacific Standard Time', - 'Pacific/Kwajalein': 'UTC+12', - 'Pacific/Majuro': 'UTC+12', - 'Pacific/Marquesas': 'Marquesas Standard Time', - 'Pacific/Midway': 'UTC-11', - 'Pacific/Nauru': 'UTC+12', - 'Pacific/Niue': 'UTC-11', - 'Pacific/Norfolk': 'Norfolk Standard Time', - 'Pacific/Noumea': 'Central Pacific Standard Time', - 'Pacific/Pago_Pago': 'UTC-11', - 'Pacific/Palau': 'Tokyo Standard Time', - 'Pacific/Pitcairn': 'UTC-08', - 'Pacific/Ponape': 'Central Pacific Standard Time', - 'Pacific/Port_Moresby': 'West Pacific Standard Time', - 'Pacific/Rarotonga': 'Hawaiian Standard Time', - 'Pacific/Saipan': 'West Pacific Standard Time', - 'Pacific/Samoa': 'UTC-11', - 'Pacific/Tahiti': 'Hawaiian Standard Time', - 'Pacific/Tarawa': 'UTC+12', - 'Pacific/Tongatapu': 'Tonga Standard Time', - 'Pacific/Truk': 'West Pacific Standard Time', - 'Pacific/Wake': 'UTC+12', - 'Pacific/Wallis': 'UTC+12', - 'Poland': 'Central European Standard Time', - 'Portugal': 'GMT Standard Time', - 'ROC': 'Taipei Standard Time', - 'ROK': 'Korea Standard Time', - 'Singapore': 'Singapore Standard Time', - 'Turkey': 'Turkey Standard Time', - 'UCT': 'UTC', - 'US/Alaska': 'Alaskan Standard Time', - 'US/Aleutian': 'Aleutian Standard Time', - 'US/Arizona': 'US Mountain Standard Time', - 'US/Central': 'Central Standard Time', - 'US/Eastern': 'Eastern Standard Time', - 'US/Hawaii': 'Hawaiian Standard Time', - 'US/Indiana-Starke': 'Central Standard Time', - 'US/Michigan': 'Eastern Standard Time', - 'US/Mountain': 'Mountain Standard Time', - 'US/Pacific': 'Pacific Standard Time', - 'US/Samoa': 'UTC-11', - 'UTC': 'UTC', - 'Universal': 'UTC', - 'W-SU': 'Russian Standard Time', - 'Zulu': 'UTC'} diff --git a/contrib/python/tzlocal/py2/ya.make b/contrib/python/tzlocal/py2/ya.make deleted file mode 100644 index bb647f9e4b1..00000000000 --- a/contrib/python/tzlocal/py2/ya.make +++ /dev/null @@ -1,34 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY2_LIBRARY() - -VERSION(2.1) - -LICENSE(MIT) - -PEERDIR( - contrib/python/pytz -) - -NO_LINT() - -NO_CHECK_IMPORTS( - tzlocal.win32 -) - -PY_SRCS( - TOP_LEVEL - tzlocal/__init__.py - tzlocal/unix.py - tzlocal/utils.py - tzlocal/win32.py - tzlocal/windows_tz.py -) - -RESOURCE_FILES( - PREFIX contrib/python/tzlocal/py2/ - .dist-info/METADATA - .dist-info/top_level.txt -) - -END() diff --git a/contrib/python/tzlocal/py3/.dist-info/METADATA b/contrib/python/tzlocal/py3/.dist-info/METADATA deleted file mode 100644 index c41851c9f81..00000000000 --- a/contrib/python/tzlocal/py3/.dist-info/METADATA +++ /dev/null @@ -1,248 +0,0 @@ -Metadata-Version: 2.1 -Name: tzlocal -Version: 5.2 -Summary: tzinfo object for the local timezone -Author-email: Lennart Regebro <[email protected]> -License: MIT -Project-URL: Source code, https://github.com/regebro/tzlocal -Project-URL: Changelog, https://github.com/regebro/tzlocal/blob/master/CHANGES.txt -Project-URL: Issue tracker, https://github.com/regebro/tzlocal/issues -Keywords: timezone -Classifier: Development Status :: 5 - Production/Stable -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: Microsoft :: Windows -Classifier: Operating System :: Unix -Classifier: Operating System :: MacOS :: MacOS X -Classifier: Typing :: Typed -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Requires-Python: >=3.8 -Description-Content-Type: text/x-rst -License-File: LICENSE.txt -Requires-Dist: tzdata ; platform_system == "Windows" -Requires-Dist: backports.zoneinfo ; python_version < "3.9" -Provides-Extra: devenv -Requires-Dist: pytest (>=4.3) ; extra == 'devenv' -Requires-Dist: pytest-mock (>=3.3) ; extra == 'devenv' -Requires-Dist: pytest-cov ; extra == 'devenv' -Requires-Dist: check-manifest ; extra == 'devenv' -Requires-Dist: zest.releaser ; extra == 'devenv' - -tzlocal -======= - -API CHANGE! ------------ - -With version 3.0 of tzlocal, tzlocal no longer returned `pytz` objects, but -`zoneinfo` objects, which has a different API. Since 4.0, it now restored -partial compatibility for `pytz` users through Paul Ganssle's -`pytz_deprecation_shim`. - -tzlocal 4.0 also adds an official function `get_localzone_name()` to get only -the timezone name, instead of a timezone object. On unix, it can raise an -error if you don't have a timezone name configured, where `get_localzone()` -will succeed, so only use that if you need the timezone name. - -4.0 also adds way more information on what is going wrong in your -configuration when the configuration files are unclear or contradictory. - -Version 5.0 removes the `pytz_deprecation_shim`, and now only returns -`zoneinfo` objects, like verion 3.0 did. If you need `pytz` objects, you have -to stay on version 4.0. If there are bugs in version 4.0, I will release -updates, but there will be no further functional changes on the 4.x branch. - - -Info ----- - -This Python module returns the `IANA time zone name -<https://www.iana.org/time-zones>`_ for your local time zone or a ``tzinfo`` -object with the local timezone information, under Unix and Windows. - -It requires Python 3.8 or later, and will use the ``backports.tzinfo`` -package, for Python 3.8. - -This module attempts to fix a glaring hole in the ``pytz`` and ``zoneinfo`` -modules, that there is no way to get the local timezone information, unless -you know the zoneinfo name, and under several Linux distros that's hard or -impossible to figure out. - -With ``tzlocal`` you only need to call ``get_localzone()`` and you will get a -``tzinfo`` object with the local time zone info. On some Unices you will -still not get to know what the timezone name is, but you don't need that when -you have the tzinfo file. However, if the timezone name is readily available -it will be used. - -What it's not for ------------------ - -It's not for converting the current time between UTC and your local time. There are -other, simpler ways of doing this. This is ig you need to know things like the name -of the time zone, or if you need to be able to convert between your time zone and -another time zone for times that are in the future or in the past. - -For current time conversions to and from UTC, look in the Python ``time`` module. - - -Supported systems ------------------ - -These are the systems that are in theory supported: - - * Windows 2000 and later - - * Any unix-like system with a ``/etc/localtime`` or ``/usr/local/etc/localtime`` - -If you have one of the above systems and it does not work, it's a bug. -Please report it. - -Please note that if you are getting a time zone called ``local``, this is not -a bug, it's actually the main feature of ``tzlocal``, that even if your -system does NOT have a configuration file with the zoneinfo name of your time -zone, it will still work. - -You can also use ``tzlocal`` to get the name of your local timezone, but only -if your system is configured to make that possible. ``tzlocal`` looks for the -timezone name in ``/etc/timezone``, ``/var/db/zoneinfo``, -``/etc/sysconfig/clock`` and ``/etc/conf.d/clock``. If your -``/etc/localtime`` is a symlink it can also extract the name from that -symlink. - -If you need the name of your local time zone, then please make sure your -system is properly configured to allow that. - -If your unix system doesn't have a timezone configured, tzlocal will default -to UTC. - -Notes on Docker ---------------- - -It turns out that Docker images frequently have broken timezone setups. -This usually resuts in a warning that the configuration is wrong, or that -the timezone offset doesn't match the found timezone. - -The easiest way to fix that is to set a TZ variable in your docker setup -to whatever timezone you want, which is usually the timezone your host -computer has. - -Usage ------ - -Load the local timezone: - - >>> from tzlocal import get_localzone - >>> tz = get_localzone() - >>> tz - zoneinfo.ZoneInfo(key='Europe/Warsaw') - -Create a local datetime: - - >>> from datetime import datetime - >>> dt = datetime(2015, 4, 10, 7, 22, tzinfo=tz) - >>> dt - datetime.datetime(2015, 4, 10, 7, 22, tzinfo=zoneinfo.ZoneInfo(key='Europe/Warsaw')) - -Lookup another timezone with ``zoneinfo`` (``backports.zoneinfo`` on Python 3.8 or earlier): - - >>> from zoneinfo import ZoneInfo - >>> eastern = ZoneInfo('US/Eastern') - -Convert the datetime: - - >>> dt.astimezone(eastern) - datetime.datetime(2015, 4, 10, 1, 22, tzinfo=zoneinfo.ZoneInfo(key='US/Eastern')) - -If you just want the name of the local timezone, use `get_localzone_name()`: - - >>> from tzlocal import get_localzone_name - >>> get_localzone_name() - "Europe/Warsaw" - -Please note that under Unix, `get_localzone_name()` may fail if there is no zone -configured, where `get_localzone()` would generally succeed. - -Troubleshooting ---------------- - -If you don't get the result you expect, try running it with debugging turned on. -Start a python interpreter that has tzlocal installed, and run the following code:: - - import logging - logging.basicConfig(level="DEBUG") - import tzlocal - tzlocal.get_localzone() - -The output should look something like this, and this will tell you what -configurations were found:: - - DEBUG:root:/etc/timezone found, contents: - Europe/Warsaw - - DEBUG:root:/etc/localtime found - DEBUG:root:2 found: - {'/etc/timezone': 'Europe/Warsaw', '/etc/localtime is a symlink to': 'Europe/Warsaw'} - zoneinfo.ZoneInfo(key='Europe/Warsaw') - - -Development ------------ - -For ease of development, there is a Makefile that will help you with basic tasks, -like creating a development environment with all the necessary tools (although -you need a supported Python version installed first):: - - $ make devenv - -To run tests:: - - $ make test - -Check the syntax:: - - $ make check - - -Maintainer ----------- - -* Lennart Regebro, [email protected] - -Contributors ------------- - -* Marc Van Olmen -* Benjamen Meyer -* Manuel Ebert -* Xiaokun Zhu -* Cameris -* Edward Betts -* McK KIM -* Cris Ewing -* Ayala Shachar -* Lev Maximov -* Jakub Wilk -* John Quarles -* Preston Landers -* Victor Torres -* Jean Jordaan -* Zackary Welch -* Mickaël Schoentgen -* Gabriel Corona -* Alex Grönholm -* Julin S -* Miroslav Šedivý -* revansSZ -* Sam Treweek -* Peter Di Pasquale -* Rongrong - -(Sorry if I forgot someone) - -License -------- - -* MIT https://opensource.org/licenses/MIT diff --git a/contrib/python/tzlocal/py3/.dist-info/top_level.txt b/contrib/python/tzlocal/py3/.dist-info/top_level.txt deleted file mode 100644 index cd5e9b12a4b..00000000000 --- a/contrib/python/tzlocal/py3/.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -tzlocal diff --git a/contrib/python/tzlocal/py3/LICENSE.txt b/contrib/python/tzlocal/py3/LICENSE.txt deleted file mode 100644 index 9be1d2fe595..00000000000 --- a/contrib/python/tzlocal/py3/LICENSE.txt +++ /dev/null @@ -1,19 +0,0 @@ -Copyright 2011-2017 Lennart Regebro - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/contrib/python/tzlocal/py3/README.rst b/contrib/python/tzlocal/py3/README.rst deleted file mode 100644 index 3dd27d68c90..00000000000 --- a/contrib/python/tzlocal/py3/README.rst +++ /dev/null @@ -1,215 +0,0 @@ -tzlocal -======= - -API CHANGE! ------------ - -With version 3.0 of tzlocal, tzlocal no longer returned `pytz` objects, but -`zoneinfo` objects, which has a different API. Since 4.0, it now restored -partial compatibility for `pytz` users through Paul Ganssle's -`pytz_deprecation_shim`. - -tzlocal 4.0 also adds an official function `get_localzone_name()` to get only -the timezone name, instead of a timezone object. On unix, it can raise an -error if you don't have a timezone name configured, where `get_localzone()` -will succeed, so only use that if you need the timezone name. - -4.0 also adds way more information on what is going wrong in your -configuration when the configuration files are unclear or contradictory. - -Version 5.0 removes the `pytz_deprecation_shim`, and now only returns -`zoneinfo` objects, like verion 3.0 did. If you need `pytz` objects, you have -to stay on version 4.0. If there are bugs in version 4.0, I will release -updates, but there will be no further functional changes on the 4.x branch. - - -Info ----- - -This Python module returns the `IANA time zone name -<https://www.iana.org/time-zones>`_ for your local time zone or a ``tzinfo`` -object with the local timezone information, under Unix and Windows. - -It requires Python 3.8 or later, and will use the ``backports.tzinfo`` -package, for Python 3.8. - -This module attempts to fix a glaring hole in the ``pytz`` and ``zoneinfo`` -modules, that there is no way to get the local timezone information, unless -you know the zoneinfo name, and under several Linux distros that's hard or -impossible to figure out. - -With ``tzlocal`` you only need to call ``get_localzone()`` and you will get a -``tzinfo`` object with the local time zone info. On some Unices you will -still not get to know what the timezone name is, but you don't need that when -you have the tzinfo file. However, if the timezone name is readily available -it will be used. - -What it's not for ------------------ - -It's not for converting the current time between UTC and your local time. There are -other, simpler ways of doing this. This is ig you need to know things like the name -of the time zone, or if you need to be able to convert between your time zone and -another time zone for times that are in the future or in the past. - -For current time conversions to and from UTC, look in the Python ``time`` module. - - -Supported systems ------------------ - -These are the systems that are in theory supported: - - * Windows 2000 and later - - * Any unix-like system with a ``/etc/localtime`` or ``/usr/local/etc/localtime`` - -If you have one of the above systems and it does not work, it's a bug. -Please report it. - -Please note that if you are getting a time zone called ``local``, this is not -a bug, it's actually the main feature of ``tzlocal``, that even if your -system does NOT have a configuration file with the zoneinfo name of your time -zone, it will still work. - -You can also use ``tzlocal`` to get the name of your local timezone, but only -if your system is configured to make that possible. ``tzlocal`` looks for the -timezone name in ``/etc/timezone``, ``/var/db/zoneinfo``, -``/etc/sysconfig/clock`` and ``/etc/conf.d/clock``. If your -``/etc/localtime`` is a symlink it can also extract the name from that -symlink. - -If you need the name of your local time zone, then please make sure your -system is properly configured to allow that. - -If your unix system doesn't have a timezone configured, tzlocal will default -to UTC. - -Notes on Docker ---------------- - -It turns out that Docker images frequently have broken timezone setups. -This usually resuts in a warning that the configuration is wrong, or that -the timezone offset doesn't match the found timezone. - -The easiest way to fix that is to set a TZ variable in your docker setup -to whatever timezone you want, which is usually the timezone your host -computer has. - -Usage ------ - -Load the local timezone: - - >>> from tzlocal import get_localzone - >>> tz = get_localzone() - >>> tz - zoneinfo.ZoneInfo(key='Europe/Warsaw') - -Create a local datetime: - - >>> from datetime import datetime - >>> dt = datetime(2015, 4, 10, 7, 22, tzinfo=tz) - >>> dt - datetime.datetime(2015, 4, 10, 7, 22, tzinfo=zoneinfo.ZoneInfo(key='Europe/Warsaw')) - -Lookup another timezone with ``zoneinfo`` (``backports.zoneinfo`` on Python 3.8 or earlier): - - >>> from zoneinfo import ZoneInfo - >>> eastern = ZoneInfo('US/Eastern') - -Convert the datetime: - - >>> dt.astimezone(eastern) - datetime.datetime(2015, 4, 10, 1, 22, tzinfo=zoneinfo.ZoneInfo(key='US/Eastern')) - -If you just want the name of the local timezone, use `get_localzone_name()`: - - >>> from tzlocal import get_localzone_name - >>> get_localzone_name() - "Europe/Warsaw" - -Please note that under Unix, `get_localzone_name()` may fail if there is no zone -configured, where `get_localzone()` would generally succeed. - -Troubleshooting ---------------- - -If you don't get the result you expect, try running it with debugging turned on. -Start a python interpreter that has tzlocal installed, and run the following code:: - - import logging - logging.basicConfig(level="DEBUG") - import tzlocal - tzlocal.get_localzone() - -The output should look something like this, and this will tell you what -configurations were found:: - - DEBUG:root:/etc/timezone found, contents: - Europe/Warsaw - - DEBUG:root:/etc/localtime found - DEBUG:root:2 found: - {'/etc/timezone': 'Europe/Warsaw', '/etc/localtime is a symlink to': 'Europe/Warsaw'} - zoneinfo.ZoneInfo(key='Europe/Warsaw') - - -Development ------------ - -For ease of development, there is a Makefile that will help you with basic tasks, -like creating a development environment with all the necessary tools (although -you need a supported Python version installed first):: - - $ make devenv - -To run tests:: - - $ make test - -Check the syntax:: - - $ make check - - -Maintainer ----------- - -* Lennart Regebro, [email protected] - -Contributors ------------- - -* Marc Van Olmen -* Benjamen Meyer -* Manuel Ebert -* Xiaokun Zhu -* Cameris -* Edward Betts -* McK KIM -* Cris Ewing -* Ayala Shachar -* Lev Maximov -* Jakub Wilk -* John Quarles -* Preston Landers -* Victor Torres -* Jean Jordaan -* Zackary Welch -* Mickaël Schoentgen -* Gabriel Corona -* Alex Grönholm -* Julin S -* Miroslav Šedivý -* revansSZ -* Sam Treweek -* Peter Di Pasquale -* Rongrong - -(Sorry if I forgot someone) - -License -------- - -* MIT https://opensource.org/licenses/MIT diff --git a/contrib/python/tzlocal/py3/tzlocal/__init__.py b/contrib/python/tzlocal/py3/tzlocal/__init__.py deleted file mode 100644 index 8296a15dc46..00000000000 --- a/contrib/python/tzlocal/py3/tzlocal/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -import sys - -if sys.platform == "win32": - from tzlocal.win32 import ( - get_localzone, - get_localzone_name, - reload_localzone, - ) -else: - from tzlocal.unix import get_localzone, get_localzone_name, reload_localzone - -from tzlocal.utils import assert_tz_offset - -__all__ = [ - "get_localzone", - "get_localzone_name", - "reload_localzone", - "assert_tz_offset", -] diff --git a/contrib/python/tzlocal/py3/tzlocal/unix.py b/contrib/python/tzlocal/py3/tzlocal/unix.py deleted file mode 100644 index 117c18d373b..00000000000 --- a/contrib/python/tzlocal/py3/tzlocal/unix.py +++ /dev/null @@ -1,231 +0,0 @@ -import logging -import os -import re -import sys -import warnings -from datetime import timezone - -from tzlocal import utils - -if sys.version_info >= (3, 9): - import zoneinfo # pragma: no cover -else: - from backports import zoneinfo # pragma: no cover - -_cache_tz = None -_cache_tz_name = None - -log = logging.getLogger("tzlocal") - - -def _get_localzone_name(_root="/"): - """Tries to find the local timezone configuration. - - This method finds the timezone name, if it can, or it returns None. - - The parameter _root makes the function look for files like /etc/localtime - beneath the _root directory. This is primarily used by the tests. - In normal usage you call the function without parameters.""" - - # First try the ENV setting. - tzenv = utils._tz_name_from_env() - if tzenv: - return tzenv - - # Are we under Termux on Android? - if os.path.exists(os.path.join(_root, "system/bin/getprop")): - log.debug("This looks like Termux") - - import subprocess - - try: - androidtz = ( - subprocess.check_output(["getprop", "persist.sys.timezone"]) - .strip() - .decode() - ) - return androidtz - except (OSError, subprocess.CalledProcessError): - # proot environment or failed to getprop - log.debug("It's not termux?") - pass - - # Now look for distribution specific configuration files - # that contain the timezone name. - - # Stick all of them in a dict, to compare later. - found_configs = {} - - for configfile in ("etc/timezone", "var/db/zoneinfo"): - tzpath = os.path.join(_root, configfile) - try: - with open(tzpath) as tzfile: - data = tzfile.read() - log.debug(f"{tzpath} found, contents:\n {data}") - - etctz = data.strip("/ \t\r\n") - if not etctz: - # Empty file, skip - continue - for etctz in etctz.splitlines(): - # Get rid of host definitions and comments: - if " " in etctz: - etctz, dummy = etctz.split(" ", 1) - if "#" in etctz: - etctz, dummy = etctz.split("#", 1) - if not etctz: - continue - - found_configs[tzpath] = etctz.replace(" ", "_") - - except (OSError, UnicodeDecodeError): - # File doesn't exist or is a directory, or it's a binary file. - continue - - # CentOS has a ZONE setting in /etc/sysconfig/clock, - # OpenSUSE has a TIMEZONE setting in /etc/sysconfig/clock and - # Gentoo has a TIMEZONE setting in /etc/conf.d/clock - # We look through these files for a timezone: - - zone_re = re.compile(r"\s*ZONE\s*=\s*\"") - timezone_re = re.compile(r"\s*TIMEZONE\s*=\s*\"") - end_re = re.compile('"') - - for filename in ("etc/sysconfig/clock", "etc/conf.d/clock"): - tzpath = os.path.join(_root, filename) - try: - with open(tzpath, "rt") as tzfile: - data = tzfile.readlines() - log.debug(f"{tzpath} found, contents:\n {data}") - - for line in data: - # Look for the ZONE= setting. - match = zone_re.match(line) - if match is None: - # No ZONE= setting. Look for the TIMEZONE= setting. - match = timezone_re.match(line) - if match is not None: - # Some setting existed - line = line[match.end() :] - etctz = line[: end_re.search(line).start()] - - # We found a timezone - found_configs[tzpath] = etctz.replace(" ", "_") - - except (OSError, UnicodeDecodeError): - # UnicodeDecode handles when clock is symlink to /etc/localtime - continue - - # systemd distributions use symlinks that include the zone name, - # see manpage of localtime(5) and timedatectl(1) - tzpath = os.path.join(_root, "etc/localtime") - if os.path.exists(tzpath) and os.path.islink(tzpath): - log.debug(f"{tzpath} found") - etctz = os.path.realpath(tzpath) - start = etctz.find("/") + 1 - while start != 0: - etctz = etctz[start:] - try: - zoneinfo.ZoneInfo(etctz) - tzinfo = f"{tzpath} is a symlink to" - found_configs[tzinfo] = etctz.replace(" ", "_") - # Only need first valid relative path in simlink. - break - except zoneinfo.ZoneInfoNotFoundError: - pass - start = etctz.find("/") + 1 - - if len(found_configs) > 0: - log.debug(f"{len(found_configs)} found:\n {found_configs}") - # We found some explicit config of some sort! - if len(found_configs) > 1: - # Uh-oh, multiple configs. See if they match: - unique_tzs = set() - zoneinfopath = os.path.join(_root, "usr", "share", "zoneinfo") - directory_depth = len(zoneinfopath.split(os.path.sep)) - - for tzname in found_configs.values(): - # Look them up in /usr/share/zoneinfo, and find what they - # really point to: - path = os.path.realpath(os.path.join(zoneinfopath, *tzname.split("/"))) - real_zone_name = "/".join(path.split(os.path.sep)[directory_depth:]) - unique_tzs.add(real_zone_name) - - if len(unique_tzs) != 1: - message = "Multiple conflicting time zone configurations found:\n" - for key, value in found_configs.items(): - message += f"{key}: {value}\n" - message += "Fix the configuration, or set the time zone in a TZ environment variable.\n" - raise zoneinfo.ZoneInfoNotFoundError(message) - - # We found exactly one config! Use it. - return list(found_configs.values())[0] - - -def _get_localzone(_root="/"): - """Creates a timezone object from the timezone name. - - If there is no timezone config, it will try to create a file from the - localtime timezone, and if there isn't one, it will default to UTC. - - The parameter _root makes the function look for files like /etc/localtime - beneath the _root directory. This is primarily used by the tests. - In normal usage you call the function without parameters.""" - - # First try the ENV setting. - tzenv = utils._tz_from_env() - if tzenv: - return tzenv - - tzname = _get_localzone_name(_root) - if tzname is None: - # No explicit setting existed. Use localtime - log.debug("No explicit setting existed. Use localtime") - for filename in ("etc/localtime", "usr/local/etc/localtime"): - tzpath = os.path.join(_root, filename) - - if not os.path.exists(tzpath): - continue - with open(tzpath, "rb") as tzfile: - tz = zoneinfo.ZoneInfo.from_file(tzfile, key="local") - break - else: - warnings.warn("Can not find any timezone configuration, defaulting to UTC.") - tz = timezone.utc - else: - tz = zoneinfo.ZoneInfo(tzname) - - if _root == "/": - # We are using a file in etc to name the timezone. - # Verify that the timezone specified there is actually used: - utils.assert_tz_offset(tz, error=False) - return tz - - -def get_localzone_name() -> str: - """Get the computers configured local timezone name, if any.""" - global _cache_tz_name - if _cache_tz_name is None: - _cache_tz_name = _get_localzone_name() - - return _cache_tz_name - - -def get_localzone() -> zoneinfo.ZoneInfo: - """Get the computers configured local timezone, if any.""" - - global _cache_tz - if _cache_tz is None: - _cache_tz = _get_localzone() - - return _cache_tz - - -def reload_localzone() -> zoneinfo.ZoneInfo: - """Reload the cached localzone. You need to call this if the timezone has changed.""" - global _cache_tz_name - global _cache_tz - _cache_tz_name = _get_localzone_name() - _cache_tz = _get_localzone() - - return _cache_tz diff --git a/contrib/python/tzlocal/py3/tzlocal/utils.py b/contrib/python/tzlocal/py3/tzlocal/utils.py deleted file mode 100644 index 3990535f17b..00000000000 --- a/contrib/python/tzlocal/py3/tzlocal/utils.py +++ /dev/null @@ -1,112 +0,0 @@ -import calendar -import datetime -import logging -import os -import time -import warnings - -try: - import zoneinfo # pragma: no cover -except ImportError: - from backports import zoneinfo # pragma: no cover - -from tzlocal import windows_tz - -log = logging.getLogger("tzlocal") - - -def get_tz_offset(tz): - """Get timezone's offset using built-in function datetime.utcoffset().""" - return int(datetime.datetime.now(tz).utcoffset().total_seconds()) - - -def assert_tz_offset(tz, error=True): - """Assert that system's timezone offset equals to the timezone offset found. - - If they don't match, we probably have a misconfiguration, for example, an - incorrect timezone set in /etc/timezone file in systemd distributions. - - If error is True, this method will raise a ValueError, otherwise it will - emit a warning. - """ - - tz_offset = get_tz_offset(tz) - system_offset = calendar.timegm(time.localtime()) - calendar.timegm(time.gmtime()) - # No one has timezone offsets less than a minute, so this should be close enough: - if abs(tz_offset - system_offset) > 60: - msg = ( - f"Timezone offset does not match system offset: {tz_offset} != {system_offset}. " - "Please, check your config files." - ) - if error: - raise ValueError(msg) - warnings.warn(msg) - - -def _tz_name_from_env(tzenv=None): - if tzenv is None: - tzenv = os.environ.get("TZ") - - if not tzenv: - return None - - log.debug(f"Found a TZ environment: {tzenv}") - - if tzenv[0] == ":": - tzenv = tzenv[1:] - - if tzenv in windows_tz.tz_win: - # Yup, it's a timezone - return tzenv - - if os.path.isabs(tzenv) and os.path.exists(tzenv): - # It's a file specification, expand it, if possible - parts = os.path.realpath(tzenv).split(os.sep) - - # Is it a zone info zone? - possible_tz = "/".join(parts[-2:]) - if possible_tz in windows_tz.tz_win: - # Yup, it is - return possible_tz - - # Maybe it's a short one, like UTC? - if parts[-1] in windows_tz.tz_win: - # Indeed - return parts[-1] - - log.debug("TZ does not contain a time zone name") - return None - - -def _tz_from_env(tzenv=None): - if tzenv is None: - tzenv = os.environ.get("TZ") - - if not tzenv: - return None - - # Some weird format that exists: - if tzenv[0] == ":": - tzenv = tzenv[1:] - - # TZ specifies a file - if os.path.isabs(tzenv) and os.path.exists(tzenv): - # Try to see if we can figure out the name - tzname = _tz_name_from_env(tzenv) - if not tzname: - # Nope, not a standard timezone name, just take the filename - tzname = tzenv.split(os.sep)[-1] - with open(tzenv, "rb") as tzfile: - return zoneinfo.ZoneInfo.from_file(tzfile, key=tzname) - - # TZ must specify a zoneinfo zone. - try: - tz = zoneinfo.ZoneInfo(tzenv) - # That worked, so we return this: - return tz - except zoneinfo.ZoneInfoNotFoundError: - # Nope, it's something like "PST4DST" etc, we can't handle that. - raise zoneinfo.ZoneInfoNotFoundError( - f"tzlocal() does not support non-zoneinfo timezones like {tzenv}. \n" - "Please use a timezone in the form of Continent/City" - ) from None diff --git a/contrib/python/tzlocal/py3/tzlocal/win32.py b/contrib/python/tzlocal/py3/tzlocal/win32.py deleted file mode 100644 index 2fa59fe6ca9..00000000000 --- a/contrib/python/tzlocal/py3/tzlocal/win32.py +++ /dev/null @@ -1,147 +0,0 @@ -import logging -from datetime import datetime - -try: - import _winreg as winreg -except ImportError: - import winreg - -try: - import zoneinfo # pragma: no cover -except ImportError: - from backports import zoneinfo # pragma: no cover - -from tzlocal import utils -from tzlocal.windows_tz import win_tz - -_cache_tz = None -_cache_tz_name = None - -log = logging.getLogger("tzlocal") - - -def valuestodict(key): - """Convert a registry key's values to a dictionary.""" - result = {} - size = winreg.QueryInfoKey(key)[1] - for i in range(size): - data = winreg.EnumValue(key, i) - result[data[0]] = data[1] - return result - - -def _get_dst_info(tz): - # Find the offset for when it doesn't have DST: - dst_offset = std_offset = None - has_dst = False - year = datetime.now().year - for dt in (datetime(year, 1, 1), datetime(year, 6, 1)): - if tz.dst(dt).total_seconds() == 0.0: - # OK, no DST during winter, get this offset - std_offset = tz.utcoffset(dt).total_seconds() - else: - has_dst = True - - return has_dst, std_offset, dst_offset - - -def _get_localzone_name(): - # Windows is special. It has unique time zone names (in several - # meanings of the word) available, but unfortunately, they can be - # translated to the language of the operating system, so we need to - # do a backwards lookup, by going through all time zones and see which - # one matches. - tzenv = utils._tz_name_from_env() - if tzenv: - return tzenv - - log.debug("Looking up time zone info from registry") - handle = winreg.ConnectRegistry(None, winreg.HKEY_LOCAL_MACHINE) - - TZLOCALKEYNAME = r"SYSTEM\CurrentControlSet\Control\TimeZoneInformation" - localtz = winreg.OpenKey(handle, TZLOCALKEYNAME) - keyvalues = valuestodict(localtz) - localtz.Close() - - if "TimeZoneKeyName" in keyvalues: - # Windows 7 and later - - # For some reason this returns a string with loads of NUL bytes at - # least on some systems. I don't know if this is a bug somewhere, I - # just work around it. - tzkeyname = keyvalues["TimeZoneKeyName"].split("\x00", 1)[0] - else: - # Don't support XP any longer - raise LookupError("Can not find Windows timezone configuration") - - timezone = win_tz.get(tzkeyname) - if timezone is None: - # Nope, that didn't work. Try adding "Standard Time", - # it seems to work a lot of times: - timezone = win_tz.get(tzkeyname + " Standard Time") - - # Return what we have. - if timezone is None: - raise zoneinfo.ZoneInfoNotFoundError(tzkeyname) - - if keyvalues.get("DynamicDaylightTimeDisabled", 0) == 1: - # DST is disabled, so don't return the timezone name, - # instead return Etc/GMT+offset - - tz = zoneinfo.ZoneInfo(timezone) - has_dst, std_offset, dst_offset = _get_dst_info(tz) - if not has_dst: - # The DST is turned off in the windows configuration, - # but this timezone doesn't have DST so it doesn't matter - return timezone - - if std_offset is None: - raise zoneinfo.ZoneInfoNotFoundError( - f"{tzkeyname} claims to not have a non-DST time!?" - ) - - if std_offset % 3600: - # I can't convert this to an hourly offset - raise zoneinfo.ZoneInfoNotFoundError( - f"tzlocal can't support disabling DST in the {timezone} zone." - ) - - # This has whole hours as offset, return it as Etc/GMT - return f"Etc/GMT{-std_offset//3600:+.0f}" - - return timezone - - -def get_localzone_name() -> str: - """Get the zoneinfo timezone name that matches the Windows-configured timezone.""" - global _cache_tz_name - if _cache_tz_name is None: - _cache_tz_name = _get_localzone_name() - - return _cache_tz_name - - -def get_localzone() -> zoneinfo.ZoneInfo: - """Returns the zoneinfo-based tzinfo object that matches the Windows-configured timezone.""" - - global _cache_tz - if _cache_tz is None: - _cache_tz = zoneinfo.ZoneInfo(get_localzone_name()) - - if not utils._tz_name_from_env(): - # If the timezone does NOT come from a TZ environment variable, - # verify that it's correct. If it's from the environment, - # we accept it, this is so you can run tests with different timezones. - utils.assert_tz_offset(_cache_tz, error=False) - - return _cache_tz - - -def reload_localzone() -> zoneinfo.ZoneInfo: - """Reload the cached localzone. You need to call this if the timezone has changed.""" - global _cache_tz - global _cache_tz_name - _cache_tz_name = _get_localzone_name() - _cache_tz = zoneinfo.ZoneInfo(_cache_tz_name) - utils.assert_tz_offset(_cache_tz, error=False) - return _cache_tz diff --git a/contrib/python/tzlocal/py3/tzlocal/windows_tz.py b/contrib/python/tzlocal/py3/tzlocal/windows_tz.py deleted file mode 100644 index 3d47576086d..00000000000 --- a/contrib/python/tzlocal/py3/tzlocal/windows_tz.py +++ /dev/null @@ -1,736 +0,0 @@ -# This file is autogenerated by the update_windows_mapping.py script -# Do not edit. -win_tz = { - "AUS Central Standard Time": "Australia/Darwin", - "AUS Eastern Standard Time": "Australia/Sydney", - "Afghanistan Standard Time": "Asia/Kabul", - "Alaskan Standard Time": "America/Anchorage", - "Aleutian Standard Time": "America/Adak", - "Altai Standard Time": "Asia/Barnaul", - "Arab Standard Time": "Asia/Riyadh", - "Arabian Standard Time": "Asia/Dubai", - "Arabic Standard Time": "Asia/Baghdad", - "Argentina Standard Time": "America/Buenos_Aires", - "Astrakhan Standard Time": "Europe/Astrakhan", - "Atlantic Standard Time": "America/Halifax", - "Aus Central W. Standard Time": "Australia/Eucla", - "Azerbaijan Standard Time": "Asia/Baku", - "Azores Standard Time": "Atlantic/Azores", - "Bahia Standard Time": "America/Bahia", - "Bangladesh Standard Time": "Asia/Dhaka", - "Belarus Standard Time": "Europe/Minsk", - "Bougainville Standard Time": "Pacific/Bougainville", - "Canada Central Standard Time": "America/Regina", - "Cape Verde Standard Time": "Atlantic/Cape_Verde", - "Caucasus Standard Time": "Asia/Yerevan", - "Cen. Australia Standard Time": "Australia/Adelaide", - "Central America Standard Time": "America/Guatemala", - "Central Asia Standard Time": "Asia/Almaty", - "Central Brazilian Standard Time": "America/Cuiaba", - "Central Europe Standard Time": "Europe/Budapest", - "Central European Standard Time": "Europe/Warsaw", - "Central Pacific Standard Time": "Pacific/Guadalcanal", - "Central Standard Time": "America/Chicago", - "Central Standard Time (Mexico)": "America/Mexico_City", - "Chatham Islands Standard Time": "Pacific/Chatham", - "China Standard Time": "Asia/Shanghai", - "Cuba Standard Time": "America/Havana", - "Dateline Standard Time": "Etc/GMT+12", - "E. Africa Standard Time": "Africa/Nairobi", - "E. Australia Standard Time": "Australia/Brisbane", - "E. Europe Standard Time": "Europe/Chisinau", - "E. South America Standard Time": "America/Sao_Paulo", - "Easter Island Standard Time": "Pacific/Easter", - "Eastern Standard Time": "America/New_York", - "Eastern Standard Time (Mexico)": "America/Cancun", - "Egypt Standard Time": "Africa/Cairo", - "Ekaterinburg Standard Time": "Asia/Yekaterinburg", - "FLE Standard Time": "Europe/Kiev", - "Fiji Standard Time": "Pacific/Fiji", - "GMT Standard Time": "Europe/London", - "GTB Standard Time": "Europe/Bucharest", - "Georgian Standard Time": "Asia/Tbilisi", - "Greenland Standard Time": "America/Godthab", - "Greenwich Standard Time": "Atlantic/Reykjavik", - "Haiti Standard Time": "America/Port-au-Prince", - "Hawaiian Standard Time": "Pacific/Honolulu", - "India Standard Time": "Asia/Calcutta", - "Iran Standard Time": "Asia/Tehran", - "Israel Standard Time": "Asia/Jerusalem", - "Jordan Standard Time": "Asia/Amman", - "Kaliningrad Standard Time": "Europe/Kaliningrad", - "Korea Standard Time": "Asia/Seoul", - "Libya Standard Time": "Africa/Tripoli", - "Line Islands Standard Time": "Pacific/Kiritimati", - "Lord Howe Standard Time": "Australia/Lord_Howe", - "Magadan Standard Time": "Asia/Magadan", - "Magallanes Standard Time": "America/Punta_Arenas", - "Marquesas Standard Time": "Pacific/Marquesas", - "Mauritius Standard Time": "Indian/Mauritius", - "Middle East Standard Time": "Asia/Beirut", - "Montevideo Standard Time": "America/Montevideo", - "Morocco Standard Time": "Africa/Casablanca", - "Mountain Standard Time": "America/Denver", - "Mountain Standard Time (Mexico)": "America/Mazatlan", - "Myanmar Standard Time": "Asia/Rangoon", - "N. Central Asia Standard Time": "Asia/Novosibirsk", - "Namibia Standard Time": "Africa/Windhoek", - "Nepal Standard Time": "Asia/Katmandu", - "New Zealand Standard Time": "Pacific/Auckland", - "Newfoundland Standard Time": "America/St_Johns", - "Norfolk Standard Time": "Pacific/Norfolk", - "North Asia East Standard Time": "Asia/Irkutsk", - "North Asia Standard Time": "Asia/Krasnoyarsk", - "North Korea Standard Time": "Asia/Pyongyang", - "Omsk Standard Time": "Asia/Omsk", - "Pacific SA Standard Time": "America/Santiago", - "Pacific Standard Time": "America/Los_Angeles", - "Pacific Standard Time (Mexico)": "America/Tijuana", - "Pakistan Standard Time": "Asia/Karachi", - "Paraguay Standard Time": "America/Asuncion", - "Qyzylorda Standard Time": "Asia/Qyzylorda", - "Romance Standard Time": "Europe/Paris", - "Russia Time Zone 10": "Asia/Srednekolymsk", - "Russia Time Zone 11": "Asia/Kamchatka", - "Russia Time Zone 3": "Europe/Samara", - "Russian Standard Time": "Europe/Moscow", - "SA Eastern Standard Time": "America/Cayenne", - "SA Pacific Standard Time": "America/Bogota", - "SA Western Standard Time": "America/La_Paz", - "SE Asia Standard Time": "Asia/Bangkok", - "Saint Pierre Standard Time": "America/Miquelon", - "Sakhalin Standard Time": "Asia/Sakhalin", - "Samoa Standard Time": "Pacific/Apia", - "Sao Tome Standard Time": "Africa/Sao_Tome", - "Saratov Standard Time": "Europe/Saratov", - "Singapore Standard Time": "Asia/Singapore", - "South Africa Standard Time": "Africa/Johannesburg", - "South Sudan Standard Time": "Africa/Juba", - "Sri Lanka Standard Time": "Asia/Colombo", - "Sudan Standard Time": "Africa/Khartoum", - "Syria Standard Time": "Asia/Damascus", - "Taipei Standard Time": "Asia/Taipei", - "Tasmania Standard Time": "Australia/Hobart", - "Tocantins Standard Time": "America/Araguaina", - "Tokyo Standard Time": "Asia/Tokyo", - "Tomsk Standard Time": "Asia/Tomsk", - "Tonga Standard Time": "Pacific/Tongatapu", - "Transbaikal Standard Time": "Asia/Chita", - "Turkey Standard Time": "Europe/Istanbul", - "Turks And Caicos Standard Time": "America/Grand_Turk", - "US Eastern Standard Time": "America/Indianapolis", - "US Mountain Standard Time": "America/Phoenix", - "UTC": "Etc/UTC", - "UTC+12": "Etc/GMT-12", - "UTC+13": "Etc/GMT-13", - "UTC-02": "Etc/GMT+2", - "UTC-08": "Etc/GMT+8", - "UTC-09": "Etc/GMT+9", - "UTC-11": "Etc/GMT+11", - "Ulaanbaatar Standard Time": "Asia/Ulaanbaatar", - "Venezuela Standard Time": "America/Caracas", - "Vladivostok Standard Time": "Asia/Vladivostok", - "Volgograd Standard Time": "Europe/Volgograd", - "W. Australia Standard Time": "Australia/Perth", - "W. Central Africa Standard Time": "Africa/Lagos", - "W. Europe Standard Time": "Europe/Berlin", - "W. Mongolia Standard Time": "Asia/Hovd", - "West Asia Standard Time": "Asia/Tashkent", - "West Bank Standard Time": "Asia/Hebron", - "West Pacific Standard Time": "Pacific/Port_Moresby", - "Yakutsk Standard Time": "Asia/Yakutsk", - "Yukon Standard Time": "America/Whitehorse", -} - -# Old name for the win_tz variable: -tz_names = win_tz - -tz_win = { - "": "Central Standard Time (Mexico)", - "Africa/Abidjan": "Greenwich Standard Time", - "Africa/Accra": "Greenwich Standard Time", - "Africa/Addis_Ababa": "E. Africa Standard Time", - "Africa/Algiers": "W. Central Africa Standard Time", - "Africa/Asmara": "E. Africa Standard Time", - "Africa/Asmera": "E. Africa Standard Time", - "Africa/Bamako": "Greenwich Standard Time", - "Africa/Bangui": "W. Central Africa Standard Time", - "Africa/Banjul": "Greenwich Standard Time", - "Africa/Bissau": "Greenwich Standard Time", - "Africa/Blantyre": "South Africa Standard Time", - "Africa/Brazzaville": "W. Central Africa Standard Time", - "Africa/Bujumbura": "South Africa Standard Time", - "Africa/Cairo": "Egypt Standard Time", - "Africa/Casablanca": "Morocco Standard Time", - "Africa/Ceuta": "Romance Standard Time", - "Africa/Conakry": "Greenwich Standard Time", - "Africa/Dakar": "Greenwich Standard Time", - "Africa/Dar_es_Salaam": "E. Africa Standard Time", - "Africa/Djibouti": "E. Africa Standard Time", - "Africa/Douala": "W. Central Africa Standard Time", - "Africa/El_Aaiun": "Morocco Standard Time", - "Africa/Freetown": "Greenwich Standard Time", - "Africa/Gaborone": "South Africa Standard Time", - "Africa/Harare": "South Africa Standard Time", - "Africa/Johannesburg": "South Africa Standard Time", - "Africa/Juba": "South Sudan Standard Time", - "Africa/Kampala": "E. Africa Standard Time", - "Africa/Khartoum": "Sudan Standard Time", - "Africa/Kigali": "South Africa Standard Time", - "Africa/Kinshasa": "W. Central Africa Standard Time", - "Africa/Lagos": "W. Central Africa Standard Time", - "Africa/Libreville": "W. Central Africa Standard Time", - "Africa/Lome": "Greenwich Standard Time", - "Africa/Luanda": "W. Central Africa Standard Time", - "Africa/Lubumbashi": "South Africa Standard Time", - "Africa/Lusaka": "South Africa Standard Time", - "Africa/Malabo": "W. Central Africa Standard Time", - "Africa/Maputo": "South Africa Standard Time", - "Africa/Maseru": "South Africa Standard Time", - "Africa/Mbabane": "South Africa Standard Time", - "Africa/Mogadishu": "E. Africa Standard Time", - "Africa/Monrovia": "Greenwich Standard Time", - "Africa/Nairobi": "E. Africa Standard Time", - "Africa/Ndjamena": "W. Central Africa Standard Time", - "Africa/Niamey": "W. Central Africa Standard Time", - "Africa/Nouakchott": "Greenwich Standard Time", - "Africa/Ouagadougou": "Greenwich Standard Time", - "Africa/Porto-Novo": "W. Central Africa Standard Time", - "Africa/Sao_Tome": "Sao Tome Standard Time", - "Africa/Timbuktu": "Greenwich Standard Time", - "Africa/Tripoli": "Libya Standard Time", - "Africa/Tunis": "W. Central Africa Standard Time", - "Africa/Windhoek": "Namibia Standard Time", - "America/Adak": "Aleutian Standard Time", - "America/Anchorage": "Alaskan Standard Time", - "America/Anguilla": "SA Western Standard Time", - "America/Antigua": "SA Western Standard Time", - "America/Araguaina": "Tocantins Standard Time", - "America/Argentina/Buenos_Aires": "Argentina Standard Time", - "America/Argentina/Catamarca": "Argentina Standard Time", - "America/Argentina/ComodRivadavia": "Argentina Standard Time", - "America/Argentina/Cordoba": "Argentina Standard Time", - "America/Argentina/Jujuy": "Argentina Standard Time", - "America/Argentina/La_Rioja": "Argentina Standard Time", - "America/Argentina/Mendoza": "Argentina Standard Time", - "America/Argentina/Rio_Gallegos": "Argentina Standard Time", - "America/Argentina/Salta": "Argentina Standard Time", - "America/Argentina/San_Juan": "Argentina Standard Time", - "America/Argentina/San_Luis": "Argentina Standard Time", - "America/Argentina/Tucuman": "Argentina Standard Time", - "America/Argentina/Ushuaia": "Argentina Standard Time", - "America/Aruba": "SA Western Standard Time", - "America/Asuncion": "Paraguay Standard Time", - "America/Atikokan": "SA Pacific Standard Time", - "America/Atka": "Aleutian Standard Time", - "America/Bahia": "Bahia Standard Time", - "America/Bahia_Banderas": "Central Standard Time (Mexico)", - "America/Barbados": "SA Western Standard Time", - "America/Belem": "SA Eastern Standard Time", - "America/Belize": "Central America Standard Time", - "America/Blanc-Sablon": "SA Western Standard Time", - "America/Boa_Vista": "SA Western Standard Time", - "America/Bogota": "SA Pacific Standard Time", - "America/Boise": "Mountain Standard Time", - "America/Buenos_Aires": "Argentina Standard Time", - "America/Cambridge_Bay": "Mountain Standard Time", - "America/Campo_Grande": "Central Brazilian Standard Time", - "America/Cancun": "Eastern Standard Time (Mexico)", - "America/Caracas": "Venezuela Standard Time", - "America/Catamarca": "Argentina Standard Time", - "America/Cayenne": "SA Eastern Standard Time", - "America/Cayman": "SA Pacific Standard Time", - "America/Chicago": "Central Standard Time", - "America/Chihuahua": "Central Standard Time (Mexico)", - "America/Ciudad_Juarez": "Mountain Standard Time", - "America/Coral_Harbour": "SA Pacific Standard Time", - "America/Cordoba": "Argentina Standard Time", - "America/Costa_Rica": "Central America Standard Time", - "America/Creston": "US Mountain Standard Time", - "America/Cuiaba": "Central Brazilian Standard Time", - "America/Curacao": "SA Western Standard Time", - "America/Danmarkshavn": "Greenwich Standard Time", - "America/Dawson": "Yukon Standard Time", - "America/Dawson_Creek": "US Mountain Standard Time", - "America/Denver": "Mountain Standard Time", - "America/Detroit": "Eastern Standard Time", - "America/Dominica": "SA Western Standard Time", - "America/Edmonton": "Mountain Standard Time", - "America/Eirunepe": "SA Pacific Standard Time", - "America/El_Salvador": "Central America Standard Time", - "America/Ensenada": "Pacific Standard Time (Mexico)", - "America/Fort_Nelson": "US Mountain Standard Time", - "America/Fort_Wayne": "US Eastern Standard Time", - "America/Fortaleza": "SA Eastern Standard Time", - "America/Glace_Bay": "Atlantic Standard Time", - "America/Godthab": "Greenland Standard Time", - "America/Goose_Bay": "Atlantic Standard Time", - "America/Grand_Turk": "Turks And Caicos Standard Time", - "America/Grenada": "SA Western Standard Time", - "America/Guadeloupe": "SA Western Standard Time", - "America/Guatemala": "Central America Standard Time", - "America/Guayaquil": "SA Pacific Standard Time", - "America/Guyana": "SA Western Standard Time", - "America/Halifax": "Atlantic Standard Time", - "America/Havana": "Cuba Standard Time", - "America/Hermosillo": "US Mountain Standard Time", - "America/Indiana/Indianapolis": "US Eastern Standard Time", - "America/Indiana/Knox": "Central Standard Time", - "America/Indiana/Marengo": "US Eastern Standard Time", - "America/Indiana/Petersburg": "Eastern Standard Time", - "America/Indiana/Tell_City": "Central Standard Time", - "America/Indiana/Vevay": "US Eastern Standard Time", - "America/Indiana/Vincennes": "Eastern Standard Time", - "America/Indiana/Winamac": "Eastern Standard Time", - "America/Indianapolis": "US Eastern Standard Time", - "America/Inuvik": "Mountain Standard Time", - "America/Iqaluit": "Eastern Standard Time", - "America/Jamaica": "SA Pacific Standard Time", - "America/Jujuy": "Argentina Standard Time", - "America/Juneau": "Alaskan Standard Time", - "America/Kentucky/Louisville": "Eastern Standard Time", - "America/Kentucky/Monticello": "Eastern Standard Time", - "America/Knox_IN": "Central Standard Time", - "America/Kralendijk": "SA Western Standard Time", - "America/La_Paz": "SA Western Standard Time", - "America/Lima": "SA Pacific Standard Time", - "America/Los_Angeles": "Pacific Standard Time", - "America/Louisville": "Eastern Standard Time", - "America/Lower_Princes": "SA Western Standard Time", - "America/Maceio": "SA Eastern Standard Time", - "America/Managua": "Central America Standard Time", - "America/Manaus": "SA Western Standard Time", - "America/Marigot": "SA Western Standard Time", - "America/Martinique": "SA Western Standard Time", - "America/Matamoros": "Central Standard Time", - "America/Mazatlan": "Mountain Standard Time (Mexico)", - "America/Mendoza": "Argentina Standard Time", - "America/Menominee": "Central Standard Time", - "America/Merida": "Central Standard Time (Mexico)", - "America/Metlakatla": "Alaskan Standard Time", - "America/Mexico_City": "Central Standard Time (Mexico)", - "America/Miquelon": "Saint Pierre Standard Time", - "America/Moncton": "Atlantic Standard Time", - "America/Monterrey": "Central Standard Time (Mexico)", - "America/Montevideo": "Montevideo Standard Time", - "America/Montreal": "Eastern Standard Time", - "America/Montserrat": "SA Western Standard Time", - "America/Nassau": "Eastern Standard Time", - "America/New_York": "Eastern Standard Time", - "America/Nipigon": "Eastern Standard Time", - "America/Nome": "Alaskan Standard Time", - "America/Noronha": "UTC-02", - "America/North_Dakota/Beulah": "Central Standard Time", - "America/North_Dakota/Center": "Central Standard Time", - "America/North_Dakota/New_Salem": "Central Standard Time", - "America/Nuuk": "Greenland Standard Time", - "America/Ojinaga": "Central Standard Time", - "America/Panama": "SA Pacific Standard Time", - "America/Pangnirtung": "Eastern Standard Time", - "America/Paramaribo": "SA Eastern Standard Time", - "America/Phoenix": "US Mountain Standard Time", - "America/Port-au-Prince": "Haiti Standard Time", - "America/Port_of_Spain": "SA Western Standard Time", - "America/Porto_Acre": "SA Pacific Standard Time", - "America/Porto_Velho": "SA Western Standard Time", - "America/Puerto_Rico": "SA Western Standard Time", - "America/Punta_Arenas": "Magallanes Standard Time", - "America/Rainy_River": "Central Standard Time", - "America/Rankin_Inlet": "Central Standard Time", - "America/Recife": "SA Eastern Standard Time", - "America/Regina": "Canada Central Standard Time", - "America/Resolute": "Central Standard Time", - "America/Rio_Branco": "SA Pacific Standard Time", - "America/Rosario": "Argentina Standard Time", - "America/Santa_Isabel": "Pacific Standard Time (Mexico)", - "America/Santarem": "SA Eastern Standard Time", - "America/Santiago": "Pacific SA Standard Time", - "America/Santo_Domingo": "SA Western Standard Time", - "America/Sao_Paulo": "E. South America Standard Time", - "America/Scoresbysund": "Azores Standard Time", - "America/Shiprock": "Mountain Standard Time", - "America/Sitka": "Alaskan Standard Time", - "America/St_Barthelemy": "SA Western Standard Time", - "America/St_Johns": "Newfoundland Standard Time", - "America/St_Kitts": "SA Western Standard Time", - "America/St_Lucia": "SA Western Standard Time", - "America/St_Thomas": "SA Western Standard Time", - "America/St_Vincent": "SA Western Standard Time", - "America/Swift_Current": "Canada Central Standard Time", - "America/Tegucigalpa": "Central America Standard Time", - "America/Thule": "Atlantic Standard Time", - "America/Thunder_Bay": "Eastern Standard Time", - "America/Tijuana": "Pacific Standard Time (Mexico)", - "America/Toronto": "Eastern Standard Time", - "America/Tortola": "SA Western Standard Time", - "America/Vancouver": "Pacific Standard Time", - "America/Virgin": "SA Western Standard Time", - "America/Whitehorse": "Yukon Standard Time", - "America/Winnipeg": "Central Standard Time", - "America/Yakutat": "Alaskan Standard Time", - "America/Yellowknife": "Mountain Standard Time", - "Antarctica/Casey": "Central Pacific Standard Time", - "Antarctica/Davis": "SE Asia Standard Time", - "Antarctica/DumontDUrville": "West Pacific Standard Time", - "Antarctica/Macquarie": "Tasmania Standard Time", - "Antarctica/Mawson": "West Asia Standard Time", - "Antarctica/McMurdo": "New Zealand Standard Time", - "Antarctica/Palmer": "SA Eastern Standard Time", - "Antarctica/Rothera": "SA Eastern Standard Time", - "Antarctica/South_Pole": "New Zealand Standard Time", - "Antarctica/Syowa": "E. Africa Standard Time", - "Antarctica/Vostok": "Central Asia Standard Time", - "Arctic/Longyearbyen": "W. Europe Standard Time", - "Asia/Aden": "Arab Standard Time", - "Asia/Almaty": "Central Asia Standard Time", - "Asia/Amman": "Jordan Standard Time", - "Asia/Anadyr": "Russia Time Zone 11", - "Asia/Aqtau": "West Asia Standard Time", - "Asia/Aqtobe": "West Asia Standard Time", - "Asia/Ashgabat": "West Asia Standard Time", - "Asia/Ashkhabad": "West Asia Standard Time", - "Asia/Atyrau": "West Asia Standard Time", - "Asia/Baghdad": "Arabic Standard Time", - "Asia/Bahrain": "Arab Standard Time", - "Asia/Baku": "Azerbaijan Standard Time", - "Asia/Bangkok": "SE Asia Standard Time", - "Asia/Barnaul": "Altai Standard Time", - "Asia/Beirut": "Middle East Standard Time", - "Asia/Bishkek": "Central Asia Standard Time", - "Asia/Brunei": "Singapore Standard Time", - "Asia/Calcutta": "India Standard Time", - "Asia/Chita": "Transbaikal Standard Time", - "Asia/Choibalsan": "Ulaanbaatar Standard Time", - "Asia/Chongqing": "China Standard Time", - "Asia/Chungking": "China Standard Time", - "Asia/Colombo": "Sri Lanka Standard Time", - "Asia/Dacca": "Bangladesh Standard Time", - "Asia/Damascus": "Syria Standard Time", - "Asia/Dhaka": "Bangladesh Standard Time", - "Asia/Dili": "Tokyo Standard Time", - "Asia/Dubai": "Arabian Standard Time", - "Asia/Dushanbe": "West Asia Standard Time", - "Asia/Famagusta": "GTB Standard Time", - "Asia/Gaza": "West Bank Standard Time", - "Asia/Harbin": "China Standard Time", - "Asia/Hebron": "West Bank Standard Time", - "Asia/Ho_Chi_Minh": "SE Asia Standard Time", - "Asia/Hong_Kong": "China Standard Time", - "Asia/Hovd": "W. Mongolia Standard Time", - "Asia/Irkutsk": "North Asia East Standard Time", - "Asia/Istanbul": "Turkey Standard Time", - "Asia/Jakarta": "SE Asia Standard Time", - "Asia/Jayapura": "Tokyo Standard Time", - "Asia/Jerusalem": "Israel Standard Time", - "Asia/Kabul": "Afghanistan Standard Time", - "Asia/Kamchatka": "Russia Time Zone 11", - "Asia/Karachi": "Pakistan Standard Time", - "Asia/Kashgar": "Central Asia Standard Time", - "Asia/Kathmandu": "Nepal Standard Time", - "Asia/Katmandu": "Nepal Standard Time", - "Asia/Khandyga": "Yakutsk Standard Time", - "Asia/Kolkata": "India Standard Time", - "Asia/Krasnoyarsk": "North Asia Standard Time", - "Asia/Kuala_Lumpur": "Singapore Standard Time", - "Asia/Kuching": "Singapore Standard Time", - "Asia/Kuwait": "Arab Standard Time", - "Asia/Macao": "China Standard Time", - "Asia/Macau": "China Standard Time", - "Asia/Magadan": "Magadan Standard Time", - "Asia/Makassar": "Singapore Standard Time", - "Asia/Manila": "Singapore Standard Time", - "Asia/Muscat": "Arabian Standard Time", - "Asia/Nicosia": "GTB Standard Time", - "Asia/Novokuznetsk": "North Asia Standard Time", - "Asia/Novosibirsk": "N. Central Asia Standard Time", - "Asia/Omsk": "Omsk Standard Time", - "Asia/Oral": "West Asia Standard Time", - "Asia/Phnom_Penh": "SE Asia Standard Time", - "Asia/Pontianak": "SE Asia Standard Time", - "Asia/Pyongyang": "North Korea Standard Time", - "Asia/Qatar": "Arab Standard Time", - "Asia/Qostanay": "Central Asia Standard Time", - "Asia/Qyzylorda": "Qyzylorda Standard Time", - "Asia/Rangoon": "Myanmar Standard Time", - "Asia/Riyadh": "Arab Standard Time", - "Asia/Saigon": "SE Asia Standard Time", - "Asia/Sakhalin": "Sakhalin Standard Time", - "Asia/Samarkand": "West Asia Standard Time", - "Asia/Seoul": "Korea Standard Time", - "Asia/Shanghai": "China Standard Time", - "Asia/Singapore": "Singapore Standard Time", - "Asia/Srednekolymsk": "Russia Time Zone 10", - "Asia/Taipei": "Taipei Standard Time", - "Asia/Tashkent": "West Asia Standard Time", - "Asia/Tbilisi": "Georgian Standard Time", - "Asia/Tehran": "Iran Standard Time", - "Asia/Tel_Aviv": "Israel Standard Time", - "Asia/Thimbu": "Bangladesh Standard Time", - "Asia/Thimphu": "Bangladesh Standard Time", - "Asia/Tokyo": "Tokyo Standard Time", - "Asia/Tomsk": "Tomsk Standard Time", - "Asia/Ujung_Pandang": "Singapore Standard Time", - "Asia/Ulaanbaatar": "Ulaanbaatar Standard Time", - "Asia/Ulan_Bator": "Ulaanbaatar Standard Time", - "Asia/Urumqi": "Central Asia Standard Time", - "Asia/Ust-Nera": "Vladivostok Standard Time", - "Asia/Vientiane": "SE Asia Standard Time", - "Asia/Vladivostok": "Vladivostok Standard Time", - "Asia/Yakutsk": "Yakutsk Standard Time", - "Asia/Yangon": "Myanmar Standard Time", - "Asia/Yekaterinburg": "Ekaterinburg Standard Time", - "Asia/Yerevan": "Caucasus Standard Time", - "Atlantic/Azores": "Azores Standard Time", - "Atlantic/Bermuda": "Atlantic Standard Time", - "Atlantic/Canary": "GMT Standard Time", - "Atlantic/Cape_Verde": "Cape Verde Standard Time", - "Atlantic/Faeroe": "GMT Standard Time", - "Atlantic/Faroe": "GMT Standard Time", - "Atlantic/Jan_Mayen": "W. Europe Standard Time", - "Atlantic/Madeira": "GMT Standard Time", - "Atlantic/Reykjavik": "Greenwich Standard Time", - "Atlantic/South_Georgia": "UTC-02", - "Atlantic/St_Helena": "Greenwich Standard Time", - "Atlantic/Stanley": "SA Eastern Standard Time", - "Australia/ACT": "AUS Eastern Standard Time", - "Australia/Adelaide": "Cen. Australia Standard Time", - "Australia/Brisbane": "E. Australia Standard Time", - "Australia/Broken_Hill": "Cen. Australia Standard Time", - "Australia/Canberra": "AUS Eastern Standard Time", - "Australia/Currie": "Tasmania Standard Time", - "Australia/Darwin": "AUS Central Standard Time", - "Australia/Eucla": "Aus Central W. Standard Time", - "Australia/Hobart": "Tasmania Standard Time", - "Australia/LHI": "Lord Howe Standard Time", - "Australia/Lindeman": "E. Australia Standard Time", - "Australia/Lord_Howe": "Lord Howe Standard Time", - "Australia/Melbourne": "AUS Eastern Standard Time", - "Australia/NSW": "AUS Eastern Standard Time", - "Australia/North": "AUS Central Standard Time", - "Australia/Perth": "W. Australia Standard Time", - "Australia/Queensland": "E. Australia Standard Time", - "Australia/South": "Cen. Australia Standard Time", - "Australia/Sydney": "AUS Eastern Standard Time", - "Australia/Tasmania": "Tasmania Standard Time", - "Australia/Victoria": "AUS Eastern Standard Time", - "Australia/West": "W. Australia Standard Time", - "Australia/Yancowinna": "Cen. Australia Standard Time", - "Brazil/Acre": "SA Pacific Standard Time", - "Brazil/DeNoronha": "UTC-02", - "Brazil/East": "E. South America Standard Time", - "Brazil/West": "SA Western Standard Time", - "CST6CDT": "Central Standard Time", - "Canada/Atlantic": "Atlantic Standard Time", - "Canada/Central": "Central Standard Time", - "Canada/Eastern": "Eastern Standard Time", - "Canada/Mountain": "Mountain Standard Time", - "Canada/Newfoundland": "Newfoundland Standard Time", - "Canada/Pacific": "Pacific Standard Time", - "Canada/Saskatchewan": "Canada Central Standard Time", - "Canada/Yukon": "Yukon Standard Time", - "Chile/Continental": "Pacific SA Standard Time", - "Chile/EasterIsland": "Easter Island Standard Time", - "Cuba": "Cuba Standard Time", - "EST5EDT": "Eastern Standard Time", - "Egypt": "Egypt Standard Time", - "Eire": "GMT Standard Time", - "Etc/GMT": "UTC", - "Etc/GMT+0": "UTC", - "Etc/GMT+1": "Cape Verde Standard Time", - "Etc/GMT+10": "Hawaiian Standard Time", - "Etc/GMT+11": "UTC-11", - "Etc/GMT+12": "Dateline Standard Time", - "Etc/GMT+2": "UTC-02", - "Etc/GMT+3": "SA Eastern Standard Time", - "Etc/GMT+4": "SA Western Standard Time", - "Etc/GMT+5": "SA Pacific Standard Time", - "Etc/GMT+6": "Central America Standard Time", - "Etc/GMT+7": "US Mountain Standard Time", - "Etc/GMT+8": "UTC-08", - "Etc/GMT+9": "UTC-09", - "Etc/GMT-0": "UTC", - "Etc/GMT-1": "W. Central Africa Standard Time", - "Etc/GMT-10": "West Pacific Standard Time", - "Etc/GMT-11": "Central Pacific Standard Time", - "Etc/GMT-12": "UTC+12", - "Etc/GMT-13": "UTC+13", - "Etc/GMT-14": "Line Islands Standard Time", - "Etc/GMT-2": "South Africa Standard Time", - "Etc/GMT-3": "E. Africa Standard Time", - "Etc/GMT-4": "Arabian Standard Time", - "Etc/GMT-5": "West Asia Standard Time", - "Etc/GMT-6": "Central Asia Standard Time", - "Etc/GMT-7": "SE Asia Standard Time", - "Etc/GMT-8": "Singapore Standard Time", - "Etc/GMT-9": "Tokyo Standard Time", - "Etc/GMT0": "UTC", - "Etc/Greenwich": "UTC", - "Etc/UCT": "UTC", - "Etc/UTC": "UTC", - "Etc/Universal": "UTC", - "Etc/Zulu": "UTC", - "Europe/Amsterdam": "W. Europe Standard Time", - "Europe/Andorra": "W. Europe Standard Time", - "Europe/Astrakhan": "Astrakhan Standard Time", - "Europe/Athens": "GTB Standard Time", - "Europe/Belfast": "GMT Standard Time", - "Europe/Belgrade": "Central Europe Standard Time", - "Europe/Berlin": "W. Europe Standard Time", - "Europe/Bratislava": "Central Europe Standard Time", - "Europe/Brussels": "Romance Standard Time", - "Europe/Bucharest": "GTB Standard Time", - "Europe/Budapest": "Central Europe Standard Time", - "Europe/Busingen": "W. Europe Standard Time", - "Europe/Chisinau": "E. Europe Standard Time", - "Europe/Copenhagen": "Romance Standard Time", - "Europe/Dublin": "GMT Standard Time", - "Europe/Gibraltar": "W. Europe Standard Time", - "Europe/Guernsey": "GMT Standard Time", - "Europe/Helsinki": "FLE Standard Time", - "Europe/Isle_of_Man": "GMT Standard Time", - "Europe/Istanbul": "Turkey Standard Time", - "Europe/Jersey": "GMT Standard Time", - "Europe/Kaliningrad": "Kaliningrad Standard Time", - "Europe/Kiev": "FLE Standard Time", - "Europe/Kirov": "Russian Standard Time", - "Europe/Kyiv": "FLE Standard Time", - "Europe/Lisbon": "GMT Standard Time", - "Europe/Ljubljana": "Central Europe Standard Time", - "Europe/London": "GMT Standard Time", - "Europe/Luxembourg": "W. Europe Standard Time", - "Europe/Madrid": "Romance Standard Time", - "Europe/Malta": "W. Europe Standard Time", - "Europe/Mariehamn": "FLE Standard Time", - "Europe/Minsk": "Belarus Standard Time", - "Europe/Monaco": "W. Europe Standard Time", - "Europe/Moscow": "Russian Standard Time", - "Europe/Nicosia": "GTB Standard Time", - "Europe/Oslo": "W. Europe Standard Time", - "Europe/Paris": "Romance Standard Time", - "Europe/Podgorica": "Central Europe Standard Time", - "Europe/Prague": "Central Europe Standard Time", - "Europe/Riga": "FLE Standard Time", - "Europe/Rome": "W. Europe Standard Time", - "Europe/Samara": "Russia Time Zone 3", - "Europe/San_Marino": "W. Europe Standard Time", - "Europe/Sarajevo": "Central European Standard Time", - "Europe/Saratov": "Saratov Standard Time", - "Europe/Simferopol": "Russian Standard Time", - "Europe/Skopje": "Central European Standard Time", - "Europe/Sofia": "FLE Standard Time", - "Europe/Stockholm": "W. Europe Standard Time", - "Europe/Tallinn": "FLE Standard Time", - "Europe/Tirane": "Central Europe Standard Time", - "Europe/Tiraspol": "E. Europe Standard Time", - "Europe/Ulyanovsk": "Astrakhan Standard Time", - "Europe/Uzhgorod": "FLE Standard Time", - "Europe/Vaduz": "W. Europe Standard Time", - "Europe/Vatican": "W. Europe Standard Time", - "Europe/Vienna": "W. Europe Standard Time", - "Europe/Vilnius": "FLE Standard Time", - "Europe/Volgograd": "Volgograd Standard Time", - "Europe/Warsaw": "Central European Standard Time", - "Europe/Zagreb": "Central European Standard Time", - "Europe/Zaporozhye": "FLE Standard Time", - "Europe/Zurich": "W. Europe Standard Time", - "GB": "GMT Standard Time", - "GB-Eire": "GMT Standard Time", - "GMT+0": "UTC", - "GMT-0": "UTC", - "GMT0": "UTC", - "Greenwich": "UTC", - "Hongkong": "China Standard Time", - "Iceland": "Greenwich Standard Time", - "Indian/Antananarivo": "E. Africa Standard Time", - "Indian/Chagos": "Central Asia Standard Time", - "Indian/Christmas": "SE Asia Standard Time", - "Indian/Cocos": "Myanmar Standard Time", - "Indian/Comoro": "E. Africa Standard Time", - "Indian/Kerguelen": "West Asia Standard Time", - "Indian/Mahe": "Mauritius Standard Time", - "Indian/Maldives": "West Asia Standard Time", - "Indian/Mauritius": "Mauritius Standard Time", - "Indian/Mayotte": "E. Africa Standard Time", - "Indian/Reunion": "Mauritius Standard Time", - "Iran": "Iran Standard Time", - "Israel": "Israel Standard Time", - "Jamaica": "SA Pacific Standard Time", - "Japan": "Tokyo Standard Time", - "Kwajalein": "UTC+12", - "Libya": "Libya Standard Time", - "MST7MDT": "Mountain Standard Time", - "Mexico/BajaNorte": "Pacific Standard Time (Mexico)", - "Mexico/BajaSur": "Mountain Standard Time (Mexico)", - "Mexico/General": "Central Standard Time (Mexico)", - "NZ": "New Zealand Standard Time", - "NZ-CHAT": "Chatham Islands Standard Time", - "Navajo": "Mountain Standard Time", - "PRC": "China Standard Time", - "PST8PDT": "Pacific Standard Time", - "Pacific/Apia": "Samoa Standard Time", - "Pacific/Auckland": "New Zealand Standard Time", - "Pacific/Bougainville": "Bougainville Standard Time", - "Pacific/Chatham": "Chatham Islands Standard Time", - "Pacific/Chuuk": "West Pacific Standard Time", - "Pacific/Easter": "Easter Island Standard Time", - "Pacific/Efate": "Central Pacific Standard Time", - "Pacific/Enderbury": "UTC+13", - "Pacific/Fakaofo": "UTC+13", - "Pacific/Fiji": "Fiji Standard Time", - "Pacific/Funafuti": "UTC+12", - "Pacific/Galapagos": "Central America Standard Time", - "Pacific/Gambier": "UTC-09", - "Pacific/Guadalcanal": "Central Pacific Standard Time", - "Pacific/Guam": "West Pacific Standard Time", - "Pacific/Honolulu": "Hawaiian Standard Time", - "Pacific/Johnston": "Hawaiian Standard Time", - "Pacific/Kanton": "UTC+13", - "Pacific/Kiritimati": "Line Islands Standard Time", - "Pacific/Kosrae": "Central Pacific Standard Time", - "Pacific/Kwajalein": "UTC+12", - "Pacific/Majuro": "UTC+12", - "Pacific/Marquesas": "Marquesas Standard Time", - "Pacific/Midway": "UTC-11", - "Pacific/Nauru": "UTC+12", - "Pacific/Niue": "UTC-11", - "Pacific/Norfolk": "Norfolk Standard Time", - "Pacific/Noumea": "Central Pacific Standard Time", - "Pacific/Pago_Pago": "UTC-11", - "Pacific/Palau": "Tokyo Standard Time", - "Pacific/Pitcairn": "UTC-08", - "Pacific/Pohnpei": "Central Pacific Standard Time", - "Pacific/Ponape": "Central Pacific Standard Time", - "Pacific/Port_Moresby": "West Pacific Standard Time", - "Pacific/Rarotonga": "Hawaiian Standard Time", - "Pacific/Saipan": "West Pacific Standard Time", - "Pacific/Samoa": "UTC-11", - "Pacific/Tahiti": "Hawaiian Standard Time", - "Pacific/Tarawa": "UTC+12", - "Pacific/Tongatapu": "Tonga Standard Time", - "Pacific/Truk": "West Pacific Standard Time", - "Pacific/Wake": "UTC+12", - "Pacific/Wallis": "UTC+12", - "Pacific/Yap": "West Pacific Standard Time", - "Poland": "Central European Standard Time", - "Portugal": "GMT Standard Time", - "ROC": "Taipei Standard Time", - "ROK": "Korea Standard Time", - "Singapore": "Singapore Standard Time", - "Turkey": "Turkey Standard Time", - "UCT": "UTC", - "US/Alaska": "Alaskan Standard Time", - "US/Aleutian": "Aleutian Standard Time", - "US/Arizona": "US Mountain Standard Time", - "US/Central": "Central Standard Time", - "US/Eastern": "Eastern Standard Time", - "US/Hawaii": "Hawaiian Standard Time", - "US/Indiana-Starke": "Central Standard Time", - "US/Michigan": "Eastern Standard Time", - "US/Mountain": "Mountain Standard Time", - "US/Pacific": "Pacific Standard Time", - "US/Samoa": "UTC-11", - "UTC": "UTC", - "Universal": "UTC", - "W-SU": "Russian Standard Time", - "Zulu": "UTC", -} diff --git a/contrib/python/tzlocal/py3/ya.make b/contrib/python/tzlocal/py3/ya.make deleted file mode 100644 index 4342acb95cd..00000000000 --- a/contrib/python/tzlocal/py3/ya.make +++ /dev/null @@ -1,35 +0,0 @@ -# Generated by devtools/yamaker (pypi). - -PY3_LIBRARY() - -VERSION(5.2) - -LICENSE(MIT) - -PEERDIR( - contrib/python/tzdata -) - -NO_LINT() - -NO_CHECK_IMPORTS( - tzlocal.win32 -) - -PY_SRCS( - TOP_LEVEL - tzlocal/__init__.py - tzlocal/unix.py - tzlocal/utils.py - tzlocal/win32.py - tzlocal/windows_tz.py -) - -RESOURCE_FILES( - PREFIX contrib/python/tzlocal/py3/ - .dist-info/METADATA - .dist-info/top_level.txt - tzlocal/py.typed -) - -END() diff --git a/contrib/python/tzlocal/ya.make b/contrib/python/tzlocal/ya.make deleted file mode 100644 index b29d8c6c043..00000000000 --- a/contrib/python/tzlocal/ya.make +++ /dev/null @@ -1,18 +0,0 @@ -PY23_LIBRARY() - -LICENSE(Service-Py23-Proxy) - -IF (PYTHON2) - PEERDIR(contrib/python/tzlocal/py2) -ELSE() - PEERDIR(contrib/python/tzlocal/py3) -ENDIF() - -NO_LINT() - -END() - -RECURSE( - py2 - py3 -) diff --git a/contrib/python/ydb/py3/.dist-info/METADATA b/contrib/python/ydb/py3/.dist-info/METADATA index 71ddeb7b6fc..d921ac7f1ff 100644 --- a/contrib/python/ydb/py3/.dist-info/METADATA +++ b/contrib/python/ydb/py3/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: ydb -Version: 3.8.1 +Version: 3.9.0 Summary: YDB Python SDK Home-page: http://github.com/ydb-platform/ydb-python-sdk Author: Yandex LLC @@ -18,6 +18,7 @@ Requires-Dist: grpcio >=1.42.0 Requires-Dist: packaging Requires-Dist: protobuf <5.0.0,>=3.13.0 Requires-Dist: aiohttp <4 +Requires-Dist: pyjwt ==2.8.0 Provides-Extra: yc Requires-Dist: yandexcloud ; extra == 'yc' diff --git a/contrib/python/ydb/py3/ya.make b/contrib/python/ydb/py3/ya.make index 24ba22bc0fe..b8611eae134 100644 --- a/contrib/python/ydb/py3/ya.make +++ b/contrib/python/ydb/py3/ya.make @@ -2,11 +2,12 @@ PY3_LIBRARY() -VERSION(3.8.1) +VERSION(3.9.0) LICENSE(Apache-2.0) PEERDIR( + contrib/python/PyJWT contrib/python/aiohttp contrib/python/grpcio contrib/python/packaging diff --git a/contrib/python/ydb/py3/ydb/aio/iam.py b/contrib/python/ydb/py3/ydb/aio/iam.py index eab8faffe05..40622f8a9d9 100644 --- a/contrib/python/ydb/py3/ydb/aio/iam.py +++ b/contrib/python/ydb/py3/ydb/aio/iam.py @@ -5,15 +5,19 @@ import abc import logging from ydb.iam import auth from .credentials import AbstractExpiringTokenCredentials +from ydb import issues logger = logging.getLogger(__name__) try: - from yandex.cloud.iam.v1 import iam_token_service_pb2_grpc - from yandex.cloud.iam.v1 import iam_token_service_pb2 import jwt except ImportError: jwt = None + +try: + from yandex.cloud.iam.v1 import iam_token_service_pb2_grpc + from yandex.cloud.iam.v1 import iam_token_service_pb2 +except ImportError: iam_token_service_pb2_grpc = None iam_token_service_pb2 = None @@ -55,6 +59,51 @@ class TokenServiceCredentials(AbstractExpiringTokenCredentials): IamTokenCredentials = TokenServiceCredentials +class OAuth2JwtTokenExchangeCredentials(AbstractExpiringTokenCredentials, auth.BaseJWTCredentials): + def __init__( + self, + token_exchange_url, + account_id, + access_key_id, + private_key, + algorithm, + token_service_url, + subject=None, + ): + super(OAuth2JwtTokenExchangeCredentials, self).__init__() + auth.BaseJWTCredentials.__init__( + self, account_id, access_key_id, private_key, algorithm, token_service_url, subject + ) + assert aiohttp is not None, "Install aiohttp library to use OAuth 2.0 token exchange credentials provider" + self._token_exchange_url = token_exchange_url + + async def _make_token_request(self): + params = { + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "requested_token_type": "urn:ietf:params:oauth:token-type:access_token", + "subject_token": self._get_jwt(), + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + } + headers = {"Content-Type": "application/x-www-form-urlencoded"} + + timeout = aiohttp.ClientTimeout(total=2) + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.post(self._token_exchange_url, data=params, headers=headers) as response: + if response.status == 403: + raise issues.Unauthenticated(await response.text()) + if response.status >= 500: + raise issues.Unavailable(await response.text()) + if response.status >= 400: + raise issues.BadRequest(await response.text()) + if response.status != 200: + raise issues.Error(await response.text()) + + response_json = await response.json() + access_token = response_json["access_token"] + expires_in = response_json["expires_in"] + return {"access_token": access_token, "expires_in": expires_in} + + class JWTIamCredentials(TokenServiceCredentials, auth.BaseJWTCredentials): def __init__( self, @@ -65,16 +114,39 @@ class JWTIamCredentials(TokenServiceCredentials, auth.BaseJWTCredentials): iam_channel_credentials=None, ): TokenServiceCredentials.__init__(self, iam_endpoint, iam_channel_credentials) - auth.BaseJWTCredentials.__init__(self, account_id, access_key_id, private_key) + auth.BaseJWTCredentials.__init__( + self, + account_id, + access_key_id, + private_key, + auth.YANDEX_CLOUD_JWT_ALGORITHM, + auth.YANDEX_CLOUD_IAM_TOKEN_SERVICE_URL, + ) def _get_token_request(self): - return iam_token_service_pb2.CreateIamTokenRequest( - jwt=auth.get_jwt( - self._account_id, - self._access_key_id, - self._private_key, - self._jwt_expiration_timeout, - ) + return iam_token_service_pb2.CreateIamTokenRequest(jwt=self._get_jwt()) + + +class NebiusJWTIamCredentials(OAuth2JwtTokenExchangeCredentials): + def __init__( + self, + account_id, + access_key_id, + private_key, + token_exchange_url=None, + ): + url = token_exchange_url + if url is None: + url = auth.NEBIUS_CLOUD_IAM_TOKEN_EXCHANGE_URL + OAuth2JwtTokenExchangeCredentials.__init__( + self, + url, + account_id, + access_key_id, + private_key, + auth.NEBIUS_CLOUD_JWT_ALGORITHM, + auth.NEBIUS_CLOUD_IAM_TOKEN_SERVICE_AUDIENCE, + account_id, ) @@ -130,3 +202,20 @@ class ServiceAccountCredentials(JWTIamCredentials): iam_endpoint, iam_channel_credentials, ) + + +class NebiusServiceAccountCredentials(NebiusJWTIamCredentials): + def __init__( + self, + service_account_id, + access_key_id, + private_key, + iam_endpoint=None, + iam_channel_credentials=None, + ): + super(NebiusServiceAccountCredentials, self).__init__( + service_account_id, + access_key_id, + private_key, + iam_endpoint, + ) diff --git a/contrib/python/ydb/py3/ydb/driver.py b/contrib/python/ydb/py3/ydb/driver.py index 89109b9b575..16bba151540 100644 --- a/contrib/python/ydb/py3/ydb/driver.py +++ b/contrib/python/ydb/py3/ydb/driver.py @@ -38,6 +38,13 @@ def credentials_from_env_variables(tracer=None): return ydb.iam.ServiceAccountCredentials.from_file(service_account_key_file) + nebius_service_account_key_file = os.getenv("YDB_NEBIUS_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS") + if nebius_service_account_key_file is not None: + ctx.trace({"credentials.nebius_service_account_key_file": True}) + import ydb.iam + + return ydb.iam.NebiusServiceAccountCredentials.from_file(nebius_service_account_key_file) + anonymous_credetials = os.getenv("YDB_ANONYMOUS_CREDENTIALS", "0") == "1" if anonymous_credetials: ctx.trace({"credentials.anonymous": True}) diff --git a/contrib/python/ydb/py3/ydb/iam/__init__.py b/contrib/python/ydb/py3/ydb/iam/__init__.py index 7167efe13ee..cf835769dbf 100644 --- a/contrib/python/ydb/py3/ydb/iam/__init__.py +++ b/contrib/python/ydb/py3/ydb/iam/__init__.py @@ -1,3 +1,4 @@ # -*- coding: utf-8 -*- from .auth import ServiceAccountCredentials # noqa +from .auth import NebiusServiceAccountCredentials # noqa from .auth import MetadataUrlCredentials # noqa diff --git a/contrib/python/ydb/py3/ydb/iam/auth.py b/contrib/python/ydb/py3/ydb/iam/auth.py index 82e7c9f6c8e..852c0c28bb3 100644 --- a/contrib/python/ydb/py3/ydb/iam/auth.py +++ b/contrib/python/ydb/py3/ydb/iam/auth.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from ydb import credentials, tracing +from ydb import credentials, tracing, issues import grpc import time import abc @@ -8,11 +8,14 @@ import json import os try: - from yandex.cloud.iam.v1 import iam_token_service_pb2_grpc - from yandex.cloud.iam.v1 import iam_token_service_pb2 import jwt except ImportError: jwt = None + +try: + from yandex.cloud.iam.v1 import iam_token_service_pb2_grpc + from yandex.cloud.iam.v1 import iam_token_service_pb2 +except ImportError: iam_token_service_pb2_grpc = None iam_token_service_pb2 = None @@ -23,22 +26,32 @@ except ImportError: DEFAULT_METADATA_URL = "http://169.254.169.254/computeMetadata/v1/instance/service-accounts/default/token" +YANDEX_CLOUD_IAM_TOKEN_SERVICE_URL = "https://iam.api.cloud.yandex.net/iam/v1/tokens" +NEBIUS_CLOUD_IAM_TOKEN_SERVICE_AUDIENCE = "token-service.iam.new.nebiuscloud.net" +NEBIUS_CLOUD_IAM_TOKEN_EXCHANGE_URL = "https://auth.new.nebiuscloud.net/oauth2/token/exchange" +YANDEX_CLOUD_JWT_ALGORITHM = "PS256" +NEBIUS_CLOUD_JWT_ALGORITHM = "RS256" -def get_jwt(account_id, access_key_id, private_key, jwt_expiration_timeout): + +def get_jwt(account_id, access_key_id, private_key, jwt_expiration_timeout, algorithm, token_service_url, subject=None): + assert jwt is not None, "Install pyjwt library to use jwt tokens" now = time.time() now_utc = datetime.utcfromtimestamp(now) exp_utc = datetime.utcfromtimestamp(now + jwt_expiration_timeout) + payload = { + "iss": account_id, + "aud": token_service_url, + "iat": now_utc, + "exp": exp_utc, + } + if subject is not None: + payload["sub"] = subject return jwt.encode( key=private_key, - algorithm="PS256", - headers={"typ": "JWT", "alg": "PS256", "kid": access_key_id}, - payload={ - "iss": account_id, - "aud": "https://iam.api.cloud.yandex.net/iam/v1/tokens", - "iat": now_utc, - "exp": exp_utc, - }, + algorithm=algorithm, + headers={"typ": "JWT", "alg": algorithm, "kid": access_key_id}, + payload=payload, ) @@ -73,12 +86,15 @@ class TokenServiceCredentials(credentials.AbstractExpiringTokenCredentials): class BaseJWTCredentials(abc.ABC): - def __init__(self, account_id, access_key_id, private_key): + def __init__(self, account_id, access_key_id, private_key, algorithm, token_service_url, subject=None): self._account_id = account_id self._jwt_expiration_timeout = 60.0 * 60 self._token_expiration_timeout = 120 self._access_key_id = access_key_id self._private_key = private_key + self._algorithm = algorithm + self._token_service_url = token_service_url + self._subject = subject def set_token_expiration_timeout(self, value): self._token_expiration_timeout = value @@ -99,6 +115,64 @@ class BaseJWTCredentials(abc.ABC): iam_channel_credentials=iam_channel_credentials, ) + def _get_jwt(self): + return get_jwt( + self._account_id, + self._access_key_id, + self._private_key, + self._jwt_expiration_timeout, + self._algorithm, + self._token_service_url, + self._subject, + ) + + +class OAuth2JwtTokenExchangeCredentials(credentials.AbstractExpiringTokenCredentials, BaseJWTCredentials): + def __init__( + self, + token_exchange_url, + account_id, + access_key_id, + private_key, + algorithm, + token_service_url, + subject=None, + tracer=None, + ): + BaseJWTCredentials.__init__(self, account_id, access_key_id, private_key, algorithm, token_service_url, subject) + super(OAuth2JwtTokenExchangeCredentials, self).__init__(tracer) + assert requests is not None, "Install requests library to use OAuth 2.0 token exchange credentials provider" + self._token_exchange_url = token_exchange_url + + def _process_response_status_code(self, response): + if response.status_code == 403: + raise issues.Unauthenticated(response.content) + if response.status_code >= 500: + raise issues.Unavailable(response.content) + if response.status_code >= 400: + raise issues.BadRequest(response.content) + if response.status_code != 200: + raise issues.Error(response.content) + + def _process_response(self, response): + self._process_response_status_code(response) + response_json = json.loads(response.content) + access_token = response_json["access_token"] + expires_in = response_json["expires_in"] + return {"access_token": access_token, "expires_in": expires_in} + + @tracing.with_trace() + def _make_token_request(self): + params = { + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "requested_token_type": "urn:ietf:params:oauth:token-type:access_token", + "subject_token": self._get_jwt(), + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + } + headers = {"Content-Type": "application/x-www-form-urlencoded"} + response = requests.post(self._token_exchange_url, data=params, headers=headers) + return self._process_response(response) + class JWTIamCredentials(TokenServiceCredentials, BaseJWTCredentials): def __init__( @@ -110,16 +184,34 @@ class JWTIamCredentials(TokenServiceCredentials, BaseJWTCredentials): iam_channel_credentials=None, ): TokenServiceCredentials.__init__(self, iam_endpoint, iam_channel_credentials) - BaseJWTCredentials.__init__(self, account_id, access_key_id, private_key) + BaseJWTCredentials.__init__( + self, account_id, access_key_id, private_key, YANDEX_CLOUD_JWT_ALGORITHM, YANDEX_CLOUD_IAM_TOKEN_SERVICE_URL + ) def _get_token_request(self): - return self._iam_token_service_pb2.CreateIamTokenRequest( - jwt=get_jwt( - self._account_id, - self._access_key_id, - self._private_key, - self._jwt_expiration_timeout, - ) + return self._iam_token_service_pb2.CreateIamTokenRequest(jwt=self._get_jwt()) + + +class NebiusJWTIamCredentials(OAuth2JwtTokenExchangeCredentials): + def __init__( + self, + account_id, + access_key_id, + private_key, + token_exchange_url=None, + ): + url = token_exchange_url + if url is None: + url = NEBIUS_CLOUD_IAM_TOKEN_EXCHANGE_URL + OAuth2JwtTokenExchangeCredentials.__init__( + self, + url, + account_id, + access_key_id, + private_key, + NEBIUS_CLOUD_JWT_ALGORITHM, + NEBIUS_CLOUD_IAM_TOKEN_SERVICE_AUDIENCE, + account_id, ) @@ -176,3 +268,20 @@ class ServiceAccountCredentials(JWTIamCredentials): iam_endpoint, iam_channel_credentials, ) + + +class NebiusServiceAccountCredentials(NebiusJWTIamCredentials): + def __init__( + self, + service_account_id, + access_key_id, + private_key, + iam_endpoint=None, + iam_channel_credentials=None, + ): + super(NebiusServiceAccountCredentials, self).__init__( + service_account_id, + access_key_id, + private_key, + iam_endpoint, + ) diff --git a/contrib/python/ydb/py3/ydb/ydb_version.py b/contrib/python/ydb/py3/ydb/ydb_version.py index a464e6dc077..10baa2a41a3 100644 --- a/contrib/python/ydb/py3/ydb/ydb_version.py +++ b/contrib/python/ydb/py3/ydb/ydb_version.py @@ -1 +1 @@ -VERSION = "3.8.1" +VERSION = "3.9.0" diff --git a/contrib/python/ydb/ya.make b/contrib/python/ydb/ya.make deleted file mode 100644 index 1886a29d3d8..00000000000 --- a/contrib/python/ydb/ya.make +++ /dev/null @@ -1,23 +0,0 @@ -PY23_LIBRARY() - -LICENSE(Service-Py23-Proxy) - -IF (PYTHON2) - PEERDIR(contrib/python/ydb/py2) -ELSE() - PEERDIR(contrib/python/ydb/py3) -ENDIF() - -PEERDIR( - contrib/ydb/public/api/grpc - contrib/ydb/public/api/grpc/draft -) - -NO_LINT() - -END() - -RECURSE( - py2 - py3 -)
\ No newline at end of file diff --git a/library/cpp/http/io/chunk.h b/library/cpp/http/io/chunk.h index 88d89fafda2..6d477b04a9b 100644 --- a/library/cpp/http/io/chunk.h +++ b/library/cpp/http/io/chunk.h @@ -35,8 +35,10 @@ public: TChunkedOutput(IOutputStream* slave); ~TChunkedOutput() override; -private: +protected: void DoWrite(const void* buf, size_t len) override; + +private: void DoFlush() override; void DoFinish() override; diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt index 374cf501922..eb3b65c176b 100644 --- a/library/cpp/tld/tlds-alpha-by-domain.txt +++ b/library/cpp/tld/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2024033102, Last Updated Mon Apr 1 07:07:01 2024 UTC +# Version 2024040700, Last Updated Sun Apr 7 07:07:01 2024 UTC AAA AARP ABB diff --git a/library/cpp/yt/misc/function_view-inl.h b/library/cpp/yt/misc/function_view-inl.h new file mode 100644 index 00000000000..ececfdf3357 --- /dev/null +++ b/library/cpp/yt/misc/function_view-inl.h @@ -0,0 +1,71 @@ +#pragma once +#ifndef FUNCTION_VIEW_INL_H_ +#error "Direct inclusion of this file is not allowed, include function_view.h" +// For the sake of sane code completion. +#include "function_view.h" +#endif + +#include <library/cpp/yt/assert/assert.h> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +template <class TResult, bool NoExcept, class... TArgs> +template <CTypeErasable<TResult(TArgs...) noexcept(NoExcept)> TConcrete> +TFunctionView<TResult(TArgs...) noexcept(NoExcept)>::TFunctionView(TConcrete& concreteRef) noexcept + : TFunctionView(&concreteRef) +{ } + +template <class TResult, bool NoExcept, class... TArgs> +template <CTypeErasable<TResult(TArgs...) noexcept(NoExcept)> TConcrete> +TFunctionView<TResult(TArgs...) noexcept(NoExcept)>::TFunctionView(TConcrete* concretePtr) noexcept +{ + Ptr_ = reinterpret_cast<void*>(concretePtr); + Invoke_ = &TFunctionView::ConcreteInvoke<TConcrete>; +} + +template <class TResult, bool NoExcept, class... TArgs> +TFunctionView<TResult(TArgs...) noexcept(NoExcept)> +TFunctionView<TResult(TArgs...) noexcept(NoExcept)>::Release() noexcept +{ + auto copy = *this; + Reset(); + return copy; +} + +template <class TResult, bool NoExcept, class... TArgs> +TResult TFunctionView<TResult(TArgs...) noexcept(NoExcept)>::operator()(TArgs... args) noexcept(NoExcept) +{ + YT_VERIFY(Ptr_); + return Invoke_(std::forward<TArgs>(args)..., Ptr_); +} + +template <class TResult, bool NoExcept, class... TArgs> +template <class TConcrete> +TResult TFunctionView<TResult(TArgs...) noexcept(NoExcept)>::ConcreteInvoke(TArgs... args, TErasedPtr ptr) noexcept(NoExcept) +{ + return (*reinterpret_cast<TConcrete*>(ptr))(std::forward<TArgs>(args)...); +} + +template <class TResult, bool NoExcept, class... TArgs> +TFunctionView<TResult(TArgs...) noexcept(NoExcept)>::operator bool() const noexcept +{ + return IsValid(); +} + +template <class TResult, bool NoExcept, class... TArgs> +bool TFunctionView<TResult(TArgs...) noexcept(NoExcept)>::IsValid() const noexcept +{ + return Ptr_ != nullptr; +} + +template <class TResult, bool NoExcept, class... TArgs> +void TFunctionView<TResult(TArgs...) noexcept(NoExcept)>::Reset() noexcept +{ + Ptr_ = nullptr; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/library/cpp/yt/misc/function_view.h b/library/cpp/yt/misc/function_view.h new file mode 100644 index 00000000000..259238521ff --- /dev/null +++ b/library/cpp/yt/misc/function_view.h @@ -0,0 +1,139 @@ +#pragma once + +#include <concepts> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +template <class TSignature> +struct TTypeErasureTraits; + +template <class TResult, bool NoExcept, class... TArgs> +struct TTypeErasureTraits<TResult(TArgs...) noexcept(NoExcept)> +{ + using TSignature = TResult(TArgs...) noexcept(NoExcept); + + // TODO(arkady-e1ppa): Support pointer-to-member-function? + template <class T> + static constexpr bool IsInvocable = NoExcept + ? requires (T obj, TArgs... args) { + { obj(std::forward<TArgs>(args)...) } noexcept -> std::same_as<TResult>; + } + : requires (T obj, TArgs... args) { + { obj(std::forward<TArgs>(args)...) } -> std::same_as<TResult>; + }; +}; + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + +// Non-owning type-erasure container. +/* + Example: + + template <class T> + class TSerializedObject + { + public: + explicit TSerializedObject(T value) + : Object_(value) + { } + + void Lock(TFunctionView<void(const T&)> callback) + { + auto guard = Guard(SpinLock_); + callback(Object_); + } + + private: + TSpinLock SpinLock_; + T Object_; + }; + + int main() + { + TSerializedObject<int> object(42); + + // object.Lock([] (const int& value) { + // fmt::println("Value is {}", value); + // }); + // ^ CE -- cannot pass rvalue. + + auto callback = [] (const int& value) { + fmt::println("Value is {}", value); + }; + + object.Lock(callback); // <- prints "Value is 42". + } +*/ +template <class TSignature> +class TFunctionView; + +//////////////////////////////////////////////////////////////////////////////// + +template <class T, class TSignature> +concept CTypeErasable = + NDetail::TTypeErasureTraits<TSignature>::template IsInvocable<T> && + (!std::same_as<T, TFunctionView<TSignature>>); + +//////////////////////////////////////////////////////////////////////////////// + +template <class TResult, bool NoExcept, class... TArgs> +class TFunctionView<TResult(TArgs...) noexcept(NoExcept)> +{ +public: + using TSignature = TResult(TArgs...) noexcept(NoExcept); + + TFunctionView() = default; + + template <CTypeErasable<TSignature> TConcrete> + TFunctionView(TConcrete& concreteRef) noexcept; + + template <CTypeErasable<TSignature> TConcrete> + TFunctionView(TConcrete* concretePtr) noexcept; + + TResult operator()(TArgs... args) noexcept(NoExcept); + + explicit operator bool() const noexcept; + + TFunctionView Release() noexcept; + + bool IsValid() const noexcept; + void Reset() noexcept; + + // bool operator==(const TFunctionView& other) const & = default; + +private: + // NB: Technically, this is UB according to C standard, which + // was not changed for C++ standard. + // This is so because it is allowed to have + // function pointers to be modelled by entities + // different from object pointers. + // No reasonable system architecture (e.g. x86 or ARM + // or any other POSIX compliant one) does this. + // No reasonable compiler (clang/gcc) does anything with this. + // Accounting for such requirement would cause this class + // to have std::variant-like storage which would make this class + // weight more. Thus, we have decided to keep it this way, + // since we are safe on x86 or ARM + clang. + using TErasedPtr = void*; + using TErasedInvoke = TResult(*)(TArgs..., TErasedPtr); + + TErasedPtr Ptr_ = nullptr; + TErasedInvoke Invoke_ = nullptr; + + template <class TConcrete> + static TResult ConcreteInvoke(TArgs... args, TErasedPtr ptr) noexcept(NoExcept); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + +#define FUNCTION_VIEW_INL_H_ +#include "function_view-inl.h" +#undef FUNCTION_VIEW_INL_H_ diff --git a/library/python/cpp_test/test_cpp.py b/library/python/cpp_test/test_cpp.py index efa10ea2833..b4c78f82716 100644 --- a/library/python/cpp_test/test_cpp.py +++ b/library/python/cpp_test/test_cpp.py @@ -10,20 +10,16 @@ from library.python.testing.style import rules import library.python.resource as lpr -STYLE_CONFIG_JSON_14 = json.dumps(yaml.safe_load(lpr.find('resfs/file/config.clang-format'))) -STYLE_CONFIG_JSON_16 = json.dumps(yaml.safe_load(lpr.find('resfs/file/config.clang-format-16'))) +# keep in sync with the logic in https://a.yandex-team.ru/arcadia/devtools/ya/handlers/style/cpp_style.py?rev=r12543375#L21 +STYLE_CONFIG_JSON = json.dumps(yaml.safe_load(lpr.find('resfs/file/config.clang-format'))) RES_FILE_PREFIX = '/cpp_style/files/' CHECKED_PATHS = list(lpr.iterkeys(RES_FILE_PREFIX, strip_prefix=True)) def check_style(filename, actual_source): - try: - clang_format_binary = yatest.common.binary_path('contrib/libs/clang14/tools/clang-format/clang-format') - config = STYLE_CONFIG_JSON_14 - except Exception: - clang_format_binary = yatest.common.binary_path('contrib/libs/clang16/tools/clang-format/clang-format') - config = STYLE_CONFIG_JSON_16 + clang_format_binary = yatest.common.binary_path('contrib/libs/clang16/tools/clang-format/clang-format') + config = STYLE_CONFIG_JSON command = [clang_format_binary, '-assume-filename=' + filename, '-style=' + config] styled_source = subprocess.check_output(command, input=actual_source) diff --git a/ydb/ci/rightlib.txt b/ydb/ci/rightlib.txt index 9a901675ddf..6cd45208af8 100644 --- a/ydb/ci/rightlib.txt +++ b/ydb/ci/rightlib.txt @@ -1 +1 @@ -d2527272d41073b1399b902b3a59218b894c8022 +72eeab5172756159750eef875745e2a6f5b0004f diff --git a/yt/cpp/mapreduce/client/client.cpp b/yt/cpp/mapreduce/client/client.cpp index 4548577834e..caa0ab270c4 100644 --- a/yt/cpp/mapreduce/client/client.cpp +++ b/yt/cpp/mapreduce/client/client.cpp @@ -1361,6 +1361,7 @@ TClientPtr CreateClientImpl( context.ProxyAddress = options.ProxyAddress_; context.ServerName = serverName; + ApplyProxyUrlAliasingRules(context.ServerName); if (context.ServerName.find('.') == TString::npos && context.ServerName.find(':') == TString::npos && @@ -1448,9 +1449,6 @@ IClientPtr CreateClientFromEnv(const TCreateClientOptions& options) if (!serverName) { ythrow yexception() << "YT_PROXY is not set"; } - - NDetail::ApplyProxyUrlAliasingRules(serverName); - return NDetail::CreateClientImpl(serverName, options); } diff --git a/yt/cpp/mapreduce/client/ya.make b/yt/cpp/mapreduce/client/ya.make index 3696a9ba8fa..2d118b24426 100644 --- a/yt/cpp/mapreduce/client/ya.make +++ b/yt/cpp/mapreduce/client/ya.make @@ -42,7 +42,6 @@ PEERDIR( yt/cpp/mapreduce/http yt/cpp/mapreduce/interface yt/cpp/mapreduce/io - yt/cpp/mapreduce/library/table_schema yt/cpp/mapreduce/raw_client ) diff --git a/yt/cpp/mapreduce/common/retry_lib.cpp b/yt/cpp/mapreduce/common/retry_lib.cpp index cf2c021eb44..772a2ab0cd4 100644 --- a/yt/cpp/mapreduce/common/retry_lib.cpp +++ b/yt/cpp/mapreduce/common/retry_lib.cpp @@ -220,10 +220,11 @@ static TMaybe<TDuration> TryGetBackoffDuration(const TErrorResponse& errorRespon // chunk client errors return config->ChunkErrorsRetryInterval; } - for (auto code : TVector<int>{ + for (auto code : { NRpc::TransportError, NRpc::Unavailable, NApi::RetriableArchiveError, + NSequoiaClient::SequoiaRetriableError, Canceled, }) { if (allCodes.contains(code)) { diff --git a/yt/cpp/mapreduce/interface/error_codes.h b/yt/cpp/mapreduce/interface/error_codes.h index e784e3ee88b..35e5e7c9139 100644 --- a/yt/cpp/mapreduce/interface/error_codes.h +++ b/yt/cpp/mapreduce/interface/error_codes.h @@ -465,5 +465,18 @@ namespace NJobProberClient { } // namespace NJobProberClient + + +// from ./ytlib/sequoia_client/public.h +namespace NSequoiaClient { + +//////////////////////////////////////////////////////////////////////////////// + + constexpr int SequoiaRetriableError = 6002; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NSequoiaClient + } // namespace NClusterErrorCodes } // namespace NYT diff --git a/yt/cpp/mapreduce/library/table_schema/arrow.cpp b/yt/cpp/mapreduce/library/table_schema/arrow.cpp deleted file mode 100644 index 73bce9bf19d..00000000000 --- a/yt/cpp/mapreduce/library/table_schema/arrow.cpp +++ /dev/null @@ -1,88 +0,0 @@ -#include "arrow.h" - -#include <yt/yt/core/misc/error.h> - -#include <library/cpp/yt/assert/assert.h> - -#include <contrib/libs/apache/arrow/cpp/src/arrow/api.h> -#include <contrib/libs/apache/arrow/cpp/src/arrow/io/api.h> -#include <contrib/libs/apache/arrow/cpp/src/arrow/io/memory.h> -#include <contrib/libs/apache/arrow/cpp/src/arrow/ipc/api.h> - -namespace NYT { - -namespace { - -//////////////////////////////////////////////////////////////////////////////// - -NTi::TTypePtr GetYTType(const std::shared_ptr<arrow::DataType>& arrowType) -{ - switch (arrowType->id()) { - case arrow::Type::type::BOOL: - return NTi::Bool(); - - case arrow::Type::type::UINT8: - return NTi::Uint8(); - case arrow::Type::type::UINT16: - return NTi::Uint16(); - case arrow::Type::type::UINT32: - return NTi::Uint32(); - case arrow::Type::type::UINT64: - return NTi::Uint64(); - - case arrow::Type::type::INT8: - return NTi::Int8(); - case arrow::Type::type::INT16: - return NTi::Int16(); - case arrow::Type::type::DATE32: - case arrow::Type::type::TIME32: - case arrow::Type::type::INT32: - return NTi::Int32(); - case arrow::Type::type::DATE64: - case arrow::Type::type::TIMESTAMP: - case arrow::Type::type::INT64: - case arrow::Type::type::TIME64: - return NTi::Int64(); - - case arrow::Type::type::HALF_FLOAT: - case arrow::Type::type::FLOAT: - return NTi::Float(); - case arrow::Type::type::DOUBLE: - return NTi::Double(); - - case arrow::Type::type::STRING: - case arrow::Type::type::BINARY: - case arrow::Type::type::FIXED_SIZE_BINARY: - return NTi::String(); - - case arrow::Type::type::LIST: - return NTi::List( - GetYTType(std::reinterpret_pointer_cast<arrow::ListType>(arrowType)->value_type())); - - case arrow::Type::type::MAP: - return NTi::Dict( - GetYTType(std::reinterpret_pointer_cast<arrow::MapType>(arrowType)->key_type()), - GetYTType(std::reinterpret_pointer_cast<arrow::MapType>(arrowType)->item_type())); - - default: - THROW_ERROR_EXCEPTION("Unsupported arrow type: %v", arrowType->ToString()); - } -} - -} // namespace - -//////////////////////////////////////////////////////////////////////////////// - -TTableSchema CreateYTTableSchemaFromArrowSchema(const std::shared_ptr<arrow::Schema>& arrowSchema) -{ - TTableSchema resultSchema; - for (const auto& field : arrowSchema->fields()) { - auto ytType = NTi::Optional(GetYTType(field->type())); - resultSchema.AddColumn(TColumnSchema().Name(TString(field->name())).TypeV3(ytType)); - } - return resultSchema; -} - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/cpp/mapreduce/library/table_schema/arrow.h b/yt/cpp/mapreduce/library/table_schema/arrow.h deleted file mode 100644 index bf06ab1f202..00000000000 --- a/yt/cpp/mapreduce/library/table_schema/arrow.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include <yt/cpp/mapreduce/interface/common.h> - -#include <contrib/libs/apache/arrow/cpp/src/arrow/api.h> - -namespace NYT { - -//////////////////////////////////////////////////////////////////////////////// - -TTableSchema CreateYTTableSchemaFromArrowSchema(const std::shared_ptr<arrow::Schema>& arrowSchema); - -//////////////////////////////////////////////////////////////////////////////// - -} // namespace NYT diff --git a/yt/cpp/mapreduce/library/table_schema/protobuf.cpp b/yt/cpp/mapreduce/library/table_schema/protobuf.cpp deleted file mode 100644 index 888da828e74..00000000000 --- a/yt/cpp/mapreduce/library/table_schema/protobuf.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "protobuf.h" diff --git a/yt/cpp/mapreduce/library/table_schema/ya.make b/yt/cpp/mapreduce/library/table_schema/ya.make deleted file mode 100644 index 06541201b52..00000000000 --- a/yt/cpp/mapreduce/library/table_schema/ya.make +++ /dev/null @@ -1,17 +0,0 @@ -LIBRARY() - -INCLUDE(${ARCADIA_ROOT}/yt/ya_cpp.make.inc) - -SRCS( - arrow.cpp - protobuf.h - protobuf.cpp -) - -PEERDIR( - yt/cpp/mapreduce/interface - - contrib/libs/apache/arrow -) - -END() diff --git a/yt/python/yt/common.py b/yt/python/yt/common.py index fb90e09651e..555adbb5e1b 100644 --- a/yt/python/yt/common.py +++ b/yt/python/yt/common.py @@ -359,6 +359,10 @@ class YtError(Exception): """Cross-cell "copy"/"move" command is explicitly disabled""" return self.contains_code(1002) + def is_sequoia_retriable_error(self): + """Probably lock conflict in Sequoia tables""" + return self.contains_code(6002) + class YtResponseError(YtError): """Represents an error in YT response.""" diff --git a/yt/python/yt/type_info/__init__.py b/yt/python/yt/type_info/__init__.py index aa7268843c5..33813771da8 100644 --- a/yt/python/yt/type_info/__init__.py +++ b/yt/python/yt/type_info/__init__.py @@ -7,5 +7,5 @@ from .typing import ( # noqa Double, String, Utf8, Yson, Json, Uuid, Date, Datetime, Timestamp, Interval, TzDate, TzDatetime, TzTimestamp, Void, Null, Optional, List, Tuple, Dict, Struct, Variant, Tagged, Decimal, EmptyTuple, EmptyStruct, - serialize_yson, deserialize_yson, + serialize_yson, deserialize_yson, deserialize_yson_v1, ) diff --git a/yt/python/yt/type_info/type_base.py b/yt/python/yt/type_info/type_base.py index 57b5452e296..11b75fd2a1a 100644 --- a/yt/python/yt/type_info/type_base.py +++ b/yt/python/yt/type_info/type_base.py @@ -82,6 +82,9 @@ class Primitive(Type): def to_yson_type(self): return self.yt_type_name + def to_yson_type_v1(self): + return self.yt_type_name_v1 + class Generic(six.with_metaclass(ABCMeta)): def __init__(self, name, yt_type_name=None): @@ -98,7 +101,7 @@ class Generic(six.with_metaclass(ABCMeta)): pass -def make_primitive_type(name, yt_type_name=None): +def make_primitive_type(name, yt_type_name=None, yt_type_name_v1=None): assert _is_utf8(name), "Name of primitive type must be UTF-8, got {}".format(_with_type(name)) assert yt_type_name is None or _is_utf8(yt_type_name), \ "YT type name of primitive type must be UTF-8, got {}".format(_with_type(name)) @@ -106,4 +109,11 @@ def make_primitive_type(name, yt_type_name=None): if yt_type_name is None: yt_type_name = name.lower() - return Primitive({"name": name, "yt_type_name": yt_type_name}) + if yt_type_name_v1 is None: + yt_type_name_v1 = name.lower() + + return Primitive({ + "name": name, + "yt_type_name": yt_type_name, + "yt_type_name_v1": yt_type_name_v1, + }) diff --git a/yt/python/yt/type_info/typing.py b/yt/python/yt/type_info/typing.py index 07cd5c189cb..0d9abc505f1 100644 --- a/yt/python/yt/type_info/typing.py +++ b/yt/python/yt/type_info/typing.py @@ -341,7 +341,9 @@ class _GenericDecimal(type_base.Generic): return self.__getitem__((type_["precision"], type_["scale"],)) -Bool = type_base.make_primitive_type("Bool") +Bool = type_base.make_primitive_type("Bool", yt_type_name_v1="boolean") +Yson = type_base.make_primitive_type("Yson", yt_type_name_v1="any") + Int8 = type_base.make_primitive_type("Int8") Uint8 = type_base.make_primitive_type("Uint8") Int16 = type_base.make_primitive_type("Int16") @@ -354,7 +356,6 @@ Float = type_base.make_primitive_type("Float") Double = type_base.make_primitive_type("Double") String = type_base.make_primitive_type("String") Utf8 = type_base.make_primitive_type("Utf8") -Yson = type_base.make_primitive_type("Yson") Json = type_base.make_primitive_type("Json") Uuid = type_base.make_primitive_type("Uuid") Date = type_base.make_primitive_type("Date") @@ -381,7 +382,8 @@ Decimal = _GenericDecimal("Decimal") EmptyTuple = Tuple.__getitem__(tuple()) EmptyStruct = Struct.__getitem__(tuple()) -PRIMITIVES = {type_.yt_type_name: type_ for type_ in locals().values() if isinstance(type_, type_base.Primitive)} +PRIMITIVES_V1 = {type_.yt_type_name_v1: type_ for type_ in locals().values() if isinstance(type_, type_base.Primitive)} +PRIMITIVES_V3 = {type_.yt_type_name: type_ for type_ in locals().values() if isinstance(type_, type_base.Primitive)} GENERICS = {type_.yt_type_name: type_ for type_ in locals().values() if isinstance(type_, type_base.Generic)} @@ -396,8 +398,8 @@ def _validate_contains(dict, key): def _parse_type(type_description): if isinstance(type_description, six.string_types): - _validate(type_description in PRIMITIVES, "unknown type \"{}\"".format(type_description)) - return PRIMITIVES[type_description] + _validate(type_description in PRIMITIVES_V3, "unknown type \"{}\"".format(type_description)) + return PRIMITIVES_V3[type_description] _validate(isinstance(type_description, dict), "type must be either a string or a map, got {}".format(type_base._with_type(type_description))) @@ -407,14 +409,23 @@ def _parse_type(type_description): type_name = type_description["type_name"] _validate(isinstance(type_name, six.string_types), "\"type_name\" must contain a string") - _validate(type_name in PRIMITIVES or type_name in GENERICS, "unknown type \"{}\"".format(type_name)) + _validate(type_name in PRIMITIVES_V3 or type_name in GENERICS, "unknown type \"{}\"".format(type_name)) - if type_name in PRIMITIVES: - return PRIMITIVES[type_name] + if type_name in PRIMITIVES_V3: + return PRIMITIVES_V3[type_name] return GENERICS[type_name].from_dict(type_description) +def _parse_type_v1(type_description, required): + if required: + _validate(isinstance(type_description, six.string_types), "\"type_description\" must be a string for v1 type") + _validate(type_description in PRIMITIVES_V1, "unknown type \"{}\"".format(type_description)) + return PRIMITIVES_V1[type_description] + else: + return Optional[_parse_type_v1(type_description, True)] + + def _check_serialization_available(): if not _TI_SERIALIZATION_AVAILABLE: raise ImportError("Module `yt.yson` is required to use type_info serialization. " @@ -441,3 +452,16 @@ def deserialize_yson(yson): return _parse_type(type_description) except (ValueError, yt.yson.YsonError) as e: six.raise_from(ValueError("deserialization failed: {}".format(e)), e) + + +def deserialize_yson_v1(yson, required): + _check_serialization_available() + + if type(yson) is six.text_type and six.PY3: + yson = yson.encode("utf-8") + + try: + type_description = yt.yson.loads(yson) + return _parse_type_v1(type_description, required) + except (ValueError, yt.yson.YsonError) as e: + six.raise_from(ValueError("deserialization failed: {}".format(e)), e) diff --git a/yt/yt/client/api/rpc_proxy/client_base.cpp b/yt/yt/client/api/rpc_proxy/client_base.cpp index 14079ff161a..c7f43164d9d 100644 --- a/yt/yt/client/api/rpc_proxy/client_base.cpp +++ b/yt/yt/client/api/rpc_proxy/client_base.cpp @@ -141,6 +141,13 @@ TFuture<ITransactionPtr> TClientBase::StartTransaction( ToProto(req->mutable_parent_id(), options.ParentId); } ToProto(req->mutable_prerequisite_transaction_ids(), options.PrerequisiteTransactionIds); + + if (options.ReplicateToMasterCellTags) { + ToProto( + req->mutable_replicate_to_master_cell_tags()->mutable_cell_tags(), + *options.ReplicateToMasterCellTags); + } + // XXX(sandello): Better? Remove these fields from the protocol at all? // COMPAT(kiselyovp): remove auto_abort from the protocol req->set_auto_abort(false); diff --git a/yt/yt/client/api/rpc_proxy/client_impl.cpp b/yt/yt/client/api/rpc_proxy/client_impl.cpp index a0d99186d2e..df1d4caada6 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/client_impl.cpp @@ -68,7 +68,7 @@ TClient::TClient( TConnectionPtr connection, const TClientOptions& clientOptions) : Connection_(std::move(connection)) - , RetryingChannel_(MaybeCreateRetryingChannel( + , RetryingChannel_(CreateSequoiaAwareRetryingChannel( CreateCredentialsInjectingChannel( Connection_->CreateChannel(false), clientOptions), @@ -103,19 +103,18 @@ void TClient::Terminate() //////////////////////////////////////////////////////////////////////////////// -IChannelPtr TClient::MaybeCreateRetryingChannel(NRpc::IChannelPtr channel, bool retryProxyBanned) const +IChannelPtr TClient::CreateSequoiaAwareRetryingChannel(NRpc::IChannelPtr channel, bool retryProxyBanned) const { const auto& config = Connection_->GetConfig(); - if (config->EnableRetries) { - return NRpc::CreateRetryingChannel( - config->RetryingChannel, - std::move(channel), - BIND([=] (const TError& error) { - return IsRetriableError(error, retryProxyBanned); - })); - } else { - return channel; - } + bool retrySequoiaErrorsOnly = !config->EnableRetries; + // NB: even if client's retries are disabled Sequoia transient failures are + // still retriable. See IsRetriableError(). + return NRpc::CreateRetryingChannel( + config->RetryingChannel, + std::move(channel), + BIND([=] (const TError& error) { + return IsRetriableError(error, retryProxyBanned, retrySequoiaErrorsOnly); + })); } IChannelPtr TClient::CreateNonRetryingChannelByAddress(const TString& address) const @@ -153,7 +152,7 @@ IChannelPtr TClient::CreateNonRetryingStickyChannel() const IChannelPtr TClient::WrapStickyChannelIntoRetrying(IChannelPtr underlying) const { - return MaybeCreateRetryingChannel( + return CreateSequoiaAwareRetryingChannel( std::move(underlying), /*retryProxyBanned*/ false); } diff --git a/yt/yt/client/api/rpc_proxy/client_impl.h b/yt/yt/client/api/rpc_proxy/client_impl.h index b2854eff49d..3c66f30c875 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.h +++ b/yt/yt/client/api/rpc_proxy/client_impl.h @@ -548,7 +548,7 @@ private: NTransactionClient::ITimestampProviderPtr CreateTimestampProvider() const; - NRpc::IChannelPtr MaybeCreateRetryingChannel(NRpc::IChannelPtr channel, bool retryProxyBanned) const; + NRpc::IChannelPtr CreateSequoiaAwareRetryingChannel(NRpc::IChannelPtr channel, bool retryProxyBanned) const; // Returns an RPC channel to use for API calls to the particular address (e.g.: AttachTransaction). // The channel is non-retrying, so should be wrapped into retrying channel on demand. NRpc::IChannelPtr CreateNonRetryingChannelByAddress(const TString& address) const; diff --git a/yt/yt/client/api/rpc_proxy/helpers.cpp b/yt/yt/client/api/rpc_proxy/helpers.cpp index 1dff960561c..ff4071bf272 100644 --- a/yt/yt/client/api/rpc_proxy/helpers.cpp +++ b/yt/yt/client/api/rpc_proxy/helpers.cpp @@ -3,6 +3,8 @@ #include <yt/yt/client/api/rowset.h> #include <yt/yt/client/api/table_client.h> +#include <yt/yt/client/sequoia_client/public.h> + #include <yt/yt/client/table_client/columnar_statistics.h> #include <yt/yt/client/table_client/column_sort_schema.h> #include <yt/yt/client/table_client/logical_type.h> @@ -1888,8 +1890,20 @@ bool IsDynamicTableRetriableError(const TError& error) error.FindMatching(NTabletClient::EErrorCode::NoSuchTablet); } -bool IsRetriableError(const TError& error, bool retryProxyBanned) +bool IsRetriableError(const TError& error, bool retryProxyBanned, bool retrySequoiaErrorsOnly) { + // For now transient Sequoia failures are always retriable even if client's + // retries are disabled. + // TODO(kvk1920): consider to make a separate flag "EnableSequoiaRetries" + // for this. + if (error.FindMatching(NSequoiaClient::EErrorCode::SequoiaRetriableError)) { + return true; + } + + if (retrySequoiaErrorsOnly) { + return false; + } + if (error.FindMatching(NRpcProxy::EErrorCode::ProxyBanned) || error.FindMatching(NRpc::EErrorCode::PeerBanned)) { diff --git a/yt/yt/client/api/rpc_proxy/helpers.h b/yt/yt/client/api/rpc_proxy/helpers.h index 722e47380e9..9d398923b03 100644 --- a/yt/yt/client/api/rpc_proxy/helpers.h +++ b/yt/yt/client/api/rpc_proxy/helpers.h @@ -275,7 +275,10 @@ NQueryTrackerClient::EQueryState ConvertQueryStateFromProto( //////////////////////////////////////////////////////////////////////////////// -bool IsRetriableError(const TError& error, bool retryProxyBanned = true); +bool IsRetriableError( + const TError& error, + bool retryProxyBanned = true, + bool retrySequoiaErrorsOnly = false); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/api/rpc_proxy/public.h b/yt/yt/client/api/rpc_proxy/public.h index 33f580a444a..fd74de1fef9 100644 --- a/yt/yt/client/api/rpc_proxy/public.h +++ b/yt/yt/client/api/rpc_proxy/public.h @@ -27,6 +27,7 @@ YT_DEFINE_ERROR_ENUM( DEFINE_ENUM(ERpcProxyFeature, ((GetInSyncWithoutKeys)(0)) + ((WideLocks) (1)) ); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/api/rpc_proxy/transaction_impl.cpp b/yt/yt/client/api/rpc_proxy/transaction_impl.cpp index 7089f413e18..28027efa251 100644 --- a/yt/yt/client/api/rpc_proxy/transaction_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/transaction_impl.cpp @@ -396,14 +396,16 @@ void TTransaction::ModifyRows( rows.reserve(modifications.Size()); bool usedStrongLocks = false; + bool usedWideLocks = false; for (const auto& modification : modifications) { auto mask = modification.Locks; + usedWideLocks |= mask.GetSize() > TLegacyLockMask::MaxCount; + if (usedWideLocks) { + break; + } + for (int index = 0; index < TLegacyLockMask::MaxCount; ++index) { - if (mask.Get(index) > MaxOldLockType) { - THROW_ERROR_EXCEPTION("New locks are not supported in RPC client yet") - << TErrorAttribute("lock_index", index) - << TErrorAttribute("lock_type", mask.Get(index)); - } + usedWideLocks |= mask.Get(index) > MaxOldLockType; usedStrongLocks |= mask.Get(index) == ELockType::SharedStrong; } } @@ -411,14 +413,19 @@ void TTransaction::ModifyRows( if (usedStrongLocks) { req->Header().set_protocol_version_minor(YTRpcModifyRowsStrongLocksVersion); } + if (usedWideLocks) { + req->RequireServerFeature(ERpcProxyFeature::WideLocks); + } for (const auto& modification : modifications) { rows.emplace_back(modification.Row); req->add_row_modification_types(static_cast<NProto::ERowModificationType>(modification.Type)); - if (usedStrongLocks) { + if (usedWideLocks) { + ToProto(req->add_row_locks(), modification.Locks); + } else if (usedStrongLocks) { auto locks = modification.Locks; YT_VERIFY(!locks.HasNewLocks()); - req->add_row_locks(locks.ToLegacyMask().GetBitmap()); + req->add_row_legacy_locks(locks.ToLegacyMask().GetBitmap()); } else { TLegacyLockBitmap bitmap = 0; for (int index = 0; index < TLegacyLockMask::MaxCount; ++index) { @@ -426,7 +433,7 @@ void TTransaction::ModifyRows( bitmap |= 1u << index; } } - req->add_row_read_locks(bitmap); + req->add_row_legacy_read_locks(bitmap); } } diff --git a/yt/yt/client/chaos_client/replication_card.cpp b/yt/yt/client/chaos_client/replication_card.cpp index 69076d4cf7c..5c32290d761 100644 --- a/yt/yt/client/chaos_client/replication_card.cpp +++ b/yt/yt/client/chaos_client/replication_card.cpp @@ -1,5 +1,7 @@ #include "replication_card.h" +#include <yt/yt/client/transaction_client/helpers.h> + #include <yt/yt/core/misc/guid.h> #include <yt/yt/core/misc/serialize.h> @@ -680,6 +682,159 @@ bool IsReplicaLocationValid( return replica->ReplicaPath == tablePath && replica->ClusterName == clusterName; } +TReplicationProgress BuildMaxProgress( + const TReplicationProgress& progress, + const TReplicationProgress& other) +{ + if (progress.Segments.empty()) { + return other; + } + if (other.Segments.empty()) { + return progress; + } + + TReplicationProgress result; + + auto progressIt = progress.Segments.begin(); + auto otherIt = other.Segments.begin(); + auto progressEnd = progress.Segments.end(); + auto otherEnd = other.Segments.end(); + + auto progressTimestamp = NullTimestamp; + auto otherTimestamp = NullTimestamp; + + bool upperKeySelected = false; + + auto tryAppendSegment =[&result] (TUnversionedOwningRow row, TTimestamp timestamp) { + if (result.Segments.empty() || result.Segments.back().Timestamp != timestamp) { + result.Segments.push_back({std::move(row), timestamp}); + } + }; + + while (progressIt != progressEnd || otherIt != otherEnd) { + int cmpResult; + + if (otherIt == otherEnd) { + cmpResult = -1; + if (!upperKeySelected && CompareRows(progressIt->LowerKey, other.UpperKey) >= 0) { + upperKeySelected = true; + otherTimestamp = NullTimestamp; + tryAppendSegment(other.UpperKey, progressTimestamp); + continue; + } + } else if (progressIt == progressEnd) { + cmpResult = 1; + if (!upperKeySelected && CompareRows(otherIt->LowerKey, progress.UpperKey) >= 0) { + upperKeySelected = true; + progressTimestamp = NullTimestamp; + tryAppendSegment(progress.UpperKey, otherTimestamp); + continue; + } + } else { + cmpResult = CompareRows(progressIt->LowerKey, otherIt->LowerKey); + } + + TUnversionedOwningRow lowerKey; + if (cmpResult < 0) { + progressTimestamp = progressIt->Timestamp; + lowerKey = progressIt->LowerKey; + ++progressIt; + } else if (cmpResult > 0) { + otherTimestamp = otherIt->Timestamp; + lowerKey = otherIt->LowerKey; + ++otherIt; + } else { + progressTimestamp = progressIt->Timestamp; + otherTimestamp = otherIt->Timestamp; + lowerKey = progressIt->LowerKey; + ++progressIt; + ++otherIt; + } + + tryAppendSegment(std::move(lowerKey), std::max(progressTimestamp, otherTimestamp)); + } + + auto cmpResult = CompareRows(progress.UpperKey, other.UpperKey); + result.UpperKey = cmpResult > 0 ? progress.UpperKey : other.UpperKey; + + if (!upperKeySelected) { + if (cmpResult > 0) { + tryAppendSegment(other.UpperKey, progressTimestamp); + } else if (cmpResult < 0) { + tryAppendSegment(progress.UpperKey, otherTimestamp); + } + } + + return result; +} + +TDuration ComputeReplicationProgressLag( + const TReplicationProgress& syncProgress, + const TReplicationProgress& replicaProgress) +{ + if (syncProgress.Segments.empty()) { + return TDuration::Zero(); + } + auto timestampDiff = [] (TTimestamp loTimestamp, TTimestamp hiTimestamp) { + if (loTimestamp >= hiTimestamp) { + return TDuration::Zero(); + } + return TimestampDiffToDuration(loTimestamp, hiTimestamp).first; + }; + + if (replicaProgress.Segments.empty()) { + return timestampDiff(GetReplicationProgressMaxTimestamp(syncProgress), NullTimestamp); + } + + auto syncIt = syncProgress.Segments.begin(); + auto replicaIt = replicaProgress.Segments.begin(); + auto syncEnd = syncProgress.Segments.end(); + auto replicaEnd = replicaProgress.Segments.end(); + + auto syncSegmentTimestamp = NullTimestamp; + auto replicaSegmentTimestamp = NullTimestamp; + auto lag = TDuration::Zero(); + + while (syncIt != syncEnd || replicaIt != replicaEnd) { + int cmpResult; + if (syncIt == syncEnd) { + cmpResult = -1; + if (CompareRows(replicaIt->LowerKey, syncProgress.UpperKey) >= 0) { + syncSegmentTimestamp = NullTimestamp; + } + } else if (replicaIt == replicaEnd) { + cmpResult = 1; + if (CompareRows(syncIt->LowerKey, replicaProgress.UpperKey) >= 0) { + replicaSegmentTimestamp = NullTimestamp; + } + } else { + cmpResult = CompareRows(replicaIt->LowerKey, syncIt->LowerKey); + } + + if (cmpResult > 0) { + syncSegmentTimestamp = syncIt->Timestamp; + ++syncIt; + } else if (cmpResult < 0) { + replicaSegmentTimestamp = replicaIt->Timestamp; + ++replicaIt; + } else { + syncSegmentTimestamp = syncIt->Timestamp; + replicaSegmentTimestamp = replicaIt->Timestamp; + ++replicaIt; + ++syncIt; + } + + lag = std::max(lag, timestampDiff(replicaSegmentTimestamp, syncSegmentTimestamp)); + } + + if (CompareRows(syncProgress.UpperKey, replicaProgress.UpperKey) > 0) { + lag = std::max(lag, timestampDiff(NullTimestamp, syncProgress.Segments.back().Timestamp)); + } + + return lag; +} + //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NChaosClient + diff --git a/yt/yt/client/chaos_client/replication_card.h b/yt/yt/client/chaos_client/replication_card.h index aba106b65ff..307622af075 100644 --- a/yt/yt/client/chaos_client/replication_card.h +++ b/yt/yt/client/chaos_client/replication_card.h @@ -184,6 +184,12 @@ bool IsReplicaLocationValid( const NYPath::TYPath& tablePath, const TString& clusterName); +TReplicationProgress BuildMaxProgress(const TReplicationProgress& progress1, const TReplicationProgress& progress2); + +TDuration ComputeReplicationProgressLag( + const TReplicationProgress& maxProgress, + const TReplicationProgress& replicaProgress); + //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NChaosClient diff --git a/yt/yt/client/sequoia_client/public.h b/yt/yt/client/sequoia_client/public.h new file mode 100644 index 00000000000..5c213ebd9f1 --- /dev/null +++ b/yt/yt/client/sequoia_client/public.h @@ -0,0 +1,17 @@ +#pragma once + +#include <yt/yt/core/misc/error_code.h> + +namespace NYT::NSequoiaClient { + +//////////////////////////////////////////////////////////////////////////////// + +YT_DEFINE_ERROR_ENUM( + ((SequoiaClientNotReady) (6000)) + ((SequoiaTableCorrupted) (6001)) + ((SequoiaRetriableError) (6002)) +); + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT::NSequoiaClient diff --git a/yt/yt/client/table_client/adapters.cpp b/yt/yt/client/table_client/adapters.cpp index 5001c2b36f7..c64915f9d68 100644 --- a/yt/yt/client/table_client/adapters.cpp +++ b/yt/yt/client/table_client/adapters.cpp @@ -118,33 +118,43 @@ void PipeReaderToWriter( while (auto batch = reader->Read(readOptions)) { yielder.TryYield(); - if (batch->IsEmpty()) { - WaitFor(reader->GetReadyEvent()) - .ThrowOnError(); - continue; - } + TSharedRange<TUnversionedRow> rows; + + try { + if (batch->IsEmpty()) { + WaitFor(reader->GetReadyEvent()) + .ThrowOnError(); + continue; + } - auto rows = batch->MaterializeRows(); + rows = batch->MaterializeRows(); - if (options.ValidateValues) { - for (auto row : rows) { - for (const auto& value : row) { - ValidateStaticValue(value); + if (options.ValidateValues) { + for (auto row : rows) { + for (const auto& value : row) { + ValidateStaticValue(value); + } } } - } - if (options.Throttler) { - i64 dataWeight = 0; - for (auto row : rows) { - dataWeight += GetDataWeight(row); + if (options.Throttler) { + i64 dataWeight = 0; + for (auto row : rows) { + dataWeight += GetDataWeight(row); + } + WaitFor(options.Throttler->Throttle(dataWeight)) + .ThrowOnError(); } - WaitFor(options.Throttler->Throttle(dataWeight)) - .ThrowOnError(); - } - if (!rows.empty() && options.PipeDelay) { - TDelayedExecutor::WaitForDuration(options.PipeDelay); + if (!rows.empty() && options.PipeDelay) { + TDelayedExecutor::WaitForDuration(options.PipeDelay); + } + } catch (const std::exception& ex) { + if (options.ReaderErrorWrapper) { + THROW_ERROR(options.ReaderErrorWrapper(ex)); + } else { + throw; + } } if (!writer->Write(rows)) { diff --git a/yt/yt/client/table_client/adapters.h b/yt/yt/client/table_client/adapters.h index 4625398b415..bbb568582b9 100644 --- a/yt/yt/client/table_client/adapters.h +++ b/yt/yt/client/table_client/adapters.h @@ -27,6 +27,7 @@ struct TPipeReaderToWriterOptions i64 BufferDataWeight = 16_MB; bool ValidateValues = false; NConcurrency::IThroughputThrottlerPtr Throttler; + std::function<TError(const TError& readerError)> ReaderErrorWrapper; // Used only for testing. TDuration PipeDelay; }; diff --git a/yt/yt/client/table_client/public.h b/yt/yt/client/table_client/public.h index 4de8d2f2b7a..73b6c8dca64 100644 --- a/yt/yt/client/table_client/public.h +++ b/yt/yt/client/table_client/public.h @@ -4,6 +4,8 @@ #include <yt/yt/client/cypress_client/public.h> +#include <yt/yt/client/tablet_client/public.h> + #include <yt/yt/client/transaction_client/public.h> #include <yt/yt/core/misc/range.h> diff --git a/yt/yt/client/table_client/schema.cpp b/yt/yt/client/table_client/schema.cpp index da847dcc23a..672cb8a5207 100644 --- a/yt/yt/client/table_client/schema.cpp +++ b/yt/yt/client/table_client/schema.cpp @@ -22,6 +22,8 @@ #include <yt/yt_proto/yt/client/table_chunk_format/proto/chunk_meta.pb.h> #include <yt/yt_proto/yt/client/table_chunk_format/proto/wire_protocol.pb.h> +#include <yt/yt_proto/yt/client/tablet_client/proto/lock_mask.pb.h> + namespace NYT::NTableClient { using namespace NChunkClient; @@ -112,6 +114,37 @@ TLockMask MaxMask(TLockMask lhs, TLockMask rhs) return lhs; } +void ToProto(NTabletClient::NProto::TLockMask* protoLockMask, const TLockMask& lockMask) +{ + auto size = lockMask.GetSize(); + YT_VERIFY(size <= TLockMask::MaxSize); + + protoLockMask->set_size(size); + + const auto& bitmap = lockMask.GetBitmap(); + auto wordCount = DivCeil(size, TLockMask::LocksPerWord); + YT_VERIFY(std::ssize(bitmap) >= wordCount); + + protoLockMask->clear_bitmap(); + for (int index = 0; index < wordCount; ++index) { + protoLockMask->add_bitmap(bitmap[index]); + } +} + +void FromProto(TLockMask* lockMask, const NTabletClient::NProto::TLockMask& protoLockMask) +{ + auto size = protoLockMask.size(); + auto wordCount = DivCeil<int>(size, TLockMask::LocksPerWord); + + TLockBitmap bitmap; + bitmap.reserve(wordCount); + for (int index = 0; index < wordCount; ++index) { + bitmap.push_back(protoLockMask.bitmap(index)); + } + + *lockMask = TLockMask(bitmap, size); +} + //////////////////////////////////////////////////////////////////////////////// TColumnSchema::TColumnSchema() diff --git a/yt/yt/client/table_client/schema.h b/yt/yt/client/table_client/schema.h index 91729bc8ccb..a556a0637dc 100644 --- a/yt/yt/client/table_client/schema.h +++ b/yt/yt/client/table_client/schema.h @@ -107,6 +107,9 @@ bool operator == (const TLockMask& lhs, const TLockMask& rhs); TLockMask MaxMask(TLockMask lhs, TLockMask rhs); +void ToProto(NTabletClient::NProto::TLockMask* protoLockMask, const TLockMask& lockMask); +void FromProto(TLockMask* lockMask, const NTabletClient::NProto::TLockMask& protoLockMask); + //////////////////////////////////////////////////////////////////////////////// class TColumnSchema diff --git a/yt/yt/client/tablet_client/public.h b/yt/yt/client/tablet_client/public.h index 5d0fe078228..e2bc658175a 100644 --- a/yt/yt/client/tablet_client/public.h +++ b/yt/yt/client/tablet_client/public.h @@ -10,6 +10,12 @@ namespace NYT::NTabletClient { //////////////////////////////////////////////////////////////////////////////// +namespace NProto { + +class TLockMask; + +} // namespace NProto + DEFINE_ENUM(ETabletState, // Individual states ((Mounting) (0)) diff --git a/yt/yt/client/unittests/replication_progress_ut.cpp b/yt/yt/client/unittests/replication_progress_ut.cpp index d9315fe015a..8637c828849 100644 --- a/yt/yt/client/unittests/replication_progress_ut.cpp +++ b/yt/yt/client/unittests/replication_progress_ut.cpp @@ -534,6 +534,210 @@ INSTANTIATE_TEST_SUITE_P( "{Segments: [<[], 1>, ..., <[0#5], 6>, <[0#6], 7>, ...], UpperKey: [0#<Max>]}") )); +//////////////////////////////////////////////////////////////////////////////// + +class TReplicationProgressComputeReplicationProgressLag + : public ::testing::Test + , public ::testing::WithParamInterface<std::tuple< + const char*, + const char*, + TDuration>> +{ }; + +TEST_P(TReplicationProgressComputeReplicationProgressLag, Simple) +{ + const auto& params = GetParam(); + auto maxProgress = ConvertTo<TReplicationProgress>(TYsonStringBuf(std::get<0>(params))); + auto replicaProgress = ConvertTo<TReplicationProgress>(TYsonStringBuf(std::get<1>(params))); + auto expected = std::get<2>(params); + + auto result = ComputeReplicationProgressLag(maxProgress, replicaProgress); + EXPECT_EQ(result, expected) + << "max_progress: " << std::get<0>(params) << std::endl + << "replica_progress: " << std::get<1>(params) << std::endl + << "expected: " << ToString(expected) << std::endl + << "actual: " << ToString(result) << std::endl; +} + +INSTANTIATE_TEST_SUITE_P( + TReplicationProgressComputeReplicationProgressLag, + TReplicationProgressComputeReplicationProgressLag, + ::testing::Values( + std::tuple( + "{segments=[{lower_key=[];timestamp=1}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=1}];upper_key=[<type=max>#]}", + TDuration::Zero()), + std::tuple( + "{segments=[{lower_key=[];timestamp=1837578930916163584};{lower_key=[1];timestamp=1837578941653581824}];" + "upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=1837578930916163584}];upper_key=[<type=max>#]}", + TDuration::Seconds(9)), + std::tuple( + "{segments=[{lower_key=[];timestamp=2147483648};{lower_key=[2];timestamp=5368709120}];" + "upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=0};{lower_key=[1];timestamp=3221225472}];upper_key=[<type=max>#]}", + TDuration::Seconds(1)), + std::tuple( + "{segments=[{lower_key=[];timestamp=0}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[1];timestamp=3221225472}];" + "upper_key=[<type=max>#]}", + TDuration::Zero()), + std::tuple( + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[1];timestamp=3221225472}];" + "upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=0}];upper_key=[<type=max>#]}", + TDuration::Seconds(2)), + std::tuple( + "{segments=[{lower_key=[];timestamp=3221225472}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[1];timestamp=4294967296};" + "{lower_key=[2];timestamp=1073741824}];upper_key=[<type=max>#]}", + TDuration::Seconds(1)), + std::tuple( + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[1];timestamp=5368709120};" + "{lower_key=[2];timestamp=1073741824}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=3221225472}];upper_key=[<type=max>#]}", + TDuration::Seconds(1)), + std::tuple( + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[1];timestamp=3221225472};" + "{lower_key=[2];timestamp=1073741824}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=3221225472};{lower_key=[1];timestamp=1073741824};" + "{lower_key=[2];timestamp=3221225472}];upper_key=[<type=max>#]}", + TDuration::Seconds(1)), + std::tuple( + "{segments=[{lower_key=[];timestamp=2147483648};{lower_key=[1];timestamp=4294967296};" + "{lower_key=[3];timestamp=8589934592};{lower_key=[5];timestamp=12884901888}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=4294967296};{lower_key=[2];timestamp=6442450944};" + "{lower_key=[4];timestamp=10737418240};{lower_key=[6];timestamp=15032385536}];" + "upper_key=[<type=max>#]}", + TDuration::Seconds(1)), + std::tuple( + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[1];timestamp=2147483648};" + "{lower_key=[2];timestamp=3221225472}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[1];timestamp=2147483648};{lower_key=[2];timestamp=3221225472}];upper_key=[3]}", + TDuration::Seconds(2)), + std::tuple( + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[2];timestamp=3221225472}];" + "upper_key=[<type=max>#]}", + "{segments=[{lower_key=[1];timestamp=2147483648};{lower_key=[2];timestamp=3221225472}];upper_key=[3]}", + TDuration::Seconds(2)), + std::tuple( + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[1];timestamp=2147483648};" + "{lower_key=[2];timestamp=3221225472}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[1];timestamp=3221225472}];upper_key=[3]}", + TDuration::Seconds(2)), + std::tuple( + "{segments=[{lower_key=[];timestamp=3221225472};{lower_key=[2];timestamp=2147483648}];" + "upper_key=[<type=max>#]}", + "{segments=[{lower_key=[1];timestamp=3221225472};{lower_key=[2];timestamp=2147483648}];" + "upper_key=[<type=max>#]}", + TDuration::Seconds(2)) +)); + +//////////////////////////////////////////////////////////////////////////////// + +class TReplicationProgressBuildMax + : public ::testing::Test + , public ::testing::WithParamInterface<std::tuple< + const char*, + const char*, + const char*>> +{ }; + +TEST_P(TReplicationProgressBuildMax, Simple) +{ + const auto& params = GetParam(); + auto progress = ConvertTo<TReplicationProgress>(TYsonStringBuf(std::get<0>(params))); + auto other = ConvertTo<TReplicationProgress>(TYsonStringBuf(std::get<1>(params))); + auto expected = ConvertTo<TReplicationProgress>(TYsonStringBuf(std::get<2>(params))); + + auto result = BuildMaxProgress(progress, other); + EXPECT_TRUE(IsReplicationProgressEqual(result, expected)) + << "progress: " << std::get<0>(params) << std::endl + << "other: " << std::get<1>(params) << std::endl + << "expected: " << ToString(expected) << std::endl + << "actual: " << ToString(result) << std::endl; + + auto result2 = BuildMaxProgress(other, progress); + EXPECT_TRUE(IsReplicationProgressEqual(result2, expected)) + << "progress: " << std::get<1>(params) << std::endl + << "other: " << std::get<0>(params) << std::endl + << "expected: " << ToString(expected) << std::endl + << "actual: " << ToString(result2) << std::endl; +} + +INSTANTIATE_TEST_SUITE_P( + TReplicationProgressBuildMax, + TReplicationProgressBuildMax, + ::testing::Values( + std::tuple( + "{segments=[{lower_key=[];timestamp=1}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=1}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=1}];upper_key=[<type=max>#]}"), + std::tuple( + "{segments=[{lower_key=[];timestamp=1};{lower_key=[1];timestamp=2}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=1}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=1};{lower_key=[1];timestamp=2}];upper_key=[<type=max>#]}"), + std::tuple( + "{segments=[{lower_key=[];timestamp=1};{lower_key=[2];timestamp=2}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=0};{lower_key=[1];timestamp=3}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=1};{lower_key=[1];timestamp=3}];upper_key=[<type=max>#]}"), + std::tuple( + "{segments=[{lower_key=[];timestamp=0}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=1};{lower_key=[1];timestamp=3}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=1};{lower_key=[1];timestamp=3}];upper_key=[<type=max>#]}"), + std::tuple( + "{segments=[{lower_key=[];timestamp=2}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=1};{lower_key=[1];timestamp=3};{lower_key=[2];timestamp=1}];" + "upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=2};{lower_key=[1];timestamp=3};{lower_key=[2];timestamp=2}];" + "upper_key=[<type=max>#]}"), + std::tuple( + "{segments=[{lower_key=[];timestamp=1};{lower_key=[1];timestamp=2};{lower_key=[2];timestamp=1}];" + "upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=2};{lower_key=[1];timestamp=1};{lower_key=[2];timestamp=2}];" + "upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=2}];upper_key=[<type=max>#]}"), + std::tuple( + "{segments=[{lower_key=[];timestamp=1};{lower_key=[1];timestamp=2};{lower_key=[3];timestamp=4};" + "{lower_key=[5];timestamp=6}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=2};{lower_key=[2];timestamp=3};{lower_key=[4];timestamp=5};" + "{lower_key=[6];timestamp=7}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=2};{lower_key=[2];timestamp=3};{lower_key=[3];timestamp=4};" + "{lower_key=[4];timestamp=5};{lower_key=[5];timestamp=6};{lower_key=[6];timestamp=7}];" + "upper_key=[<type=max>#]}"), + std::tuple( + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[1];timestamp=2147483648};" + "{lower_key=[2];timestamp=3221225472}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[1];timestamp=2147483648};{lower_key=[2];timestamp=3221225472}];upper_key=[3]}", + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[1];timestamp=2147483648};" + "{lower_key=[2];timestamp=3221225472}];upper_key=[<type=max>#]}"), + std::tuple( + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[2];timestamp=3221225472}];" + "upper_key=[<type=max>#]}", + "{segments=[{lower_key=[1];timestamp=2147483648};{lower_key=[2];timestamp=3221225472}];upper_key=[3]}", + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[1];timestamp=2147483648};" + "{lower_key=[2];timestamp=3221225472}];upper_key=[<type=max>#]}"), + std::tuple( + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[1];timestamp=2147483648};" + "{lower_key=[2];timestamp=1073741824}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[1];timestamp=3221225472}];upper_key=[3]}", + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[1];timestamp=3221225472};" + "{lower_key=[3];timestamp=1073741824}];upper_key=[<type=max>#]}"), + std::tuple( + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[1];timestamp=2147483648};" + "{lower_key=[2];timestamp=1073741824};{lower_key=[4];timestamp=2147483648}];upper_key=[<type=max>#]}", + "{segments=[{lower_key=[1];timestamp=3221225472}];upper_key=[3]}", + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[1];timestamp=3221225472};" + "{lower_key=[3];timestamp=1073741824};{lower_key=[4];timestamp=2147483648}];upper_key=[<type=max>#]}"), + std::tuple( + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[2];timestamp=3221225472}];" + "upper_key=[<type=max>#]}", + "{segments=[{lower_key=[1];timestamp=1073741824};{lower_key=[2];timestamp=3221225472}];" + "upper_key=[<type=max>#]}", + "{segments=[{lower_key=[];timestamp=1073741824};{lower_key=[2];timestamp=3221225472}];" + "upper_key=[<type=max>#]}") + +)); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/concurrency/fiber.cpp b/yt/yt/core/concurrency/fiber.cpp index 6da8552aef2..9bdb6fb35c9 100644 --- a/yt/yt/core/concurrency/fiber.cpp +++ b/yt/yt/core/concurrency/fiber.cpp @@ -4,6 +4,7 @@ #include <yt/yt/core/profiling/timing.h> +#include <yt/yt/core/misc/intrusive_mpsc_stack.h> #include <yt/yt/core/misc/singleton.h> #include <yt/yt/core/misc/finally.h> @@ -113,6 +114,9 @@ private: class TFiberRegistry { + template <class Tag> + using TFiberStack = TIntrusiveMPSCStack<TFiber, Tag>; + public: //! Do not rename, change the signature, or drop Y_NO_INLINE. //! Used in devtools/gdb/yt_fibers_printer.py. @@ -121,43 +125,107 @@ public: return LeakySingleton<TFiberRegistry>(); } - TFiber::TCookie Register(TFiber* fiber) + void Register(TFiber* fiber) { - auto guard = Guard(Lock_); - return Fibers_.insert(Fibers_.begin(), fiber); + RegisterQueue_.Push(fiber); + + if (auto guard = TTryGuard(Lock_)) { + GuardedProcessQueues(); + } } - void Unregister(TFiber::TCookie cookie) + void Unregister(TFiber* fiber) { - auto guard = Guard(Lock_); - Fibers_.erase(cookie); + UnregisterQueue_.Push(fiber); + + if (auto guard = TTryGuard(Lock_)) { + GuardedProcessQueues(); + } } - std::vector<TFiberPtr> List() + void ReadFibers(TFunctionView<void(TFiber::TFiberList&)> callback) { auto guard = Guard(Lock_); - std::vector<TFiberPtr> fibers; - for (const auto& fiber : Fibers_) { - fibers.push_back(fiber); - } - return fibers; + + GuardedProcessQueues(); + + callback(Fibers_); + + GuardedProcessQueues(); + } + + ~TFiberRegistry() + { + GuardedProcessQueues(); } private: + TFiberStack<NDetail::TFiberRegisterTag> RegisterQueue_; + TFiberStack<NDetail::TFiberUnregisterTag> UnregisterQueue_; + NThreading::TForkAwareSpinLock Lock_; - std::list<TFiber*> Fibers_; + TFiber::TFiberList Fibers_; + + void GuardedProcessQueues() + { + Fibers_.Append(RegisterQueue_.PopAll()); + + auto toUnregister = UnregisterQueue_.PopAll(); + + while (auto fiber = toUnregister.PopBack()) { + fiber->UnregisterAndDelete(); + } + + // NB: Around this line guard is released. We do not properly double check + // if queues are actually empty after this. + // We are okay with this since we expect to have occasional calls of this method + // which would unstuck most of the fibers. In dtor of this singleton we + // release the last batch of stuck fibers. + }; + + void DebugPrint() + { + Cerr << "Debug print begin\n"; + Cerr << "---------------------------------------------------------------" << '\n'; + for (auto* iter = Fibers_.Begin(); iter != Fibers_.End(); iter = iter->Next) { + auto* fiber = static_cast<TFiber*>(iter); + auto* regNode = static_cast<TIntrusiveNode<TFiber, NDetail::TFiberRegisterTag>*>(fiber); + auto* delNode = static_cast<TIntrusiveNode<TFiber, NDetail::TFiberUnregisterTag>*>(fiber); + + Cerr << Format("Fiber node at %v. Next is %v, Prev is %v", iter, iter->Next, iter->Prev) << '\n'; + Cerr << Format("Fiber address after cast is %v", fiber) << '\n'; + Cerr << Format("Fiber registration queue status: Next: %v, Prev: %v", regNode->Next, regNode->Prev) << '\n'; + // NB: Reading deletion queue is data race. Don't do this under tsan. + Cerr << Format("Fiber deletion queue status: Next: %v, Prev: %v", delNode->Next, delNode->Prev) << '\n'; + Cerr << "---------------------------------------------------------------" << '\n'; + } + Cerr << "Debug print end\n"; + } }; //////////////////////////////////////////////////////////////////////////////// +TFiber* TFiber::CreateFiber(EExecutionStackKind stackKind) +{ + return new TFiber(stackKind); +} + +void TFiber::ReleaseFiber(TFiber* fiber) +{ + YT_VERIFY(fiber); + fiber->SetFinished(); + fiber->Clear(); + TFiberRegistry::Get()->Unregister(fiber); +} + TFiber::TFiber(EExecutionStackKind stackKind) : Stack_(CreateExecutionStack(stackKind)) - , RegistryCookie_(TFiberRegistry::Get()->Register(this)) , MachineContext_({ this, TArrayRef(static_cast<char*>(Stack_->GetStack()), Stack_->GetSize()), }) { + TFiberRegistry::Get()->Register(this); TFiberProfiler::Get()->OnFiberCreated(); TFiberProfiler::Get()->OnStackAllocated(Stack_->GetSize()); } @@ -166,7 +234,6 @@ TFiber::~TFiber() { YT_VERIFY(GetState() == EFiberState::Finished); TFiberProfiler::Get()->OnStackFreed(Stack_->GetSize()); - TFiberRegistry::Get()->Unregister(RegistryCookie_); } bool TFiber::CheckFreeStackSpace(size_t space) const @@ -222,7 +289,6 @@ void TFiber::SetWaiting() void TFiber::SetFinished() { State_.store(EFiberState::Finished); - Clear(); } void TFiber::SetIdle() @@ -276,9 +342,17 @@ void TFiber::Clear() Fls_.reset(); } -std::vector<TFiberPtr> TFiber::List() +void TFiber::ReadFibers(TFunctionView<void(TFiberList&)> callback) { - return TFiberRegistry::Get()->List(); + return TFiberRegistry::Get()->ReadFibers(callback); +} + +void TFiber::UnregisterAndDelete() noexcept +{ + YT_VERIFY(!static_cast<TUnregisterBase*>(this)->IsLinked()); + + static_cast<TRegisterBase*>(this)->Unlink(); + delete this; } namespace NDetail { diff --git a/yt/yt/core/concurrency/fiber.h b/yt/yt/core/concurrency/fiber.h index 1212fd97524..e42510b26a9 100644 --- a/yt/yt/core/concurrency/fiber.h +++ b/yt/yt/core/concurrency/fiber.h @@ -4,23 +4,56 @@ #include "propagating_storage.h" #include "fls.h" +#include <yt/yt/core/misc/intrusive_mpsc_stack.h> + +#include <library/cpp/yt/misc/function_view.h> + #include <util/system/context.h> #include <atomic> -#include <list> namespace NYT::NConcurrency { //////////////////////////////////////////////////////////////////////////////// +class TFiberRegistry; + +//////////////////////////////////////////////////////////////////////////////// + +namespace NDetail { + +//////////////////////////////////////////////////////////////////////////////// + +struct TFiberRegisterTag +{ }; + +struct TFiberUnregisterTag +{ }; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NDetail + +//////////////////////////////////////////////////////////////////////////////// + +// Do not change inheritence order or layout. +// Some offsets are hardcoded at devtools/gdb/yt_fibers_printer.py. class TFiber - : public TRefCounted + : public TIntrusiveNode<TFiber, NDetail::TFiberRegisterTag> + , public TIntrusiveNode<TFiber, NDetail::TFiberUnregisterTag> , public ITrampoLine { + using TRegisterBase = TIntrusiveNode<TFiber, NDetail::TFiberRegisterTag>; + using TUnregisterBase = TIntrusiveNode<TFiber, NDetail::TFiberUnregisterTag>; + public: - using TCookie = std::list<TFiber*>::iterator; + using TFiberList = TSimpleIntrusiveList<TFiber, NDetail::TFiberRegisterTag>; + + static TFiber* CreateFiber(EExecutionStackKind stackKind = EExecutionStackKind::Small); + + // Set this as AfterSwitch to release fiber's resources. + static void ReleaseFiber(TFiber* fiber); - explicit TFiber(EExecutionStackKind stackKind = EExecutionStackKind::Small); ~TFiber(); void Recreate(); @@ -33,7 +66,6 @@ public: void SetRunning(); void SetWaiting(); - void SetFinished(); void SetIdle(); bool TryIntrospectWaiting(EFiberState& state, const std::function<void()>& func); @@ -42,11 +74,11 @@ public: const TPropagatingStorage& GetPropagatingStorage() const; TFls* GetFls() const; - static std::vector<TFiberPtr> List(); + static void ReadFibers(TFunctionView<void(TFiberList&)> callback); private: const std::shared_ptr<TExecutionStack> Stack_; - const TCookie RegistryCookie_; + TExceptionSafeContext MachineContext_; std::atomic<TFiberId> FiberId_ = InvalidFiberId; @@ -55,12 +87,17 @@ private: std::unique_ptr<TFls> Fls_; + explicit TFiber(EExecutionStackKind stackKind = EExecutionStackKind::Small); + + void SetFinished(); void Clear(); void DoRunNaked() override; -}; -DEFINE_REFCOUNTED_TYPE(TFiber) + void UnregisterAndDelete() noexcept; + + friend class ::NYT::NConcurrency::TFiberRegistry; +}; //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/concurrency/fiber_scheduler_thread.cpp b/yt/yt/core/concurrency/fiber_scheduler_thread.cpp index 62c0e8ad680..62f8afd79ae 100644 --- a/yt/yt/core/concurrency/fiber_scheduler_thread.cpp +++ b/yt/yt/core/concurrency/fiber_scheduler_thread.cpp @@ -17,6 +17,8 @@ #include <library/cpp/yt/memory/memory_tag.h> +#include <library/cpp/yt/misc/function_view.h> + #include <library/cpp/yt/threading/fork_aware_spin_lock.h> #include <util/thread/lfstack.h> @@ -25,6 +27,12 @@ namespace NYT::NConcurrency { +// NB(arkady-e1ppa): Please run core tests with this macro undefined +// if you are changing fibers. +#define YT_REUSE_FIBERS + +//////////////////////////////////////////////////////////////////////////////// + using namespace NLogging; using namespace NProfiling; @@ -61,8 +69,41 @@ DEFINE_REFCOUNTED_TYPE(TRefCountedGauge) //////////////////////////////////////////////////////////////////////////////// +// TODO(arkady-e1ppa): Add noexcept perhaps? +using TAfterSwitch = TFunctionView<void()>; + +// NB: We use MakeAfterSwitch to wrap our lambda in order to move closure +// on the caller's stack (initially it is on the suspended fiber's stack). +// We do that because callback can resume fiber which will destroy the +// closure on its stack creating the risk of stack-use-after-scope. +// The only safe place at that moment is caller's stack frame. +template <CInvocable<void()> T> +auto MakeAfterSwitch(T&& lambda) +{ + class TMoveOnCall + { + public: + explicit TMoveOnCall(T&& lambda) + : Lambda_(std::move(lambda)) + { } + + void operator()() + { + auto lambda = std::move(Lambda_); + lambda(); + } + + private: + T Lambda_; + }; + + return TMoveOnCall(std::move(lambda)); +} + +//////////////////////////////////////////////////////////////////////////////// + void RunInFiberContext(TFiber* fiber, TClosure callback); -void SwitchFromThread(TFiberPtr targetFiber); +void SwitchFromThread(TFiber* targetFiber); //////////////////////////////////////////////////////////////////////////////// @@ -84,9 +125,9 @@ struct TFiberContext const TRefCountedGaugePtr WaitingFibersCounter; TExceptionSafeContext MachineContext; - TClosure AfterSwitch; - TFiberPtr ResumerFiber; - TFiberPtr CurrentFiber; + TAfterSwitch AfterSwitch; + TFiber* ResumerFiber = nullptr; + TFiber* CurrentFiber = nullptr; }; YT_THREAD_LOCAL(TFiberContext*) FiberContext; @@ -150,50 +191,50 @@ Y_FORCE_INLINE TExceptionSafeContext* GetMachineContext() return &TryGetFiberContext()->MachineContext; } -Y_FORCE_INLINE void SetAfterSwitch(TClosure&& closure) +Y_FORCE_INLINE void SetAfterSwitch(TAfterSwitch afterSwitch) { auto* context = TryGetFiberContext(); - YT_VERIFY(!context->AfterSwitch); - context->AfterSwitch = std::move(closure); + YT_VERIFY(!context->AfterSwitch.IsValid()); + context->AfterSwitch = afterSwitch; } -Y_FORCE_INLINE TClosure ExtractAfterSwitch() +Y_FORCE_INLINE TAfterSwitch ExtractAfterSwitch() { auto* context = TryGetFiberContext(); - return std::move(context->AfterSwitch); + return context->AfterSwitch.Release(); } -Y_FORCE_INLINE void SetResumerFiber(TFiberPtr fiber) +Y_FORCE_INLINE void SetResumerFiber(TFiber* fiber) { auto* context = TryGetFiberContext(); YT_VERIFY(!context->ResumerFiber); - context->ResumerFiber = std::move(fiber); + context->ResumerFiber = fiber; } -Y_FORCE_INLINE TFiberPtr ExtractResumerFiber() +Y_FORCE_INLINE TFiber* ExtractResumerFiber() { - return std::move(TryGetFiberContext()->ResumerFiber); + return std::exchange(TryGetFiberContext()->ResumerFiber, nullptr); } Y_FORCE_INLINE TFiber* TryGetResumerFiber() { - return TryGetFiberContext()->ResumerFiber.Get(); + return TryGetFiberContext()->ResumerFiber; } -Y_FORCE_INLINE TFiberPtr SwapCurrentFiber(TFiberPtr fiber) +Y_FORCE_INLINE TFiber* SwapCurrentFiber(TFiber* fiber) { - return std::exchange(TryGetFiberContext()->CurrentFiber, std::move(fiber)); + return std::exchange(TryGetFiberContext()->CurrentFiber, fiber); } Y_FORCE_INLINE TFiber* TryGetCurrentFiber() { auto* context = TryGetFiberContext(); - return context ? context->CurrentFiber.Get() : nullptr; + return context ? context->CurrentFiber : nullptr; } Y_FORCE_INLINE TFiber* GetCurrentFiber() { - auto* fiber = TryGetFiberContext()->CurrentFiber.Get(); + auto* fiber = TryGetFiberContext()->CurrentFiber; YT_VERIFY(fiber); return fiber; } @@ -227,15 +268,14 @@ void SwitchMachineContext(TExceptionSafeContext* from, TExceptionSafeContext* to YT_VERIFY(!ExtractAfterSwitch()); } -void SwitchFromThread(TFiberPtr targetFiber) +void SwitchFromThread(TFiber* targetFiber) { YT_ASSERT(targetFiber); targetFiber->SetRunning(); auto* targetContext = targetFiber->GetMachineContext(); - - auto currentFiber = SwapCurrentFiber(std::move(targetFiber)); + auto currentFiber = SwapCurrentFiber(targetFiber); YT_VERIFY(!currentFiber); SwitchMachineContext(GetMachineContext(), targetContext); @@ -243,28 +283,29 @@ void SwitchFromThread(TFiberPtr targetFiber) YT_VERIFY(!TryGetCurrentFiber()); } -[[noreturn]] void SwitchToThread() +[[noreturn]] void SwitchToThread(TAfterSwitch afterSwitch) { auto currentFiber = SwapCurrentFiber(nullptr); auto* currentContext = currentFiber->GetMachineContext(); - currentFiber.Reset(); + SetAfterSwitch(afterSwitch); SwitchMachineContext(currentContext, GetMachineContext()); YT_ABORT(); } -void SwitchFromFiber(TFiberPtr targetFiber) +void SwitchFromFiber(TFiber* targetFiber, TAfterSwitch afterSwitch) { YT_ASSERT(targetFiber); targetFiber->SetRunning(); auto* targetContext = targetFiber->GetMachineContext(); - auto currentFiber = SwapCurrentFiber(std::move(targetFiber)); + auto currentFiber = SwapCurrentFiber(targetFiber); YT_VERIFY(currentFiber->GetState() != EFiberState::Waiting); auto* currentContext = currentFiber->GetMachineContext(); + SetAfterSwitch(afterSwitch); SwitchMachineContext(currentContext, targetContext); YT_VERIFY(TryGetCurrentFiber() == currentFiber); @@ -282,19 +323,25 @@ public: return LeakySingleton<TIdleFiberPool>(); } - void EnqueueIdleFiber(TFiberPtr fiber) + // NB(lukyan): Switch out and add fiber to idle fibers. + // Save fiber in AfterSwitch because it can be immediately concurrently reused. + void SwichFromFiberAndMakeItIdle(TFiber* currentFiber, TFiber* targetFiber) { - IdleFibers_.Enqueue(std::move(fiber)); - if (DestroyingIdleFibers_.load()) { - DoDestroyIdleFibers(); - } + auto afterSwitch = MakeAfterSwitch([currentFiber, this] { + currentFiber->SetIdle(); + EnqueueIdleFiber(currentFiber); + }); + + SwitchFromFiber(targetFiber, afterSwitch); } - TFiberPtr TryDequeueIdleFiber() + TFiber* GetFiber() { - TFiberPtr fiber; - IdleFibers_.Dequeue(&fiber); - return fiber; + if (auto* fiber = TryDequeueIdleFiber()) { + return fiber; + } + + return TFiber::CreateFiber(); } private: @@ -303,9 +350,23 @@ private: BIND_NO_PROPAGATE(&TIdleFiberPool::DestroyIdleFibers, this), /*priority*/ -100); - TLockFreeStack<TFiberPtr> IdleFibers_; + TLockFreeStack<TFiber*> IdleFibers_; std::atomic<bool> DestroyingIdleFibers_ = false; + void EnqueueIdleFiber(TFiber* fiber) + { + IdleFibers_.Enqueue(fiber); + if (DestroyingIdleFibers_.load()) { + DoDestroyIdleFibers(); + } + } + + TFiber* TryDequeueIdleFiber() + { + TFiber* fiber = nullptr; + IdleFibers_.Dequeue(&fiber); + return fiber; + } void DestroyIdleFibers() { @@ -319,11 +380,11 @@ private: TFiberContext fiberContext; TFiberContextGuard fiberContextGuard(&fiberContext); - std::vector<TFiberPtr> fibers; + std::vector<TFiber*> fibers; IdleFibers_.DequeueAll(&fibers); - for (const auto& fiber : fibers) { - SwitchFromThread(std::move(fiber)); + for (auto fiber : fibers) { + SwitchFromThread(fiber); } }; @@ -354,6 +415,20 @@ private: //////////////////////////////////////////////////////////////////////////////// +Y_FORCE_INLINE TClosure PickCallback(TFiberSchedulerThread* fiberThread) +{ + TCallback<void()> callback; + // We wrap fiberThread->OnExecute() into a propagating storage guard to ensure + // that the propagating storage created there won't spill into the fiber callbacks. + TNullPropagatingStorageGuard guard; + YT_VERIFY(guard.GetOldStorage().IsNull()); + callback = fiberThread->OnExecute(); + + return callback; +} + +//////////////////////////////////////////////////////////////////////////////// + void FiberTrampoline() { RunAfterSwitch(); @@ -361,19 +436,13 @@ void FiberTrampoline() YT_LOG_DEBUG("Fiber started"); auto* currentFiber = GetCurrentFiber(); + TFiber* successorFiber = nullptr; // Break loop to terminate fiber while (auto* fiberThread = TryGetFiberThread()) { YT_VERIFY(!TryGetResumerFiber()); - TCallback<void()> callback; - { - // We wrap fiberThread->OnExecute() into a propagating storage guard to ensure - // that the propagating storage created there won't spill into the fiber callbacks. - TNullPropagatingStorageGuard guard; - YT_VERIFY(guard.GetOldStorage().IsNull()); - callback = fiberThread->OnExecute(); - } + auto callback = PickCallback(fiberThread); if (!callback) { break; @@ -387,30 +456,11 @@ void FiberTrampoline() // Trace context can be restored for resumer fiber, so current trace context and memory tag are // not necessarily null. Check them after switch from and returning into current fiber. - if (auto resumerFiber = ExtractResumerFiber()) { + if (successorFiber = ExtractResumerFiber()) { // Suspend current fiber. #ifdef YT_REUSE_FIBERS - { - // TODO(lukyan): Use simple callbacks without memory allocation. - // Make TFiber::MakeIdle method instead of lambda function. - // Switch out and add fiber to idle fibers. - // Save fiber in AfterSwitch because it can be immediately concurrently reused. - SetAfterSwitch(BIND_NO_PROPAGATE([currentFiber = MakeStrong(currentFiber)] () mutable { - currentFiber->SetIdle(); - TIdleFiberPool::Get()->EnqueueIdleFiber(std::move(currentFiber)); - })); - } - - // Switched to ResumerFiber or thread main. - SwitchFromFiber(std::move(resumerFiber)); + TIdleFiberPool::Get()->SwichFromFiberAndMakeItIdle(currentFiber, successorFiber); #else - SetAfterSwitch(BIND_NO_PROPAGATE([ - currentFiber = MakeStrong(currentFiber), - resumerFiber = std::move(resumerFiber) - ] () mutable { - currentFiber.Reset(); - SwitchFromThread(std::move(resumerFiber)); - })); break; #endif } @@ -418,56 +468,60 @@ void FiberTrampoline() YT_LOG_DEBUG("Fiber finished"); - SetAfterSwitch(BIND_NO_PROPAGATE([currentFiber = MakeStrong(currentFiber)] () mutable { - currentFiber->SetFinished(); - currentFiber.Reset(); - })); + auto afterSwitch = MakeAfterSwitch([currentFiber, successorFiber] () mutable { + TFiber::ReleaseFiber(currentFiber); + +#ifdef YT_REUSE_FIBERS + Y_UNUSED(successorFiber); +#else + if (successorFiber != nullptr) { + SwitchFromThread(successorFiber); + } +#endif + }); // All allocated objects in this frame must be destroyed here. - SwitchToThread(); + SwitchToThread(afterSwitch); } -void YieldFiber(TClosure afterSwitch) +void YieldFiber(TAfterSwitch afterSwitch) { YT_VERIFY(TryGetCurrentFiber()); - SetAfterSwitch(std::move(afterSwitch)); - // Try to get resumer. auto targetFiber = ExtractResumerFiber(); // If there is no resumer switch to idle fiber. Or switch to thread main. #ifdef YT_REUSE_FIBERS if (!targetFiber) { - targetFiber = TIdleFiberPool::Get()->TryDequeueIdleFiber(); + targetFiber = TIdleFiberPool::Get()->GetFiber(); } #endif if (!targetFiber) { - targetFiber = New<TFiber>(); + targetFiber = TFiber::CreateFiber(); } auto waitingFibersCounter = GetWaitingFibersCounter(); waitingFibersCounter->Increment(1); - SwitchFromFiber(std::move(targetFiber)); + SwitchFromFiber(targetFiber, afterSwitch); YT_VERIFY(TryGetResumerFiber()); waitingFibersCounter->Increment(-1); } -void ResumeFiber(TFiberPtr targetFiber) +void ResumeFiber(TFiber* targetFiber) { - TMemoryTagGuard guard(NullMemoryTag); - - auto currentFiber = MakeStrong(GetCurrentFiber()); + auto currentFiber = GetCurrentFiber(); SetResumerFiber(currentFiber); - SetAfterSwitch(BIND_NO_PROPAGATE([currentFiber = std::move(currentFiber)] { + auto afterSwitch = MakeAfterSwitch([currentFiber] { currentFiber->SetWaiting(); - })); + }); + + SwitchFromFiber(targetFiber, afterSwitch); - SwitchFromFiber(std::move(targetFiber)); YT_VERIFY(!TryGetResumerFiber()); } @@ -800,13 +854,13 @@ Y_NO_INLINE void RunInFiberContext(TFiber* fiber, TClosure callback) class TResumeGuard { public: - TResumeGuard(TFiberPtr fiber, TCancelerPtr canceler) - : Fiber_(std::move(fiber)) + TResumeGuard(TFiber* fiber, TCancelerPtr canceler) noexcept + : Fiber_(fiber) , Canceler_(std::move(canceler)) { } - explicit TResumeGuard(TResumeGuard&& other) - : Fiber_(std::move(other.Fiber_)) + explicit TResumeGuard(TResumeGuard&& other) noexcept + : Fiber_(other.Release()) , Canceler_(std::move(other.Canceler_)) { } @@ -819,7 +873,7 @@ public: { YT_VERIFY(Fiber_); Canceler_.Reset(); - NDetail::ResumeFiber(std::move(Fiber_)); + NDetail::ResumeFiber(Release()); } ~TResumeGuard() @@ -831,13 +885,20 @@ public: Canceler_.Reset(); GetFinalizerInvoker()->Invoke( - BIND_NO_PROPAGATE(&NDetail::ResumeFiber, Passed(std::move(Fiber_)))); + BIND_NO_PROPAGATE([fiber = Release()] { + NDetail::ResumeFiber(fiber); + })); } } private: - TFiberPtr Fiber_; + TFiber* Fiber_; TCancelerPtr Canceler_; + + TFiber* Release() + { + return std::exchange(Fiber_, nullptr); + } }; } // namespace NDetail @@ -865,7 +926,7 @@ void TFiberSchedulerThread::ThreadMain() NDetail::TFiberContext fiberContext(this, ThreadGroupName_); NDetail::TFiberContextGuard fiberContextGuard(&fiberContext); - NDetail::SwitchFromThread(New<TFiber>()); + NDetail::SwitchFromThread(TFiber::CreateFiber()); YT_LOG_DEBUG("Thread stopped (Name: %v)", GetThreadName()); @@ -968,29 +1029,29 @@ void WaitUntilSet(TFuture<void> future, IInvokerPtr invoker) // TODO(lukyan): transfer resumer as argument of AfterSwitch. // Use CallOnTop like in boost. - auto afterSwitch = BIND_NO_PROPAGATE([ + auto afterSwitch = NDetail::MakeAfterSwitch([ canceler, invoker = std::move(invoker), future = std::move(future), - currentFiber = MakeStrong(currentFiber) + currentFiber ] () mutable { currentFiber->SetWaiting(); future.Subscribe(BIND_NO_PROPAGATE([ invoker = std::move(invoker), - currentFiber = std::move(currentFiber), + currentFiber, canceler = std::move(canceler) ] (const TError&) mutable { YT_LOG_DEBUG("Waking up fiber (TargetFiberId: %x)", canceler->GetFiberId()); invoker->Invoke( - BIND_NO_PROPAGATE(NDetail::TResumeGuard(std::move(currentFiber), std::move(canceler)))); + BIND_NO_PROPAGATE(NDetail::TResumeGuard(currentFiber, std::move(canceler)))); })); }); { NDetail::TFiberSwitchHandler::TGuard switchGuard; - NDetail::YieldFiber(std::move(afterSwitch)); + NDetail::YieldFiber(afterSwitch); } if (canceler->IsCanceled()) { diff --git a/yt/yt/core/concurrency/periodic_executor_base-inl.h b/yt/yt/core/concurrency/periodic_executor_base-inl.h index a2985b72514..6852a398e93 100644 --- a/yt/yt/core/concurrency/periodic_executor_base-inl.h +++ b/yt/yt/core/concurrency/periodic_executor_base-inl.h @@ -40,6 +40,13 @@ void TPeriodicExecutorBase<TInvocationTimePolicy>::Start() } template <CInvocationTimePolicy TInvocationTimePolicy> +bool TPeriodicExecutorBase<TInvocationTimePolicy>::IsStarted() const +{ + auto guard = Guard(SpinLock_); + return Started_; +} + +template <CInvocationTimePolicy TInvocationTimePolicy> void TPeriodicExecutorBase<TInvocationTimePolicy>::DoStop(TGuard<NThreading::TSpinLock>& guard) { if (!Started_) { diff --git a/yt/yt/core/concurrency/periodic_executor_base.h b/yt/yt/core/concurrency/periodic_executor_base.h index 1b171d86695..ddf6a853e87 100644 --- a/yt/yt/core/concurrency/periodic_executor_base.h +++ b/yt/yt/core/concurrency/periodic_executor_base.h @@ -88,6 +88,8 @@ public: //! Starts the instance. void Start(); + bool IsStarted() const; + //! Stops the instance, cancels all subsequent invocations. //! Returns a future that becomes set when all outstanding callback //! invocations are finished and no more invocations are expected to happen. diff --git a/yt/yt/core/concurrency/private.h b/yt/yt/core/concurrency/private.h index 5d300edee32..7c1558e0ad5 100644 --- a/yt/yt/core/concurrency/private.h +++ b/yt/yt/core/concurrency/private.h @@ -50,7 +50,7 @@ using TMpscSuspendableSingleQueueSchedulerThreadPtr = TIntrusivePtr<TMpscSuspend //////////////////////////////////////////////////////////////////////////////// -DECLARE_REFCOUNTED_CLASS(TFiber) +class TFiber; DECLARE_REFCOUNTED_CLASS(TSchedulerThread) diff --git a/yt/yt/core/concurrency/public.h b/yt/yt/core/concurrency/public.h index ee0b58635a2..7bcc13437f2 100644 --- a/yt/yt/core/concurrency/public.h +++ b/yt/yt/core/concurrency/public.h @@ -6,11 +6,6 @@ namespace NYT::NConcurrency { //////////////////////////////////////////////////////////////////////////////// -// Enables fiber instances reuse for improved performance. -#define YT_REUSE_FIBERS - -//////////////////////////////////////////////////////////////////////////////// - DECLARE_REFCOUNTED_CLASS(TActionQueue) DECLARE_REFCOUNTED_STRUCT(IThreadPool) @@ -107,7 +102,7 @@ DECLARE_REFCOUNTED_STRUCT(IPoolWeightProvider) DECLARE_REFCOUNTED_STRUCT(ITwoLevelFairShareThreadPool) -DECLARE_REFCOUNTED_CLASS(TFiber) +class TFiber; DECLARE_REFCOUNTED_STRUCT(TFairThrottlerConfig) DECLARE_REFCOUNTED_STRUCT(TFairThrottlerBucketConfig) diff --git a/yt/yt/core/dns/config.h b/yt/yt/core/dns/config.h index 7e68a0fbc8f..73a15ce8203 100644 --- a/yt/yt/core/dns/config.h +++ b/yt/yt/core/dns/config.h @@ -12,27 +12,12 @@ class TAresDnsResolverConfig : public virtual NYTree::TYsonStruct { public: - bool EnableIPv4; - bool EnableIPv6; - - //! If true, when determining local host name, it will additionally be resolved - //! into FQDN by calling |getaddrinfo|. Setting this option to false may be - //! useful in MTN environment, in which hostnames are barely resolvable. - //! NB: Set this option to false only if you are sure that process is not being - //! exposed under localhost name to anyone; in particular, any kind of discovery - //! should be done using some other kind of addresses. - bool ResolveHostNameIntoFqdn; - //! If set, localhost name will be forcefully set to the given value rather - //! than retrieved via |NYT::NNet::UpdateLocalHostName|. - std::optional<TString> LocalHostNameOverride; int Retries; TDuration RetryDelay; TDuration ResolveTimeout; TDuration MaxResolveTimeout; std::optional<double> Jitter; TDuration WarningTimeout; - //! Used to check that bootstrap is being initialized from a correct container. - std::optional<TString> ExpectedLocalHostName; //! If set, Ares forcefully uses TCP for DNS queries. //! See ARES_FLAG_USEVC. diff --git a/yt/yt/core/misc/error.cpp b/yt/yt/core/misc/error.cpp index cc6ae70bf90..7784292c707 100644 --- a/yt/yt/core/misc/error.cpp +++ b/yt/yt/core/misc/error.cpp @@ -886,8 +886,26 @@ std::optional<TError> TError::FindMatching(TErrorCode code) const } for (const auto& innerError : InnerErrors()) { - auto innerResult = innerError.FindMatching(code); - if (innerResult) { + if (auto innerResult = innerError.FindMatching(code)) { + return innerResult; + } + } + + return {}; +} + +std::optional<TError> TError::FindMatching(const THashSet<TErrorCode>& codes) const +{ + if (!Impl_) { + return {}; + } + + if (codes.contains(GetCode())) { + return *this; + } + + for (const auto& innerError : InnerErrors()) { + if (auto innerResult = innerError.FindMatching(codes)) { return innerResult; } } diff --git a/yt/yt/core/misc/error.h b/yt/yt/core/misc/error.h index 76a5dac28c9..f3783aab7d9 100644 --- a/yt/yt/core/misc/error.h +++ b/yt/yt/core/misc/error.h @@ -190,6 +190,8 @@ public: std::optional<TError> FindMatching(TErrorCode code) const; + std::optional<TError> FindMatching(const THashSet<TErrorCode>& codes) const; + template <class... TArgs> requires std::constructible_from<TError, TArgs...> TError Wrap(TArgs&&... args) const &; diff --git a/yt/yt/core/misc/intrusive_list-inl.h b/yt/yt/core/misc/intrusive_list-inl.h new file mode 100644 index 00000000000..9bf00d4eb18 --- /dev/null +++ b/yt/yt/core/misc/intrusive_list-inl.h @@ -0,0 +1,152 @@ +#ifndef INTRUSIVE_LIST_INL_H_ +#error "Direct inclusion of this file is not allowed, include intrusive_list.h" +// For the sake of sane code completion. +#include "intrusive_list.h" +#endif + +#include <concepts> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T, class Tag> +void TIntrusiveNode<T, Tag>::Unlink() noexcept +{ + if (Next) { + Next->Prev = Prev; + } + + if (Prev) { + Prev->Next = Next; + } + + Prev = Next = nullptr; +} + +template <class T, class Tag> +void TIntrusiveNode<T, Tag>::LinkBefore(TIntrusiveNode* next) noexcept +{ + YT_VERIFY(!IsLinked()); + + Prev = next->Prev; + Prev->Next = this; + Next = next; + next->Prev = this; +} + +template <class T, class Tag> +bool TIntrusiveNode<T, Tag>::IsLinked() const noexcept +{ + return (Next != nullptr) || (Prev != nullptr); +} + +template <class T, class Tag> +T* TIntrusiveNode<T, Tag>::AsItem() noexcept +{ + return static_cast<T*>(this); +} + +//////////////////////////////////////////////////////////////////////////////// + +template <class T, class Tag> +TSimpleIntrusiveList<T, Tag>::TSimpleIntrusiveList() noexcept +{ + InitializeEmpty(); +} + +template <class T, class Tag> +TSimpleIntrusiveList<T, Tag>::TSimpleIntrusiveList(TList&& other) noexcept +{ + InitializeEmpty(); + Append(std::move(other)); +} + +template <class T, class Tag> +void TSimpleIntrusiveList<T, Tag>::PushBack(TNode* node) noexcept +{ + node->LinkBefore(&Head_); +} + +template <class T, class Tag> +void TSimpleIntrusiveList<T, Tag>::PushFront(TNode* node) noexcept +{ + node->LinkBefore(Head_.Next); +} + +template <class T, class Tag> +T*TSimpleIntrusiveList<T, Tag>::PopBack() noexcept +{ + if (IsEmpty()) { + return nullptr; + } + + TNode* back = Head_.Prev; + back->Unlink(); + return back->AsItem(); +} + +template <class T, class Tag> +T* TSimpleIntrusiveList<T, Tag>::PopFront() noexcept +{ + if (IsEmpty()) { + return nullptr; + } + + TNode* front = Head_.Next; + front->Unlink(); + return front->AsItem(); +} + +template <class T, class Tag> +void TSimpleIntrusiveList<T, Tag>::Append(TList&& other) noexcept +{ + if (other.IsEmpty()) { + return; + } + + auto* other_front = other.Head_.Next; + auto* current_back = Head_.Prev; + current_back->Next = other_front; + other_front->Prev = current_back; + + auto* other_back = other.Head_.Prev; + other_back->Next = &Head_; + Head_.Prev = other_back; + + other.InitializeEmpty(); +} + +template <class T, class Tag> +bool TSimpleIntrusiveList<T, Tag>::IsEmpty() const noexcept +{ + return Head_.Next == &Head_; +} + +template <class T, class Tag> +TIntrusiveNode<T, Tag>* TSimpleIntrusiveList<T, Tag>::Begin() +{ + return Head_.Next; +} + +template <class T, class Tag> +TIntrusiveNode<T, Tag>* TSimpleIntrusiveList<T, Tag>::End() +{ + return &Head_; +} + +template <class T, class Tag> +TSimpleIntrusiveList<T, Tag>::~TSimpleIntrusiveList() +{ + YT_VERIFY(IsEmpty()); +} + +template <class T, class Tag> +void TSimpleIntrusiveList<T, Tag>::InitializeEmpty() +{ + Head_.Next = Head_.Prev = &Head_; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/core/misc/intrusive_list.h b/yt/yt/core/misc/intrusive_list.h new file mode 100644 index 00000000000..208e5fea182 --- /dev/null +++ b/yt/yt/core/misc/intrusive_list.h @@ -0,0 +1,87 @@ +#pragma once + +#include <cstdlib> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +struct TIntrusiveNodeDefaultTag +{ }; + +//////////////////////////////////////////////////////////////////////////////// + +// NB1: util/intrlist.h inits pointers differently +// and also doesn't provide a way to directly modify +// Next/Prev making it unusable in lockfree. +// NB2: yt/containers/intrusive_linked_list.h doesn't provide tag +// and gives quite a bit of overhead in the list (ItemToNode field, extra pointer +// and size field). It would be a bit more annoying to hardcode in introspection. +// TODO(arkady-e1ppa): Change util/intrlist.h to support lf algos and use it one day? +template <class T, class Tag = TIntrusiveNodeDefaultTag> +class TIntrusiveNode +{ +public: + TIntrusiveNode* Next = nullptr; + TIntrusiveNode* Prev = nullptr; + + TIntrusiveNode() = default; + + TIntrusiveNode(const TIntrusiveNode& other) = delete; + TIntrusiveNode& operator=(const TIntrusiveNode& other) = delete; + + void Unlink() noexcept; + + void LinkBefore(TIntrusiveNode* next) noexcept; + + bool IsLinked() const noexcept; + + T* AsItem() noexcept; +}; + +//////////////////////////////////////////////////////////////////////////////// + +template <class T, class Tag = TIntrusiveNodeDefaultTag> +class TSimpleIntrusiveList +{ + using TNode = TIntrusiveNode<T, Tag>; + using TList = TSimpleIntrusiveList<T, Tag>; + +public: + TSimpleIntrusiveList() noexcept; + TSimpleIntrusiveList(TSimpleIntrusiveList&& other) noexcept; + + TSimpleIntrusiveList(const TSimpleIntrusiveList& other) = delete; + TSimpleIntrusiveList& operator=(const TSimpleIntrusiveList& other) = delete; + TSimpleIntrusiveList& operator=(TSimpleIntrusiveList&& other) = delete; + + ~TSimpleIntrusiveList(); + + void PushBack(TNode* node) noexcept; + void PushFront(TNode* node) noexcept; + + T* PopBack() noexcept; + T* PopFront() noexcept; + + void Append(TList&& other) noexcept; + + bool IsEmpty() const noexcept; + + TNode* Begin(); + + TNode* End(); + +private: + // Sentinel node. + TNode Head_; + + void InitializeEmpty(); +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + +#define INTRUSIVE_LIST_INL_H_ +#include "intrusive_list-inl.h" +#undef INTRUSIVE_LIST_INL_H_ diff --git a/yt/yt/core/misc/intrusive_mpsc_stack-inl.h b/yt/yt/core/misc/intrusive_mpsc_stack-inl.h new file mode 100644 index 00000000000..e4c553587e3 --- /dev/null +++ b/yt/yt/core/misc/intrusive_mpsc_stack-inl.h @@ -0,0 +1,52 @@ +#ifndef INTRUSIVE_MPSC_STACK_INL_H_ +#error "Direct inclusion of this file is not allowed, include intrusive_mpsc_stack.h" +// For the sake of sane code completion. +#include "intrusive_mpsc_stack.h" +#endif + +#include <concepts> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T, class Tag> +TIntrusiveMPSCStack<T, Tag>::TIntrusiveMPSCStack() noexcept +{ + static_assert(std::derived_from<T, TIntrusiveNode<T, Tag>>, "Class must inherit from CRTP-base TIntrusiveNode"); +} + +template <class T, class Tag> +void TIntrusiveMPSCStack<T, Tag>::Push(TNode* item) noexcept +{ + // NB: This saves up extra CAS in case of non-empty stack. + item->Next = Head_.load(std::memory_order::relaxed); + + while (!Head_.compare_exchange_weak( + item->Next, + item, + std::memory_order::release, + std::memory_order::relaxed)) + { } +} + +template <class T, class Tag> +TSimpleIntrusiveList<T, Tag> TIntrusiveMPSCStack<T, Tag>::PopAll() noexcept +{ + TNode* head = Head_.exchange(nullptr, std::memory_order::acquire); + + TSimpleIntrusiveList<T, Tag> list; + + while (head != nullptr) { + auto tmp = head; + head = head->Next; + tmp->Next = nullptr; + list.PushFront(tmp); + } + + return list; +} + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT diff --git a/yt/yt/core/misc/intrusive_mpsc_stack.h b/yt/yt/core/misc/intrusive_mpsc_stack.h new file mode 100644 index 00000000000..80d460553e1 --- /dev/null +++ b/yt/yt/core/misc/intrusive_mpsc_stack.h @@ -0,0 +1,33 @@ +#pragma once + +#include "intrusive_list.h" + +#include <atomic> + +namespace NYT { + +//////////////////////////////////////////////////////////////////////////////// + +template <class T, class Tag = TIntrusiveNodeDefaultTag> +class TIntrusiveMPSCStack +{ + using TNode = TIntrusiveNode<T, Tag>; + +public: + TIntrusiveMPSCStack() noexcept; + + void Push(TNode* item) noexcept; + + TSimpleIntrusiveList<T, Tag> PopAll() noexcept; + +private: + std::atomic<TNode*> Head_ = nullptr; +}; + +//////////////////////////////////////////////////////////////////////////////// + +} // namespace NYT + +#define INTRUSIVE_MPSC_STACK_INL_H_ +#include "intrusive_mpsc_stack-inl.h" +#undef INTRUSIVE_MPSC_STACK_INL_H_ diff --git a/yt/yt/core/net/address.cpp b/yt/yt/core/net/address.cpp index 756be2a889a..e06ae312ff4 100644 --- a/yt/yt/core/net/address.cpp +++ b/yt/yt/core/net/address.cpp @@ -928,7 +928,6 @@ public: /*logger*/ {}, DnsProfiler.WithPrefix("/resolve_cache")) { - SetDnsResolver(CreateAresDnsResolver(config)); Configure(std::move(config)); } @@ -984,6 +983,9 @@ public: { Config_ = std::move(config); + SetDnsResolver(CreateAresDnsResolver(Config_)); + TAsyncExpiringCache::Reconfigure(Config_); + if (Config_->LocalHostNameOverride) { WriteLocalHostName(*Config_->LocalHostNameOverride); YT_LOG_INFO("Localhost name configured via config override (LocalHostName: %v)", diff --git a/yt/yt/core/rpc/config.cpp b/yt/yt/core/rpc/config.cpp index e715e9b4e40..0c23e7085dd 100644 --- a/yt/yt/core/rpc/config.cpp +++ b/yt/yt/core/rpc/config.cpp @@ -249,6 +249,7 @@ void TBalancingChannelConfig::Register(TRegistrar registrar) int endpointConfigCount = 0; if (config->Addresses) { ++endpointConfigCount; + SortUnique(*config->Addresses); } if (config->Endpoints) { ++endpointConfigCount; diff --git a/yt/yt/library/backtrace_introspector/introspect.cpp b/yt/yt/library/backtrace_introspector/introspect.cpp index a555a8a5482..38e146ab082 100644 --- a/yt/yt/library/backtrace_introspector/introspect.cpp +++ b/yt/yt/library/backtrace_introspector/introspect.cpp @@ -40,61 +40,67 @@ std::vector<TFiberIntrospectionInfo> IntrospectFibers() { YT_LOG_INFO("Fiber introspection started"); - auto fibers = TFiber::List(); - YT_LOG_INFO("Collecting waiting fibers backtraces"); std::vector<TFiberIntrospectionInfo> infos; THashSet<TFiberId> waitingFiberIds; - THashSet<TFiberId> fiberIds; - for (const auto& fiber : fibers) { - auto fiberId = fiber->GetFiberId(); - if (fiberId == InvalidFiberId) { - continue; - } + THashMap<TFiberId, EFiberState> fiberStates; - InsertOrCrash(fiberIds, fiberId); + auto introspectionAction = [&] (NYT::NConcurrency::TFiber::TFiberList& fibers) { + for (auto* iter = fibers.Begin(); iter != fibers.End(); iter = iter->Next) { + auto* fiber = iter->AsItem(); - EFiberState state; - if (!fiber->TryIntrospectWaiting(state, [&] { - YT_LOG_DEBUG("Waiting fiber is successfully locked for introspection (FiberId: %x)", - fiberId); - - const auto& propagatingStorage = fiber->GetPropagatingStorage(); - const auto* traceContext = TryGetTraceContextFromPropagatingStorage(propagatingStorage); + auto fiberId = fiber->GetFiberId(); + if (fiberId == InvalidFiberId) { + continue; + } - TFiberIntrospectionInfo info{ - .State = EFiberState::Waiting, - .FiberId = fiberId, - .WaitingSince = fiber->GetWaitingSince(), - .TraceId = traceContext ? traceContext->GetTraceId() : TTraceId(), - .TraceLoggingTag = traceContext ? traceContext->GetLoggingTag() : TString(), - }; + EmplaceOrCrash(fiberStates, fiberId, EFiberState::Introspecting); - auto optionalContext = TrySynthesizeLibunwindContextFromMachineContext(*fiber->GetMachineContext()); - if (!optionalContext) { - YT_LOG_WARNING("Failed to synthesize libunwind context (FiberId: %x)", + EFiberState state; + if (!fiber->TryIntrospectWaiting(state, [&] { + YT_LOG_DEBUG("Waiting fiber is successfully locked for introspection (FiberId: %x)", fiberId); - return; - } - TLibunwindCursor cursor(*optionalContext); - while (!cursor.IsFinished()) { - info.Backtrace.push_back(cursor.GetCurrentIP()); - cursor.MoveNext(); - } + const auto& propagatingStorage = fiber->GetPropagatingStorage(); + const auto* traceContext = TryGetTraceContextFromPropagatingStorage(propagatingStorage); + + TFiberIntrospectionInfo info{ + .State = EFiberState::Waiting, + .FiberId = fiberId, + .WaitingSince = fiber->GetWaitingSince(), + .TraceId = traceContext ? traceContext->GetTraceId() : TTraceId(), + .TraceLoggingTag = traceContext ? traceContext->GetLoggingTag() : TString(), + }; + + auto optionalContext = TrySynthesizeLibunwindContextFromMachineContext(*fiber->GetMachineContext()); + if (!optionalContext) { + YT_LOG_WARNING("Failed to synthesize libunwind context (FiberId: %x)", + fiberId); + return; + } + + TLibunwindCursor cursor(*optionalContext); + while (!cursor.IsFinished()) { + info.Backtrace.push_back(cursor.GetCurrentIP()); + cursor.MoveNext(); + } - infos.push_back(std::move(info)); - InsertOrCrash(waitingFiberIds, fiberId); + infos.push_back(std::move(info)); + InsertOrCrash(waitingFiberIds, fiberId); - YT_LOG_DEBUG("Fiber introspection completed (FiberId: %x)", - info.FiberId); - })) { - YT_LOG_DEBUG("Failed to lock fiber for introspection (FiberId: %x, State: %v)", - fiberId, - state); + YT_LOG_DEBUG("Fiber introspection completed (FiberId: %x)", + info.FiberId); + })) { + YT_LOG_DEBUG("Failed to lock fiber for introspection (FiberId: %x, State: %v)", + fiberId, + state); + fiberStates[fiberId] = state; + } } - } + }; + + TFiber::ReadFibers(introspectionAction); YT_LOG_INFO("Collecting running fibers backtraces"); @@ -123,8 +129,7 @@ std::vector<TFiberIntrospectionInfo> IntrospectFibers() }); } - for (const auto& fiber : fibers) { - auto fiberId = fiber->GetFiberId(); + for (const auto& [fiberId, fiberState] : fiberStates) { if (fiberId == InvalidFiberId) { continue; } @@ -136,7 +141,7 @@ std::vector<TFiberIntrospectionInfo> IntrospectFibers() } infos.push_back(TFiberIntrospectionInfo{ - .State = fiber->GetState(), + .State = fiberState, .FiberId = fiberId, }); } diff --git a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto index 860672c48fc..3795f80077b 100644 --- a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto +++ b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto @@ -10,9 +10,10 @@ import "yt_proto/yt/core/misc/proto/guid.proto"; import "yt_proto/yt/core/misc/proto/error.proto"; import "yt_proto/yt/core/ytree/proto/attributes.proto"; import "yt_proto/yt/core/ytree/proto/request_complexity_limits.proto"; -import "yt_proto/yt/client/hive/proto/timestamp_map.proto"; import "yt_proto/yt/client/chunk_client/proto/data_statistics.proto"; import "yt_proto/yt/client/chaos_client/proto/replication_card.proto"; +import "yt_proto/yt/client/hive/proto/timestamp_map.proto"; +import "yt_proto/yt/client/tablet_client/proto/lock_mask.proto"; //////////////////////////////////////////////////////////////////////////////// // Scalars @@ -365,6 +366,11 @@ message TRowsetStatistics message TReqStartTransaction { + message TReplicateToMasterCellTags + { + repeated int32 cell_tags = 1; + } + required ETransactionType type = 1; optional int64 timeout = 2; @@ -392,6 +398,8 @@ message TReqStartTransaction optional uint64 start_timestamp = 14; + optional TReplicateToMasterCellTags replicate_to_master_cell_tags = 15; + optional TMutatingOptions mutating_options = 103; } @@ -905,14 +913,17 @@ message TReqModifyRows required string path = 2; repeated ERowModificationType row_modification_types = 3; // COMPAT(lukyan): Remove after RPC protocol version update - repeated uint32 row_read_locks = 7; - repeated uint64 row_locks = 8; + repeated uint32 row_legacy_read_locks = 7; + // COMPAT(ponasenko-rs): Remove after RPC protocol version update + repeated uint64 row_legacy_locks = 8; optional bool require_sync_replica = 4; optional NYT.NProto.TGuid upstream_replica_id = 5; optional bool allow_missing_key_columns = 10; + repeated NYT.NTabletClient.NProto.TLockMask row_locks = 11; + required TRowsetDescriptor rowset_descriptor = 200; } diff --git a/yt/yt_proto/yt/client/cell_master/proto/cell_directory.proto b/yt/yt_proto/yt/client/cell_master/proto/cell_directory.proto index 18397dfd1ba..e8093315bc4 100644 --- a/yt/yt_proto/yt/client/cell_master/proto/cell_directory.proto +++ b/yt/yt_proto/yt/client/cell_master/proto/cell_directory.proto @@ -17,46 +17,46 @@ message TCellDirectoryItem { repeated uint32 roles = 2; // EMasterCellRole(s) - required int64 rpc_timeout = 4; + optional int64 rpc_timeout = 4; // TPeerConnectionConfig required NYT.NProto.TGuid cell_id = 1; - required bool ignore_peer_state = 5; + optional bool ignore_peer_state = 5; // TBalancingChannelConfig repeated string addresses = 3; - required bool disable_balancing_on_single_address = 6; - required TServiceDiscoveryEndpointsConfig endpoints = 7; + optional bool disable_balancing_on_single_address = 6; + optional TServiceDiscoveryEndpointsConfig endpoints = 7; optional int64 hedging_delay = 8; - required bool cancel_primary_request_on_hedging = 9; + optional bool cancel_primary_request_on_hedging = 9; // TDynamicChannelPoolConfig - required int64 max_concurrent_discover_requests = 10; - required int64 random_peer_eviction_period = 11; - required bool enable_peer_polling = 12; - required int64 peer_polling_period = 13; - required int64 peer_polling_period_splay = 14; - required int64 peer_polling_request_timeout = 15; - required int64 discovery_session_timeout = 16; + optional int64 max_concurrent_discover_requests = 10; + optional int64 random_peer_eviction_period = 11; + optional bool enable_peer_polling = 12; + optional int64 peer_polling_period = 13; + optional int64 peer_polling_period_splay = 14; + optional int64 peer_polling_request_timeout = 15; + optional int64 discovery_session_timeout = 16; // TViablePeerRegistryConfig - required int64 max_peer_count = 17; - required int64 hashes_per_peer = 18; - required uint32 peer_priority_strategy = 19; // EPeerPriorityStrategy - required int64 min_peer_count_for_priority_awareness = 20; - required bool enable_power_of_two_choices_strategy = 21; + optional int64 max_peer_count = 17; + optional int64 hashes_per_peer = 18; + optional uint32 peer_priority_strategy = 19; // EPeerPriorityStrategy + optional int64 min_peer_count_for_priority_awareness = 20; + optional bool enable_power_of_two_choices_strategy = 21; // TBalancingChannelConfigBase - required int64 discover_timeout = 22; - required int64 acknowledgement_timeout = 23; - required int64 rediscover_period = 24; - required int64 rediscover_splay = 25; - required int64 hard_backoff_time = 26; - required int64 soft_backoff_time = 27; + optional int64 discover_timeout = 22; + optional int64 acknowledgement_timeout = 23; + optional int64 rediscover_period = 24; + optional int64 rediscover_splay = 25; + optional int64 hard_backoff_time = 26; + optional int64 soft_backoff_time = 27; // TRetryingChannelConfig - required int64 retry_backoff_time = 28; - required int64 retry_attempts = 29; + optional int64 retry_backoff_time = 28; + optional int64 retry_attempts = 29; optional int64 retry_timeout = 30; } diff --git a/yt/yt_proto/yt/client/tablet_client/proto/lock_mask.proto b/yt/yt_proto/yt/client/tablet_client/proto/lock_mask.proto new file mode 100644 index 00000000000..55d74bf6a07 --- /dev/null +++ b/yt/yt_proto/yt/client/tablet_client/proto/lock_mask.proto @@ -0,0 +1,13 @@ +package NYT.NTabletClient.NProto; + +option go_package = "a.yandex-team.ru/yt/go/proto/client/tablet_client"; + +//////////////////////////////////////////////////////////////////////////////// + +message TLockMask +{ + repeated uint64 bitmap = 1; + required uint32 size = 2; +} + +//////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt_proto/yt/client/ya.make b/yt/yt_proto/yt/client/ya.make index 8c742acce8b..7552ce952d7 100644 --- a/yt/yt_proto/yt/client/ya.make +++ b/yt/yt_proto/yt/client/ya.make @@ -38,6 +38,8 @@ SRCS( table_chunk_format/proto/column_meta.proto table_chunk_format/proto/wire_protocol.proto + tablet_client/proto/lock_mask.proto + transaction_client/proto/timestamp_service.proto query_client/proto/query_statistics.proto |