diff options
| author | Maxim Yurchuk <maxim-yurchuk@ydb.tech> | 2025-05-30 21:14:52 +0000 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2025-05-30 21:14:52 +0000 |
| commit | c75cf6fa89ba44e2fa74a15232593b2e8423ed3f (patch) | |
| tree | a7f428153ad7b3109180af04a9c84af2b0ab2a16 | |
| parent | b21606bc4b50665ea3fdca703e13a4b4d7a44284 (diff) | |
| parent | 8728b9da66674488bde07a092040097e46de9366 (diff) | |
| download | ydb-c75cf6fa89ba44e2fa74a15232593b2e8423ed3f.tar.gz | |
Library import 250529-1108 (#19003)
520 files changed, 28936 insertions, 2420 deletions
diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 7242b741e5e..54e169b6d43 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -20,8 +20,11 @@ ydb/core/kqp/ut/batch_operations [*/*] chunk chunk ydb/core/kqp/ut/cost KqpCost.OlapWriteRow ydb/core/kqp/ut/federated_query/large_results KqpScriptExecResults.ExecuteScriptWithLargeFile ydb/core/kqp/ut/federated_query/s3 sole chunk chunk +ydb/core/kqp/ut/olap KqpOlap.PredicatePushdown ydb/core/kqp/ut/indexes KqpMultishardIndex.WriteIntoRenamingSyncIndex ydb/core/kqp/ut/olap KqpOlapDictionary.DifferentPages +ydb/core/kqp/ut/olap KqpOlapJson.BloomMixIndexesVariants +ydb/core/kqp/ut/olap KqpOlapJson.BloomNGrammIndexesVariants ydb/core/kqp/ut/olap KqpOlapJson.CompactionVariants ydb/core/kqp/ut/olap KqpOlapJson.DuplicationCompactionVariants ydb/core/kqp/ut/olap KqpOlapJson.SwitchAccessorCompactionVariants diff --git a/build/conf/opensource.conf b/build/conf/opensource.conf index 067aacf3cb0..5b9a24ebd5f 100644 --- a/build/conf/opensource.conf +++ b/build/conf/opensource.conf @@ -59,8 +59,8 @@ when ($OPENSOURCE == "yes" && $EXPORT_CMAKE == "yes") { # USE_GLOBAL_CMD=yes does not work for Windows with ya make builds but works with exported CMake files: YMAKE-657. USE_GLOBAL_CMD=yes EXPORT_SEM=yes - EXPORTED_BUILD_SYSTEM_SOURCE_ROOT="${PROJECT_SOURCE_DIR}" - EXPORTED_BUILD_SYSTEM_BUILD_ROOT="${PROJECT_BINARY_DIR}" + EXPORTED_BUILD_SYSTEM_SOURCE_ROOT=${PROJECT_SOURCE_DIR} + EXPORTED_BUILD_SYSTEM_BUILD_ROOT=${PROJECT_BINARY_DIR} } when ($EXPORT_CMAKE == "yes") { diff --git a/build/conf/proto.conf b/build/conf/proto.conf index a3b209bf575..49a9deb10e8 100644 --- a/build/conf/proto.conf +++ b/build/conf/proto.conf @@ -500,7 +500,7 @@ macro USE_JAVALITE() { # tag:proto tag:java-specific JAVA_PROTO_COMPILER_VERSION = 3.25.5 JAVA_PROTO_RUNTIME_VERSION = 3.25.5 -JAVA_PROTO_COMMON_VERSION = 2.48.0 +JAVA_PROTO_COMMON_VERSION = 2.51.0 JAVA_GRPC_VERSION = 1.51.0 KOTLIN_GRPC_VERSION = 1.3.1 JAVA_NETTY_NETTY_VERSION = 4.1.79.Final diff --git a/build/export_generators/ide-gradle/a.yaml b/build/export_generators/ide-gradle/a.yaml index de4db37b53c..4be898247ad 100644 --- a/build/export_generators/ide-gradle/a.yaml +++ b/build/export_generators/ide-gradle/a.yaml @@ -1,6 +1,20 @@ service: ya_make title: ya ide gradle export testing +arcanum: + review: + auto_assign: true + groups: + - name: ya_ide_gradle_developer + roles: ["ya_ide_gradle:developer"] + + rules: + - reviewers: + - name: ya_ide_gradle_developer + ship: 1 + assign: 1 + ignore_self_ship: false + ci: secret: sec-01hmxvzfvzwhj0k6qe9vh9w9yw runtime: diff --git a/build/export_generators/ide-gradle/build.gradle.kts.any.jinja b/build/export_generators/ide-gradle/build.gradle.kts.any.jinja index d5126e76721..ca90ffeaba4 100644 --- a/build/export_generators/ide-gradle/build.gradle.kts.any.jinja +++ b/build/export_generators/ide-gradle/build.gradle.kts.any.jinja @@ -6,20 +6,23 @@ {#- Always replace (arcadia_root) === (SOURCE_ROOT in ymake) to $arcadia_root in Gradle -#} {%- if depend -%} {#- Replace (export_root) === (BUILD_ROOT in ymake) to $arcadia_root in Gradle, because prebuilt tools in arcadia, not in build_root -#} -"{{ arg|replace(export_root, "$arcadia_root")|replace(arcadia_root, "$arcadia_root") }}" +"{{ arg|replace(export_root, "$output_root")|replace(project_root, "$project_root")|replace(arcadia_root, "$arcadia_root") }}" {%- elif output and arg[0] != '/' -%} {#- Relative outputs in buildDir -#} "$buildDir/{{ arg }}" +{%- elif output -%} +{#- Replace (export_root) === (BUILD_ROOT in ymake) to baseBuildDir in Gradle - root of all build folders for modules -#} +"{{ arg|replace(export_root, "$baseBuildDir")|replace(project_root, "$project_root")|replace(arcadia_root, "$output_root")|replace(output_root, "$output_root") }}" {%- else -%} {#- Replace (export_root) === (BUILD_ROOT in ymake) to baseBuildDir in Gradle - root of all build folders for modules -#} -"{{ arg|replace(export_root, "$baseBuildDir")|replace(arcadia_root, "$arcadia_root") }}" +"{{ arg|replace(export_root, "$baseBuildDir")|replace(project_root, "$project_root")|replace(arcadia_root, "$arcadia_root")|replace(output_root, "$output_root") }}" {%- endif -%} {%- endmacro -%} {%- if proto_template -%} {%- macro PatchGeneratedProto(arg, relative = false) -%} {%- if relative -%} -"{{ arg|replace(export_root + "/", "")|replace(arcadia_root + "/", "") }}" +"{{ arg|replace(export_root + "/", "")|replace(arcadia_root + "/", "")|replace(output_root + "/", "") }}" {%- else -%} "{{ arg|replace(export_root, "$baseBuildDir")|replace(arcadia_root, "$baseBuildDir") }}" {%- endif -%} @@ -34,7 +37,7 @@ {%- include "[generator]/proto_import.jinja" -%} {%- include "[generator]/proto_builddir.jinja" -%} {%- include "[generator]/proto_plugins.jinja" -%} -{%- include "[generator]/proto_configuration.jinja" -%} +{%- include "[generator]/configuration.jinja" -%} {%- include "[generator]/protobuf.jinja" -%} {%- include "[generator]/proto_prepare.jinja" -%} {%- include "[generator]/build.gradle.kts.common.jinja" -%} diff --git a/build/export_generators/ide-gradle/builddir.jinja b/build/export_generators/ide-gradle/builddir.jinja index 9ee3cd071d4..7e5e53e4974 100644 --- a/build/export_generators/ide-gradle/builddir.jinja +++ b/build/export_generators/ide-gradle/builddir.jinja @@ -1,6 +1,16 @@ {#- empty string #} val baseBuildDir = "{{ export_root }}/gradle.build" -buildDir = file(baseBuildDir + "{%- if common_dir %}/{{ common_dir }}{% endif -%}/" + project.path.replace(":", "/")) +layout.buildDirectory = file(baseBuildDir + "{%- if common_dir %}/{{ common_dir }}{% endif -%}/" + project.path.replace(":", "/")) subprojects { - buildDir = file(baseBuildDir + "{%- if common_dir %}/{{ common_dir }}{% endif -%}/" + project.path.replace(":", "/")) + layout.buildDirectory = file(baseBuildDir + "{%- if common_dir %}/{{ common_dir }}{% endif -%}/" + project.path.replace(":", "/")) +} + +var mkBuildDir = tasks.register("mkBuildDir") { + doFirst { + mkdir(layout.buildDirectory.asFile.get()) + } +} + +tasks.withType<JavaExec>().configureEach { + dependsOn(mkBuildDir) } diff --git a/build/export_generators/ide-gradle/codegen.jinja b/build/export_generators/ide-gradle/codegen.jinja index 14ab67543fe..31d8c0df04d 100644 --- a/build/export_generators/ide-gradle/codegen.jinja +++ b/build/export_generators/ide-gradle/codegen.jinja @@ -13,7 +13,7 @@ tasks.compileJava.configure { tasks.compileTestJava.configure { dependsOn({{ taskvar }}) } -{%- if not(disable_lombok_plugin) and (current_target.use_annotation_processor|select('startsWith', 'contrib/java/org/projectlombok/lombok')|length) %} +{%- if with_lombok_plugin %} tasks.getByName("delombok").mustRunAfter({{ taskvar }}) {%- endif -%} diff --git a/build/export_generators/ide-gradle/codegen_common.jinja b/build/export_generators/ide-gradle/codegen_common.jinja deleted file mode 100644 index e69de29bb2d..00000000000 --- a/build/export_generators/ide-gradle/codegen_common.jinja +++ /dev/null diff --git a/build/export_generators/ide-gradle/codegen_run_java_program.jinja b/build/export_generators/ide-gradle/codegen_run_java_program.jinja index 87fb1bedeaa..2b826534904 100644 --- a/build/export_generators/ide-gradle/codegen_run_java_program.jinja +++ b/build/export_generators/ide-gradle/codegen_run_java_program.jinja @@ -15,9 +15,9 @@ val {{ varprefix }}{{ run['_object_index'] }} = task<JavaExec>("{{ varprefix }}{ {%- set classpaths = run.classpath|reject('eq', '@.cplst') -%} {%- if classpaths|length -%} {% for classpath in classpaths -%} -{%- set rel_file_classpath = classpath|replace('@', '')|replace(export_root, '')|replace(arcadia_root, '') %} +{%- set rel_file_classpath = classpath|replace('@', '')|replace(export_root, '')|replace(arcadia_root, '')|replace(output_root, '') %} - val classpaths = "$arcadia_root/" + File("$arcadia_root{{ rel_file_classpath }}").readText().trim().replace(":", ":$arcadia_root/") + val classpaths = "$output_root/" + File("$output_root{{ rel_file_classpath }}").readText().trim().replace(":", ":$output_root/") classpath = files(classpaths.split(":")) {%- endfor -%} {%- endif %} diff --git a/build/export_generators/ide-gradle/common_vars.jinja b/build/export_generators/ide-gradle/common_vars.jinja index 92ae2bf8d14..20349ac618d 100644 --- a/build/export_generators/ide-gradle/common_vars.jinja +++ b/build/export_generators/ide-gradle/common_vars.jinja @@ -7,23 +7,31 @@ {%- endif -%} {%- set publish = all_targets|selectattr('publish', 'eq', true)|length -%} +{%- if not output_root -%} +{%- set output_root = arcadia_root -%} +{%- endif -%} -{#- Kotlin -#} +{#- Kotlin -#} {%- set with_kotlin = all_targets|selectattr('with_kotlin', 'eq', true)|length -%} {%- if with_kotlin -%} {%- set kotlin_version = all_targets|selectattr('kotlin_version')|map(attribute='kotlin_version')|first -%} {%- set with_kotlinc_plugin_allopen = all_targets|selectattr('with_kotlinc_plugin_allopen')|map(attribute='with_kotlinc_plugin_allopen')|sum -%} {%- set with_kotlinc_plugin_noarg = all_targets|selectattr('with_kotlinc_plugin_noarg')|map(attribute='with_kotlinc_plugin_noarg')|sum -%} +{%- set with_kotlinc_plugin_serialization = all_targets|selectattr('with_kotlinc_plugin_serialization')|map(attribute='with_kotlinc_plugin_serialization')|sum -%} +{%- set with_kotlinc_plugin_lombok = all_targets|selectattr('with_kotlinc_plugin_lombok')|map(attribute='with_kotlinc_plugin_lombok')|sum -%} {#- KAPT -#} -{%- set with_kapt = target.with_kapt -%} -{%- set with_test_kapt = extra_targets|selectattr('with_kapt', 'eq', true)|map(attribute='with_kapt')|length -%} +{%- set with_kapt = target.with_kapt and target.kapt.classpaths|length -%} +{%- set with_test_kapt = extra_targets|selectattr('with_kapt', 'eq', true)|map(attribute='with_kapt')|length and extra_targets|selectattr('with_kapt', 'eq', true)|selectattr('kapt')|map(attribute='kapt')|sum|selectattr('classpath')|map(attribute='classpaths')|sum|length -%} {%- endif -%} -{#- ErrorProne -#} +{#- ErrorProne -#} {%- set with_errorprone = not(disable_errorprone) and (target.use_errorprone) and (target.consumer|selectattr('jar', 'startsWith', 'contrib/java/com/google/errorprone/error_prone_annotations')|length) -%} {%- set with_test_errorprone = not(disable_errorprone) and (extra_targets|selectattr('use_errorprone', 'eq', true)|length) and (extra_targets|selectattr('consumer')|map(attribute='consumer')|sum|selectattr('jar', 'startsWith', 'contrib/java/com/google/errorprone/error_prone_annotations')|length) -%} {%- if with_errorprone or with_test_errorprone -%} {%- set errorprones = all_targets|selectattr('consumer')|map(attribute='consumer')|sum|selectattr('classpath')|selectattr('jar', 'startsWith', 'contrib/java/com/google/errorprone/error_prone_annotations')|unique -%} {%- endif -%} +{#- Lombok -#} +{%- set with_lombok_plugin = not(disable_lombok_plugin) and (all_targets|selectattr('use_annotation_processor')|map(attribute='use_annotation_processor')|sum|select('startsWith', 'contrib/java/org/projectlombok/lombok')|length) -%} + {%- include "[generator]/jdk.jinja" -%} diff --git a/build/export_generators/ide-gradle/configuration.jinja b/build/export_generators/ide-gradle/configuration.jinja index c049cba2dc5..2c80fadf76f 100644 --- a/build/export_generators/ide-gradle/configuration.jinja +++ b/build/export_generators/ide-gradle/configuration.jinja @@ -1,5 +1,6 @@ {#- empty string #} val arcadia_root = "{{ arcadia_root }}" +val output_root = "{{ output_root }}" val project_root = "{{ project_root }}" {% if mainClass -%} diff --git a/build/export_generators/ide-gradle/dependencies.jinja b/build/export_generators/ide-gradle/dependencies.jinja index 7ad9fd7d79d..5bccf0c2710 100644 --- a/build/export_generators/ide-gradle/dependencies.jinja +++ b/build/export_generators/ide-gradle/dependencies.jinja @@ -1,22 +1,23 @@ {%- macro AnnotationProcessors(funcName, annotation_processors) -%} {%- if annotation_processors|length -%} -{%- for annotation_processor in annotation_processors %} - {{ funcName }}(files("$arcadia_root/{{ annotation_processor }}")) +{%- for annotation_processor in annotation_processors|unique|sort -%} +{%- set parts = rsplit(annotation_processor, "/", 4) %} + {{ funcName }}("{{ parts[0]|replace("contrib/java/", "")|replace("/", ".") }}:{{ parts[1] }}:{{ parts[2] }}") {%- endfor -%} {%- endif -%} {%- endmacro -%} {%- macro Kapts(funcName, kapts) -%} {%- if kapts|length -%} -{%- for kapt in kapts %} - {{ funcName }}(files("$arcadia_root/{{ kapt }}")) +{%- for kapt in kapts|unique|sort %} + {{ funcName }}(files("$output_root/{{ kapt }}")) {%- endfor -%} {%- endif -%} {%- endmacro -%} {%- macro AddFileJars(file_jars) -%} {%- for file_jar in file_jars %} - "$arcadia_root/{{ file_jar }}"{%- if not loop.last -%},{%- endif -%} + "$output_root/{{ file_jar }}"{%- if not loop.last -%},{%- endif -%} {%- endfor -%} {%- endmacro -%} diff --git a/build/export_generators/ide-gradle/import.jinja b/build/export_generators/ide-gradle/import.jinja index 52a4a4bd5ba..f0fafce3c47 100644 --- a/build/export_generators/ide-gradle/import.jinja +++ b/build/export_generators/ide-gradle/import.jinja @@ -3,9 +3,13 @@ import net.ltgt.gradle.errorprone.CheckSeverity import net.ltgt.gradle.errorprone.errorprone {% endif -%} + {%- if with_kotlin -%} import org.jetbrains.kotlin.gradle.tasks.KotlinCompile {%- if with_kapt or with_test_kapt %} import org.jetbrains.kotlin.gradle.internal.KaptGenerateStubsTask +{%- endif -%} +{%- if with_lombok_plugin %} +import io.freefair.gradle.plugins.lombok.tasks.Delombok {%- endif %} {% endif -%} diff --git a/build/export_generators/ide-gradle/kotlin_plugins.jinja b/build/export_generators/ide-gradle/kotlin_plugins.jinja index 0a2e7cd48d2..f84fd57adbb 100644 --- a/build/export_generators/ide-gradle/kotlin_plugins.jinja +++ b/build/export_generators/ide-gradle/kotlin_plugins.jinja @@ -56,6 +56,13 @@ noArg { } {% endif -%} +{%- if with_kotlin and with_lombok_plugin %} + +tasks.withType<Delombok>().configureEach { + classpath = classpath + files(sourceSets.main.get().output) +} +{% endif -%} + {%- if with_kotlin %} kotlin { diff --git a/build/export_generators/ide-gradle/plugins.jinja b/build/export_generators/ide-gradle/plugins.jinja index fa8fb33b540..db19095720c 100644 --- a/build/export_generators/ide-gradle/plugins.jinja +++ b/build/export_generators/ide-gradle/plugins.jinja @@ -14,7 +14,7 @@ plugins { {%- if with_kotlinc_plugin_allopen|length %} kotlin("plugin.allopen") version "{{ kotlin_version }}" {%- endif -%} -{%- if not(disable_lombok_plugin) and (with_kotlinc_plugin_lombok|length) %} +{%- if with_kotlinc_plugin_lombok|length %} kotlin("plugin.lombok") version "{{ kotlin_version }}" {%- endif -%} {%- if with_kotlinc_plugin_noarg|length %} @@ -32,7 +32,7 @@ plugins { id("net.ltgt.errorprone") version "4.1.0" {%- endif -%} -{%- if not(disable_lombok_plugin) and (target.use_annotation_processor|select('startsWith', 'contrib/java/org/projectlombok/lombok')|length) %} +{%- if with_lombok_plugin %} id("io.freefair.lombok") version "8.6" {%- endif %} } diff --git a/build/export_generators/ide-gradle/proto_dependencies.jinja b/build/export_generators/ide-gradle/proto_dependencies.jinja index dc3ca68e4be..ec2e3c83d1b 100644 --- a/build/export_generators/ide-gradle/proto_dependencies.jinja +++ b/build/export_generators/ide-gradle/proto_dependencies.jinja @@ -2,7 +2,7 @@ dependencies { {%- for library in target.consumer if library.classpath -%} {%- if library.prebuilt and library.jar and (library.type != "contrib" or target.handler.build_contribs) %} - implementation(files("$arcadia_root/{{ library.jar }}")) + implementation(files("$output_root/{{ library.jar }}")) {%- else -%} {%- set classpath = library.classpath -%} {%- if classpath|replace('"','') == classpath -%} diff --git a/build/export_generators/ide-gradle/proto_plugins.jinja b/build/export_generators/ide-gradle/proto_plugins.jinja index 8bce751c965..28b9910492a 100644 --- a/build/export_generators/ide-gradle/proto_plugins.jinja +++ b/build/export_generators/ide-gradle/proto_plugins.jinja @@ -1,7 +1,7 @@ {#- empty string #} plugins { id("java-library") - id("com.google.protobuf") version "0.9.5" + id("com.google.protobuf") version "0.9.4" {%- if publish %} `maven-publish` `signing` diff --git a/build/export_generators/ide-gradle/source_sets.jinja b/build/export_generators/ide-gradle/source_sets.jinja index 70e6795c414..a0a20e8d962 100644 --- a/build/export_generators/ide-gradle/source_sets.jinja +++ b/build/export_generators/ide-gradle/source_sets.jinja @@ -61,7 +61,7 @@ sourceSets { {%- set resdir_glob = split(resource_set, ':', 2) -%} {%- if resdir_glob[0][0] == "/" -%} {#- Absolute path in glob -#} -{%- set srcdir = resdir_glob[0] -%} +{%- set resdir = resdir_glob[0] -%} {%- else -%} {%- set resdir = reldir + resdir_glob[0] -%} {%- endif -%} diff --git a/build/external_resources/gdb/resources.json b/build/external_resources/gdb/resources.json index ea1833d4ee2..f7b21801fad 100644 --- a/build/external_resources/gdb/resources.json +++ b/build/external_resources/gdb/resources.json @@ -7,10 +7,10 @@ "uri": "sbr:3833498694" }, "linux-aarch64": { - "uri": "sbr:8784096409" + "uri": "sbr:8803622849" }, "linux-x86_64": { - "uri": "sbr:8784094118" + "uri": "sbr:8803613328" } } } diff --git a/build/external_resources/ymake/public.resources.json b/build/external_resources/ymake/public.resources.json index 5a77bbc24d9..2db330a1d5b 100644 --- a/build/external_resources/ymake/public.resources.json +++ b/build/external_resources/ymake/public.resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:8736412769" + "uri": "sbr:8815664221" }, "darwin-arm64": { - "uri": "sbr:8736411948" + "uri": "sbr:8815662661" }, "linux": { - "uri": "sbr:8736414621" + "uri": "sbr:8815666753" }, "linux-aarch64": { - "uri": "sbr:8736411209" + "uri": "sbr:8815660959" }, "win32": { - "uri": "sbr:8736413688" + "uri": "sbr:8815665372" } } } diff --git a/build/external_resources/ymake/resources.json b/build/external_resources/ymake/resources.json index 50467843c36..e52cc47f4e6 100644 --- a/build/external_resources/ymake/resources.json +++ b/build/external_resources/ymake/resources.json @@ -1,19 +1,19 @@ { "by_platform": { "darwin": { - "uri": "sbr:8736398606" + "uri": "sbr:8815667970" }, "darwin-arm64": { - "uri": "sbr:8736397686" + "uri": "sbr:8815666411" }, "linux": { - "uri": "sbr:8736400297" + "uri": "sbr:8815670616" }, "linux-aarch64": { - "uri": "sbr:8736396712" + "uri": "sbr:8815665005" }, "win32": { - "uri": "sbr:8736399378" + "uri": "sbr:8815669272" } } } diff --git a/build/mapping.conf.json b/build/mapping.conf.json index 5ecd508a105..ad8dce755c1 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -540,6 +540,10 @@ "8546855765": "{registry_endpoint}/8546855765", "8580453620": "{registry_endpoint}/8580453620", "8689590287": "{registry_endpoint}/8689590287", + "8764605919": "{registry_endpoint}/8764605919", + "8797051845": "{registry_endpoint}/8797051845", + "8813611998": "{registry_endpoint}/8813611998", + "8850031153": "{registry_endpoint}/8850031153", "5486731632": "{registry_endpoint}/5486731632", "5514350352": "{registry_endpoint}/5514350352", "5514360398": "{registry_endpoint}/5514360398", @@ -768,6 +772,8 @@ "8601993004": "{registry_endpoint}/8601993004", "8680458709": "{registry_endpoint}/8680458709", "8736412769": "{registry_endpoint}/8736412769", + "8796259581": "{registry_endpoint}/8796259581", + "8815664221": "{registry_endpoint}/8815664221", "5766171800": "{registry_endpoint}/5766171800", "5805430761": "{registry_endpoint}/5805430761", "5829025456": "{registry_endpoint}/5829025456", @@ -850,6 +856,8 @@ "8601992132": "{registry_endpoint}/8601992132", "8680458278": "{registry_endpoint}/8680458278", "8736411948": "{registry_endpoint}/8736411948", + "8796258690": "{registry_endpoint}/8796258690", + "8815662661": "{registry_endpoint}/8815662661", "5766173070": "{registry_endpoint}/5766173070", "5805432830": "{registry_endpoint}/5805432830", "5829031598": "{registry_endpoint}/5829031598", @@ -932,6 +940,8 @@ "8601994579": "{registry_endpoint}/8601994579", "8680459548": "{registry_endpoint}/8680459548", "8736414621": "{registry_endpoint}/8736414621", + "8796261345": "{registry_endpoint}/8796261345", + "8815666753": "{registry_endpoint}/8815666753", "5766171341": "{registry_endpoint}/5766171341", "5805430188": "{registry_endpoint}/5805430188", "5829023352": "{registry_endpoint}/5829023352", @@ -1014,6 +1024,8 @@ "8601991137": "{registry_endpoint}/8601991137", "8680457575": "{registry_endpoint}/8680457575", "8736411209": "{registry_endpoint}/8736411209", + "8796257699": "{registry_endpoint}/8796257699", + "8815660959": "{registry_endpoint}/8815660959", "8270821739": "{registry_endpoint}/8270821739", "8295446553": "{registry_endpoint}/8295446553", "8326170338": "{registry_endpoint}/8326170338", @@ -1026,6 +1038,8 @@ "8601993868": "{registry_endpoint}/8601993868", "8680458980": "{registry_endpoint}/8680458980", "8736413688": "{registry_endpoint}/8736413688", + "8796260519": "{registry_endpoint}/8796260519", + "8815665372": "{registry_endpoint}/8815665372", "5766172695": "{registry_endpoint}/5766172695", "5805432230": "{registry_endpoint}/5805432230", "5829029743": "{registry_endpoint}/5829029743", @@ -1109,6 +1123,7 @@ "6447362348": "{registry_endpoint}/6447362348", "6133337898": "{registry_endpoint}/6133337898", "8509776935": "{registry_endpoint}/8509776935", + "8803622849": "{registry_endpoint}/8803622849", "6164128408": "{registry_endpoint}/6164128408", "6682380912": "{registry_endpoint}/6682380912", "6478494211": "{registry_endpoint}/6478494211", @@ -1117,6 +1132,7 @@ "6447316775": "{registry_endpoint}/6447316775", "6133419349": "{registry_endpoint}/6133419349", "8509757921": "{registry_endpoint}/8509757921", + "8803613328": "{registry_endpoint}/8803613328", "6785543406": "{registry_endpoint}/6785543406", "1277521710": "{registry_endpoint}/1277521710", "1812152858": "{registry_endpoint}/1812152858", @@ -1962,6 +1978,10 @@ "8546855765": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "8580453620": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "8689590287": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", + "8764605919": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", + "8797051845": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", + "8813611998": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", + "8850031153": "devtools/ya/test/programs/test_tool/bin/test_tool for linux", "5486731632": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", "5514350352": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", "5514360398": "devtools/ya/test/programs/test_tool/bin3/test_tool3 for linux", @@ -2190,6 +2210,8 @@ "8601993004": "devtools/ymake/bin/ymake for darwin", "8680458709": "devtools/ymake/bin/ymake for darwin", "8736412769": "devtools/ymake/bin/ymake for darwin", + "8796259581": "devtools/ymake/bin/ymake for darwin", + "8815664221": "devtools/ymake/bin/ymake for darwin", "5766171800": "devtools/ymake/bin/ymake for darwin-arm64", "5805430761": "devtools/ymake/bin/ymake for darwin-arm64", "5829025456": "devtools/ymake/bin/ymake for darwin-arm64", @@ -2272,6 +2294,8 @@ "8601992132": "devtools/ymake/bin/ymake for darwin-arm64", "8680458278": "devtools/ymake/bin/ymake for darwin-arm64", "8736411948": "devtools/ymake/bin/ymake for darwin-arm64", + "8796258690": "devtools/ymake/bin/ymake for darwin-arm64", + "8815662661": "devtools/ymake/bin/ymake for darwin-arm64", "5766173070": "devtools/ymake/bin/ymake for linux", "5805432830": "devtools/ymake/bin/ymake for linux", "5829031598": "devtools/ymake/bin/ymake for linux", @@ -2354,6 +2378,8 @@ "8601994579": "devtools/ymake/bin/ymake for linux", "8680459548": "devtools/ymake/bin/ymake for linux", "8736414621": "devtools/ymake/bin/ymake for linux", + "8796261345": "devtools/ymake/bin/ymake for linux", + "8815666753": "devtools/ymake/bin/ymake for linux", "5766171341": "devtools/ymake/bin/ymake for linux-aarch64", "5805430188": "devtools/ymake/bin/ymake for linux-aarch64", "5829023352": "devtools/ymake/bin/ymake for linux-aarch64", @@ -2436,6 +2462,8 @@ "8601991137": "devtools/ymake/bin/ymake for linux-aarch64", "8680457575": "devtools/ymake/bin/ymake for linux-aarch64", "8736411209": "devtools/ymake/bin/ymake for linux-aarch64", + "8796257699": "devtools/ymake/bin/ymake for linux-aarch64", + "8815660959": "devtools/ymake/bin/ymake for linux-aarch64", "8270821739": "devtools/ymake/bin/ymake for win32", "8295446553": "devtools/ymake/bin/ymake for win32", "8326170338": "devtools/ymake/bin/ymake for win32", @@ -2448,6 +2476,8 @@ "8601993868": "devtools/ymake/bin/ymake for win32", "8680458980": "devtools/ymake/bin/ymake for win32", "8736413688": "devtools/ymake/bin/ymake for win32", + "8796260519": "devtools/ymake/bin/ymake for win32", + "8815665372": "devtools/ymake/bin/ymake for win32", "5766172695": "devtools/ymake/bin/ymake for win32-clang-cl", "5805432230": "devtools/ymake/bin/ymake for win32-clang-cl", "5829029743": "devtools/ymake/bin/ymake for win32-clang-cl", @@ -2531,6 +2561,7 @@ "6447362348": "gdb-14-linux-aarch64-9db71d8b25a56ee316036c53fd941934a95831a3", "6133337898": "gdb-14-linux-aarch64-b1fa9be28bbf4ee845d6a39a049c7b60018a3695", "8509776935": "gdb-14-linux-aarch64-b96bb13a4532bc0f19859cd0fd8590f36e4ac9f4", + "8803622849": "gdb-14-linux-aarch64-e086c237e91cadf80f94fc0b2c3268624f9e1058", "6164128408": "gdb-14-linux-x86_64-2f60b923acb68d45b101313954b70779c13a19b8", "6682380912": "gdb-14-linux-x86_64-533b7767fda1a4ec616af8b575d10c4694ae1f2f", "6478494211": "gdb-14-linux-x86_64-67ef8a9f1868bf62959ac3a14d552999a108ed38", @@ -2539,6 +2570,7 @@ "6447316775": "gdb-14-linux-x86_64-9db71d8b25a56ee316036c53fd941934a95831a3", "6133419349": "gdb-14-linux-x86_64-b1fa9be28bbf4ee845d6a39a049c7b60018a3695", "8509757921": "gdb-14-linux-x86_64-b96bb13a4532bc0f19859cd0fd8590f36e4ac9f4", + "8803613328": "gdb-14-linux-x86_64-e086c237e91cadf80f94fc0b2c3268624f9e1058", "6785543406": "grpc-java-linux-x86_64-231a5a5d0bf5248caf1a9539ef29dd2b20a9d05d", "1277521710": "infra/kernel/tools/atop/build/atop-static.tar.gz", "1812152858": "junk/zubchick/buf/buf for linux", diff --git a/build/platform/java/protoc/resources.json b/build/platform/java/protoc/resources.json deleted file mode 100644 index 993c58da48d..00000000000 --- a/build/platform/java/protoc/resources.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "by_platform": { - "darwin-arm64": { - "uri": "sbr:7805450611" - }, - "darwin-x86_64": { - "uri": "sbr:7805450231" - }, - "linux-x86_64": { - "uri": "sbr:7805449010" - }, - "linux-aarch64": { - "uri": "sbr:7805449449" - }, - "win32-x86_64": { - "uri": "sbr:7805449878" - } - } -} diff --git a/build/platform/java/protoc/ya.make b/build/platform/java/protoc/ya.make deleted file mode 100644 index ef0afecefd8..00000000000 --- a/build/platform/java/protoc/ya.make +++ /dev/null @@ -1,9 +0,0 @@ -RESOURCES_LIBRARY() - -IF (NOT HOST_OS_DARWIN AND NOT HOST_OS_LINUX AND NOT HOST_OS_WINDOWS) - MESSAGE(FATAL_ERROR Unsupported host platform for java protoc) -ENDIF() - -DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE_BY_JSON(JAVA_PROTOC resources.json) - -END() diff --git a/build/platform/test_tool/host.ya.make.inc b/build/platform/test_tool/host.ya.make.inc index 4ba3497ba94..f9f79cc59da 100644 --- a/build/platform/test_tool/host.ya.make.inc +++ b/build/platform/test_tool/host.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8689635281) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8850053943) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8689630811) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8850052492) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8689643844) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8850056916) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8689626339) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8850050057) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8689639545) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8850055427) ENDIF() diff --git a/build/platform/test_tool/host_os.ya.make.inc b/build/platform/test_tool/host_os.ya.make.inc index 17b38edaa57..c660053e0a8 100644 --- a/build/platform/test_tool/host_os.ya.make.inc +++ b/build/platform/test_tool/host_os.ya.make.inc @@ -1,12 +1,12 @@ IF (HOST_OS_DARWIN AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8689582949) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8850029905) ELSEIF (HOST_OS_DARWIN AND HOST_ARCH_ARM64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8689578500) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8850028995) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8689590287) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8850031153) ELSEIF (HOST_OS_LINUX AND HOST_ARCH_AARCH64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8689574454) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8850028133) ELSEIF (HOST_OS_WINDOWS AND HOST_ARCH_X86_64) - DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8689586690) + DECLARE_EXTERNAL_RESOURCE(TEST_TOOL_HOST sbr:8850030615) ENDIF() diff --git a/build/platform/yfm/resources.json b/build/platform/yfm/resources.json index a27dd38d54e..8e46986187c 100644 --- a/build/platform/yfm/resources.json +++ b/build/platform/yfm/resources.json @@ -1,16 +1,16 @@ { "by_platform": { "win32-x86_64": { - "uri": "sbr:8778298308" + "uri": "sbr:8835051443" }, "darwin-x86_64": { - "uri": "sbr:8778297068" + "uri": "sbr:8835048369" }, "linux-x86_64": { - "uri": "sbr:8778295807" + "uri": "sbr:8835045019" }, "darwin-arm64": { - "uri": "sbr:8778297068" + "uri": "sbr:8835048369" } } } diff --git a/build/plugins/_dart_fields.py b/build/plugins/_dart_fields.py index eba83ebbb5e..17d0c546a0d 100644 --- a/build/plugins/_dart_fields.py +++ b/build/plugins/_dart_fields.py @@ -1141,6 +1141,7 @@ class TestFiles: 'maps/renderer/libs/terrain', 'maps/renderer/libs/threading', 'maps/renderer/libs/vec', + 'maps/renderer/libs/yql', 'maps/renderer/libs/yt', 'maps/renderer/tilemill', 'maps/renderer/tools/fontograph', diff --git a/build/plugins/nots.py b/build/plugins/nots.py index 5f9c4480ac4..6a96b796ee0 100644 --- a/build/plugins/nots.py +++ b/build/plugins/nots.py @@ -909,7 +909,7 @@ def on_ts_test_for_configure( test_files = df.TestFiles.ts_test_srcs(unit, (), {})[df.TestFiles.KEY] if not test_files: - ymake.report_configure_error("No tests found") + ymake.report_configure_error(f"No tests found for {test_runner}") return from lib.nots.package_manager import constants diff --git a/build/sysincl/macro.yml b/build/sysincl/macro.yml index 0fd2bdf2f75..5b08c600786 100644 --- a/build/sysincl/macro.yml +++ b/build/sysincl/macro.yml @@ -781,18 +781,6 @@ - BOOST_PP_ITERATE(): - contrib/restricted/boost/variant/include/boost/variant/detail/substitute.hpp -- source_filter: "^contrib/libs/gperftools" - includes: - - STACKTRACE_INL_HEADER: - - contrib/libs/gperftools/srcstacktrace_win32-inl.inc - - contrib/libs/gperftools/srcstacktrace_generic-inl.inc - - contrib/libs/gperftools/srcstacktrace_emscripten-inl.inc - - contrib/libs/gperftools/srcstacktrace_x86-inl.inc - - contrib/libs/gperftools/srcstacktrace_powerpc-inl.inc - - contrib/libs/gperftools/srcstacktrace_aarch64-inl.inc - - contrib/libs/gperftools/srcstacktrace_riscv-inl.inc - - contrib/libs/gperftools/srcstacktrace_unimplemented-inl.inc - - source_filter: "^contrib/libs/lzo2" includes: - LZO_CODE_MATCH_INCLUDE_FILE: diff --git a/build/ymake.core.conf b/build/ymake.core.conf index 1c4d6ce2047..c36714a94cb 100644 --- a/build/ymake.core.conf +++ b/build/ymake.core.conf @@ -13,7 +13,7 @@ CPP_FAKEID=2024-01-23 GO_FAKEID=11100371 ANDROID_FAKEID=2023-05-17 CLANG_TIDY_FAKEID=2023-06-06 -CYTHON_FAKEID=16586418 +CYTHON_FAKEID=16618405 JAVA_FAKEID=14386852 PROTO_FAKEID=0 FBS_FAKEID=2024-03-13 @@ -2632,18 +2632,37 @@ macro _COPY_FILE_IMPL(TEXT[], AUTO_DST="", NOAUTO_DST="", OUTPUT_INCLUDES[], IND ### ### The file will be just copied if AUTO boolean parameter is not specified. You should explicitly ### mention it in SRCS under new name (or specify AUTO boolean parameter) for further processing. -macro COPY_FILE(File, Destination, AUTO?"AUTO_DST":"NOAUTO_DST", OUTPUT_INCLUDES[], INDUCED_DEPS[], TEXT?"TEXT":"$_COPY_FILE_CONTEXT") { - .CMD=$_COPY_FILE_IMPL($TEXT $File $AUTO $Destination OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS) - .SEM=$_COPY_FILE_IMPL($File $AUTO $Destination) -} +macro COPY_FILE(File, Destination, AUTO?"true":"false", OUTPUT_INCLUDES[], INDUCED_DEPS[], TEXT?"true":"false") { + # FIXME: the "TEXT" argument is effectively unused since _COPY_FILE_CONTEXT defaulting to TEXT + .CMD=$_COPY_FILE_CMD + .STRUCT_CMD=yes + .SEM=$_COPY_FILE_SEM + .STRUCT_SEM=yes +} +_COPY_FILE_CMD=$IF( \ + ${parse_bool:AUTO} \ + THEN $_COPY_FILE_IMPL(TEXT $File AUTO_DST $Destination OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS) \ + ELSE $_COPY_FILE_IMPL(TEXT $File NOAUTO_DST $Destination OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS) \ +) +_COPY_FILE_SEM=$IF( \ + ${parse_bool:AUTO} \ + THEN $_COPY_FILE_IMPL($File AUTO_DST $Destination) \ + ELSE $_COPY_FILE_IMPL($File NOAUTO_DST $Destination) \ +) ### @usage: COPY_FILE_WITH_CONTEXT(FILE DEST [AUTO] [OUTPUT_INCLUDES DEPS...]) ### ### Copy file to build root the same way as it is done for COPY_FILE, but also ### propagates the context of the source file. -macro COPY_FILE_WITH_CONTEXT(FILE, DEST, AUTO?"AUTO_DST":"NOAUTO_DST", OUTPUT_INCLUDES[], INDUCED_DEPS[]) { - .CMD=$_COPY_FILE_IMPL($FILE $AUTO $DEST OUTPUT_INCLUDES_INP $FILE OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS) +macro COPY_FILE_WITH_CONTEXT(FILE, DEST, AUTO?"true":"false", OUTPUT_INCLUDES[], INDUCED_DEPS[]) { + .CMD=$_COPY_FILE_WITH_CONTEXT_CMD + .STRUCT_CMD=yes } +_COPY_FILE_WITH_CONTEXT_CMD=$IF( \ + ${parse_bool:AUTO} \ + THEN $_COPY_FILE_IMPL($FILE AUTO_DST $DEST OUTPUT_INCLUDES_INP $FILE OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS) \ + ELSE $_COPY_FILE_IMPL($FILE NOAUTO_DST $DEST OUTPUT_INCLUDES_INP $FILE OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS) \ +) ### This is to join $ALL_RES_ and $EXT macro _ARF_HELPER(Args...) { @@ -4540,8 +4559,9 @@ macro DECLARE_IN_DIRS(var_prefix, PATTERN, SRCDIR="", RECURSIVE?"**/":"", EXCLUD ### For absolute paths use ${ARCADIA_ROOT} and ${ARCADIA_BUILD_ROOT}, or ### ${CURDIR} and ${BINDIR} which are expanded where the outputs are used. ### Note that Tool is always built for the host platform, so be careful to provide that tool can be built for all Arcadia major host platforms (Linux, MacOS and Windows). -macro RUN_PROGRAM(Tool, IN{input}[], IN_NOPARSE{input}[], OUT{output}[], OUT_NOAUTO{output}[], TOOL{tool}[], OUTPUT_INCLUDES[], INDUCED_DEPS[], IN_DEPS[], STDOUT="", STDOUT_NOAUTO="", CWD="", ENV[], Args...) { +macro RUN_PROGRAM(Tool, IN{input}[], IN_NOPARSE{input}[], OUT{output}[], OUT_NOAUTO{output}[], TOOL{tool}[], OUTPUT_INCLUDES[], INDUCED_DEPS[], IN_DEPS[], STDOUT="", STDOUT_NOAUTO="", CWD="", ENV{env}[], Args...) { .CMD=${cwd:CWD} ${env:ENV} ${tool:Tool} $Args ${hide;input:IN} ${hide;context=TEXT;input=TEXT:IN_NOPARSE} ${hide;input:IN_DEPS} ${hide;output_include:OUTPUT_INCLUDES} $INDUCED_DEPS ${hide;tool:TOOL} ${hide;output:OUT} ${hide;noauto;output:OUT_NOAUTO} ${stdout;output:STDOUT} ${stdout;noauto;output:STDOUT_NOAUTO} ${hide;kv:"p PR"} ${hide;kv:"pc yellow"} ${hide;kv:"show_out"} + .STRUCT_CMD=yes .SEM=custom_runs-ITEM && custom_runs-depends ${input:IN} ${context=TEXT;input=TEXT:IN_NOPARSE} ${tool:Tool} ${tool:TOOL} ${pre=&& custom_runs-env :ENV} && custom_runs-command ${tool:Tool} $Args ${pre=> :STDOUT} ${pre=> :STDOUT_NOAUTO} && custom_runs-outputs ${output:OUT} ${noauto;output:OUT_NOAUTO} ${output:STDOUT} ${noauto;output:STDOUT_NOAUTO} ${pre=&& custom_runs-cwd :CWD} } @@ -4568,6 +4588,7 @@ macro RUN_PROGRAM(Tool, IN{input}[], IN_NOPARSE{input}[], OUT{output}[], OUT_NOA ### ${CURDIR} and ${BINDIR} which are expanded where the outputs are used. macro RUN_LUA(ScriptPath, IN{input}[], IN_NOPARSE{input}[], OUT{output}[], OUT_NOAUTO{output}[], TOOL{tool}[], OUTPUT_INCLUDES[], INDUCED_DEPS[], STDOUT="", STDOUT_NOAUTO="", CWD="", ENV[], Args...) { .CMD=${cwd:CWD} ${env:ENV} $LUA_TOOL ${input:ScriptPath} $Args ${hide;input:IN} ${hide;context=TEXT;input=TEXT:IN_NOPARSE} ${hide;output_include:OUTPUT_INCLUDES} $INDUCED_DEPS ${hide;tool:TOOL} ${hide;output:OUT} ${hide;noauto;output:OUT_NOAUTO} ${stdout;output:STDOUT} ${stdout;noauto;output:STDOUT_NOAUTO} ${hide;kv:"p LU"} ${hide;kv:"pc yellow"} ${hide;kv:"show_out"} + .STRUCT_CMD=yes } # tag:python-specific @@ -4594,6 +4615,7 @@ macro RUN_LUA(ScriptPath, IN{input}[], IN_NOPARSE{input}[], OUT{output}[], OUT_N ### ${CURDIR} and ${BINDIR} which are expanded where the outputs are used. macro RUN_PYTHON3(ScriptPath, IN{input}[], IN_NOPARSE{input}[], OUT{output}[], OUT_NOAUTO{output}[], OUT_GLOBAL{output}[], TOOL{tool}[], OUTPUT_INCLUDES[], INDUCED_DEPS[], STDOUT="", STDOUT_NOAUTO="", CWD="", ENV[], Args...) { .CMD=${cwd:CWD} ${env:ENV} $YMAKE_PYTHON3 ${input:ScriptPath} $Args ${hide;input:IN} ${hide;context=TEXT;input=TEXT:IN_NOPARSE} ${hide;output_include:OUTPUT_INCLUDES} $INDUCED_DEPS ${hide;tool:TOOL} ${hide;output:OUT} ${hide;noauto;output:OUT_NOAUTO} ${hide;global;output:OUT_GLOBAL} ${stdout;output:STDOUT} ${stdout;noauto;output:STDOUT_NOAUTO} ${hide;kv:"p PY"} ${hide;kv:"pc yellow"} ${hide;kv:"show_out"} + .STRUCT_CMD=yes .SEM=custom_runs-ITEM && custom_runs-depends ${input:IN} ${context=TEXT;input=TEXT:IN_NOPARSE} ${input:ScriptPath} ${tool:TOOL} ${pre=&& custom_runs-env :ENV} && custom_runs-command python3 ${input:ScriptPath} $Args ${pre=> :STDOUT} ${pre=> :STDOUT_NOAUTO} && custom_runs-outputs ${output:OUT} ${noauto;output:OUT_NOAUTO} ${global;output:OUT_GLOBAL} ${output:STDOUT} ${noauto;output:STDOUT_NOAUTO} ${pre=&& custom_runs-cwd :CWD} && custom_runs-cmake_packages-ITEM && custom_runs-cmake_packages-name Python3 } @@ -4621,7 +4643,8 @@ macro RUN_PYTHON3(ScriptPath, IN{input}[], IN_NOPARSE{input}[], OUT{output}[], O ### ${CURDIR} and ${BINDIR} which are expanded where the outputs are used. ### Note that Tool is always built for the host platform, so be careful to provide that tool can be built for all Arcadia major host platforms (Linux, MacOS and Windows). macro RUN_PY3_PROGRAM(Tool, IN{input}[], IN_NOPARSE{input}[], OUT{output}[], OUT_NOAUTO{output}[], TOOL{tool}[], OUTPUT_INCLUDES[], INDUCED_DEPS[], STDOUT="", STDOUT_NOAUTO="", CWD="", ENV[], Args...) { - .CMD=$RUN_PROGRAM($Tool $Args ${pre=IN :IN} ${pre=IN_NOPARSE :IN_NOPARSE} ${pre=OUT :OUT} ${pre=OUT_NOAUTO :OUT_NOAUTO} ${pre=TOOL :TOOL} ${pre=OUTPUT_INCLUDES :OUTPUT_INCLUDES} ${pre=INDUCED_DEPS :INDUCED_DEPS} ${pre=STDOUT :STDOUT} ${pre=STDOUT_NOAUTO :STDOUT_NOAUTO} ${pre=CWD :CWD} ${pre=ENV :ENV}) + .CMD=$RUN_PROGRAM($Tool $Args IN $IN IN_NOPARSE $IN_NOPARSE OUT $OUT OUT_NOAUTO $OUT_NOAUTO TOOL $TOOL OUTPUT_INCLUDES $OUTPUT_INCLUDES INDUCED_DEPS $INDUCED_DEPS STDOUT $STDOUT STDOUT_NOAUTO $STDOUT_NOAUTO CWD $CWD ENV $ENV) + .STRUCT_CMD=yes .SEM=$RUN_PYTHON3(${ARCADIA_ROOT}/${Tool}/__main__.py $Args ${pre=IN :IN} ${pre=IN_NOPARSE :IN_NOPARSE} ${pre=OUT :OUT} ${pre=OUT_NOAUTO :OUT_NOAUTO} ${pre=TOOL :TOOL} ${pre=OUTPUT_INCLUDES :OUTPUT_INCLUDES} ${pre=INDUCED_DEPS :INDUCED_DEPS} ${pre=STDOUT :STDOUT} ${pre=STDOUT_NOAUTO :STDOUT_NOAUTO} ${pre=CWD :CWD} ${pre=ENV :ENV}) } @@ -4629,6 +4652,7 @@ macro RUN_PY3_PROGRAM(Tool, IN{input}[], IN_NOPARSE{input}[], OUT{output}[], OUT macro _RUN_ANTLR_BASE(IN{input}[], IN_NOPARSE{input}[], OUT{output}[], OUT_NOAUTO{output}[], OUTPUT_INCLUDES[], INDUCED_DEPS[], TOOL[], STDOUT="", STDOUT_NOAUTO="", CWD="", JAR{input}[], SEM="run_java", ENV[], HIDE_OUTPUT?"stderr2stdout":"stdout2stderr", Args...) { PEERDIR(build/platform/java/jdk $JDK_RESOURCE_PEERDIR) .CMD=${cwd:CWD} ${env:ENV} $YMAKE_PYTHON ${input;pre=build/scripts/;suf=.py:HIDE_OUTPUT} $JDK_RESOURCE/bin/java -jar ${input:JAR} $Args ${hide;tool:TOOL} ${hide;input:IN} ${hide;context=TEXT;input=TEXT:IN_NOPARSE} ${hide;output_include:OUTPUT_INCLUDES} $INDUCED_DEPS ${hide;output:OUT} ${hide;noauto;output:OUT_NOAUTO} ${stdout;output:STDOUT} ${stdout;noauto;output:STDOUT_NOAUTO} ${hide;kv:"p JV"} ${hide;kv:"pc light-blue"} ${hide;kv:"show_out"} + .STRUCT_CMD=yes .SEM=custom_runs-ITEM && custom_runs-depends ${input:IN} && custom_runs-command $SEM && custom_runs-command $Args && custom_runs-outputs ${output:OUT} ${noauto;output:OUT_NOAUTO} ${pre=&& custom_runs-cwd :CWD} } diff --git a/contrib/libs/cxxsupp/libcxxcuda11/ya.make b/contrib/libs/cxxsupp/libcxxcuda11/ya.make index e3f58f3349c..7430ee6b7e7 100644 --- a/contrib/libs/cxxsupp/libcxxcuda11/ya.make +++ b/contrib/libs/cxxsupp/libcxxcuda11/ya.make @@ -18,11 +18,6 @@ VERSION(2023-10-05) ORIGINAL_SOURCE(https://github.com/llvm/llvm-project/archive/dc129d6f715cf83a2072fc8de8b4e4c70bca6935.tar.gz) -SUBSCRIBER( - g:cpp-committee - g:cpp-contrib -) - ADDINCL( GLOBAL contrib/libs/cxxsupp/libcxxcuda11/include contrib/libs/cxxsupp/libcxxcuda11/src diff --git a/contrib/libs/cxxsupp/ya.make b/contrib/libs/cxxsupp/ya.make index 3bf5aa4ce3c..17c7e592e36 100644 --- a/contrib/libs/cxxsupp/ya.make +++ b/contrib/libs/cxxsupp/ya.make @@ -34,6 +34,7 @@ IF (NOT USE_STL_SYSTEM) RECURSE( libcxx libcxxabi + libcxxcuda11 libcxxmsvc libcxxrt openmp diff --git a/contrib/libs/ftxui/.yandex_meta/__init__.py b/contrib/libs/ftxui/.yandex_meta/__init__.py new file mode 100644 index 00000000000..77b5f726f43 --- /dev/null +++ b/contrib/libs/ftxui/.yandex_meta/__init__.py @@ -0,0 +1,12 @@ +from devtools.yamaker.project import CMakeNinjaNixProject + +ftxui = CMakeNinjaNixProject( + owners=["segoon", "g:taxi-common"], + nixattr="ftxui", + flags=[ + "-DFTXUI_BUILD_EXAMPLES=OFF", + ], + disable_includes=["emscripten.h"], + put_with={"ftxui-component": ["ftxui-dom", "ftxui-screen"]}, + arcdir="contrib/libs/ftxui", +) diff --git a/contrib/libs/ftxui/.yandex_meta/devtools.copyrights.report b/contrib/libs/ftxui/.yandex_meta/devtools.copyrights.report new file mode 100644 index 00000000000..a5e98f6c5ab --- /dev/null +++ b/contrib/libs/ftxui/.yandex_meta/devtools.copyrights.report @@ -0,0 +1,227 @@ +# File format ($ symbol means the beginning of a line): +# +# $ # this message +# $ # ======================= +# $ # comments (all commentaries should starts with some number of spaces and # symbol) +# $ IGNORE_FILES {file1.ext1} {file2.ext2} - (optional) ignore listed files when generating license macro and credits +# $ RENAME {original license id} TO {new license id} # user comments - (optional) use {new license id} instead {original license id} in ya.make files +# $ # user comments +# $ +# ${action} {license id} {license text hash} +# $BELONGS ./ya/make/file/relative/path/1/ya.make ./ya/make/2/ya.make +# ${all_file_action} filename +# $ # user commentaries (many lines) +# $ generated description - files with this license, license text... (some number of lines that starts with some number of spaces, do not modify) +# ${action} {license spdx} {license text hash} +# $BELONGS ./ya/make/file/relative/path/3/ya.make +# ${all_file_action} filename +# $ # user commentaries +# $ generated description +# $ ... +# +# You can modify action, all_file_action and add commentaries +# Available actions: +# keep - keep license in contrib and use in credits +# skip - skip license +# remove - remove all files with this license +# rename - save license text/links into licenses texts file, but not store SPDX into LINCENSE macro. You should store correct license id into devtools.license.spdx.txt file +# +# {all file action} records will be generated when license text contains filename that exists on filesystem (in contrib directory) +# We suppose that that files can contain some license info +# Available all file actions: +# FILE_IGNORE - ignore file (do nothing) +# FILE_INCLUDE - include all file data into licenses text file +# ======================= + +KEEP COPYRIGHT_SERVICE_LABEL 4171695d4c508b83336ebaf6e7826344 +BELONGS ya.make + License text: + // Copyright 2024 Arthur Sonzogni. All rights reserved. + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + include/ftxui/dom/selection.hpp [1:1] + include/ftxui/screen/image.hpp [1:1] + include/ftxui/screen/pixel.hpp [1:1] + src/ftxui/dom/selection.cpp [1:1] + src/ftxui/dom/selection_style.cpp [1:1] + +KEEP COPYRIGHT_SERVICE_LABEL 77a150e6b5fd683af533fccf8202f0e2 +BELONGS ya.make + License text: + // Copyright 2020 Arthur Sonzogni. All rights reserved. + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + include/ftxui/component/captured_mouse.hpp [1:1] + include/ftxui/component/component_base.hpp [1:1] + include/ftxui/component/event.hpp [1:1] + include/ftxui/component/mouse.hpp [1:1] + include/ftxui/component/receiver.hpp [1:1] + include/ftxui/component/screen_interactive.hpp [1:1] + include/ftxui/dom/elements.hpp [1:1] + include/ftxui/dom/node.hpp [1:1] + include/ftxui/dom/requirement.hpp [1:1] + include/ftxui/dom/take_any_args.hpp [1:1] + include/ftxui/screen/box.hpp [1:1] + include/ftxui/screen/color.hpp [1:1] + include/ftxui/screen/color_info.hpp [1:1] + include/ftxui/screen/screen.hpp [1:1] + include/ftxui/screen/string.hpp [1:1] + include/ftxui/screen/terminal.hpp [1:1] + include/ftxui/util/autoreset.hpp [1:1] + include/ftxui/util/ref.hpp [1:1] + src/ftxui/component/button.cpp [1:1] + src/ftxui/component/checkbox.cpp [1:1] + src/ftxui/component/component.cpp [1:1] + src/ftxui/component/container.cpp [1:1] + src/ftxui/component/event.cpp [1:1] + src/ftxui/component/menu.cpp [1:1] + src/ftxui/component/radiobox.cpp [1:1] + src/ftxui/component/screen_interactive.cpp [1:1] + src/ftxui/component/slider.cpp [1:1] + src/ftxui/component/terminal_input_parser.cpp [1:1] + src/ftxui/component/terminal_input_parser.hpp [1:1] + src/ftxui/dom/automerge.cpp [1:1] + src/ftxui/dom/blink.cpp [1:1] + src/ftxui/dom/bold.cpp [1:1] + src/ftxui/dom/border.cpp [1:1] + src/ftxui/dom/clear_under.cpp [1:1] + src/ftxui/dom/color.cpp [1:1] + src/ftxui/dom/composite_decorator.cpp [1:1] + src/ftxui/dom/dbox.cpp [1:1] + src/ftxui/dom/dim.cpp [1:1] + src/ftxui/dom/flex.cpp [1:1] + src/ftxui/dom/flexbox.cpp [1:1] + src/ftxui/dom/flexbox_config.cpp [1:1] + src/ftxui/dom/flexbox_helper.hpp [1:1] + src/ftxui/dom/focus.cpp [1:1] + src/ftxui/dom/frame.cpp [1:1] + src/ftxui/dom/gauge.cpp [1:1] + src/ftxui/dom/graph.cpp [1:1] + src/ftxui/dom/gridbox.cpp [1:1] + src/ftxui/dom/hbox.cpp [1:1] + src/ftxui/dom/inverted.cpp [1:1] + src/ftxui/dom/node.cpp [1:1] + src/ftxui/dom/node_decorator.cpp [1:1] + src/ftxui/dom/node_decorator.hpp [1:1] + src/ftxui/dom/paragraph.cpp [1:1] + src/ftxui/dom/reflect.cpp [1:1] + src/ftxui/dom/separator.cpp [1:1] + src/ftxui/dom/size.cpp [1:1] + src/ftxui/dom/spinner.cpp [1:1] + src/ftxui/dom/text.cpp [1:1] + src/ftxui/dom/underlined.cpp [1:1] + src/ftxui/dom/util.cpp [1:1] + src/ftxui/dom/vbox.cpp [1:1] + src/ftxui/screen/box.cpp [1:1] + src/ftxui/screen/color.cpp [1:1] + src/ftxui/screen/color_info.cpp [1:1] + src/ftxui/screen/image.cpp [1:1] + src/ftxui/screen/screen.cpp [1:1] + src/ftxui/screen/string.cpp [1:1] + src/ftxui/screen/terminal.cpp [1:1] + +KEEP COPYRIGHT_SERVICE_LABEL 93d6be06c4778e8c36ab73cda0a8b1b6 +BELONGS ya.make + License text: + // Copyright 2023 Arthur Sonzogni. All rights reserved. + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + include/ftxui/dom/direction.hpp [1:1] + include/ftxui/dom/linear_gradient.hpp [1:1] + src/ftxui/component/window.cpp [1:1] + src/ftxui/dom/hyperlink.cpp [1:1] + src/ftxui/dom/linear_gradient.cpp [1:1] + src/ftxui/dom/strikethrough.cpp [1:1] + src/ftxui/dom/underlined_double.cpp [1:1] + src/ftxui/screen/string_internal.hpp [1:1] + +KEEP COPYRIGHT_SERVICE_LABEL 9c98a8e31090498eefef43ce64b1d472 +BELONGS ya.make + License text: + // Copyright 2021 Arthur Sonzogni. All rights reserved. + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + include/ftxui/component/component.hpp [1:1] + include/ftxui/component/component_options.hpp [1:1] + include/ftxui/dom/canvas.hpp [1:1] + include/ftxui/dom/deprecated.hpp [1:1] + include/ftxui/dom/flexbox_config.hpp [1:1] + include/ftxui/dom/table.hpp [1:1] + include/ftxui/screen/deprecated.hpp [1:1] + src/ftxui/component/catch_event.cpp [1:1] + src/ftxui/component/collapsible.cpp [1:1] + src/ftxui/component/dropdown.cpp [1:1] + src/ftxui/component/maybe.cpp [1:1] + src/ftxui/component/renderer.cpp [1:1] + src/ftxui/component/resizable_split.cpp [1:1] + src/ftxui/dom/box_helper.cpp [1:1] + src/ftxui/dom/box_helper.hpp [1:1] + src/ftxui/dom/canvas.cpp [1:1] + src/ftxui/dom/flexbox_helper.cpp [1:1] + src/ftxui/dom/scroll_indicator.cpp [1:1] + src/ftxui/dom/table.cpp [1:1] + +KEEP COPYRIGHT_SERVICE_LABEL aca04e53f0e1ecbb7d19857e1cdfc607 +BELONGS ya.make + License text: + // Copyright (c) 2011, Auerhaus Development, LLC + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + src/ftxui/component/animation.cpp [20:20] + +KEEP COPYRIGHT_SERVICE_LABEL e5499a91f4cbd29bff1d70364b5776c5 +BELONGS ya.make + License text: + Copyright (c) 2019 Arthur Sonzogni. + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + LICENSE [3:3] + +KEEP COPYRIGHT_SERVICE_LABEL f55239307282d9c3ee551812c3dca300 +BELONGS ya.make + License text: + // Copyright 2025 Arthur Sonzogni. All rights reserved. + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + src/ftxui/dom/italic.cpp [1:1] + +KEEP COPYRIGHT_SERVICE_LABEL fbfd3582cf1e2879c92ce92071ec2c70 +BELONGS ya.make + License text: + // Copyright 2022 Arthur Sonzogni. All rights reserved. + Scancode info: + Original SPDX id: COPYRIGHT_SERVICE_LABEL + Score : 100.00 + Match type : COPYRIGHT + Files with this license: + include/ftxui/component/animation.hpp [1:1] + include/ftxui/component/loop.hpp [1:1] + include/ftxui/component/task.hpp [1:1] + src/ftxui/component/component_options.cpp [1:1] + src/ftxui/component/hoverable.cpp [1:1] + src/ftxui/component/input.cpp [1:1] + src/ftxui/component/loop.cpp [1:1] + src/ftxui/component/modal.cpp [1:1] + src/ftxui/component/util.cpp [1:1] + src/ftxui/screen/util.hpp [1:1] diff --git a/contrib/libs/ftxui/.yandex_meta/devtools.licenses.report b/contrib/libs/ftxui/.yandex_meta/devtools.licenses.report new file mode 100644 index 00000000000..b64bc87d2bc --- /dev/null +++ b/contrib/libs/ftxui/.yandex_meta/devtools.licenses.report @@ -0,0 +1,237 @@ +# File format ($ symbol means the beginning of a line): +# +# $ # this message +# $ # ======================= +# $ # comments (all commentaries should starts with some number of spaces and # symbol) +# $ IGNORE_FILES {file1.ext1} {file2.ext2} - (optional) ignore listed files when generating license macro and credits +# $ RENAME {original license id} TO {new license id} # user comments - (optional) use {new license id} instead {original license id} in ya.make files +# $ # user comments +# $ +# ${action} {license id} {license text hash} +# $BELONGS ./ya/make/file/relative/path/1/ya.make ./ya/make/2/ya.make +# ${all_file_action} filename +# $ # user commentaries (many lines) +# $ generated description - files with this license, license text... (some number of lines that starts with some number of spaces, do not modify) +# ${action} {license spdx} {license text hash} +# $BELONGS ./ya/make/file/relative/path/3/ya.make +# ${all_file_action} filename +# $ # user commentaries +# $ generated description +# $ ... +# +# You can modify action, all_file_action and add commentaries +# Available actions: +# keep - keep license in contrib and use in credits +# skip - skip license +# remove - remove all files with this license +# rename - save license text/links into licenses texts file, but not store SPDX into LINCENSE macro. You should store correct license id into devtools.license.spdx.txt file +# +# {all file action} records will be generated when license text contains filename that exists on filesystem (in contrib directory) +# We suppose that that files can contain some license info +# Available all file actions: +# FILE_IGNORE - ignore file (do nothing) +# FILE_INCLUDE - include all file data into licenses text file +# ======================= + +SKIP CC-BY-3.0 AND MIT 563237d11bdd0adf26ab44cfb87e9eb0 +BELONGS ya.make + License text: + <a href="http://opensource.org/licenses/MIT"><img src="https://img.shields.io/github/license/arthursonzogni/FTXUI?color=black"></img></a> + Scancode info: + Original SPDX id: CC-BY-3.0 + Score : 19.35 + Match type : NOTICE + Links : http://creativecommons.org/licenses/by/3.0/, http://creativecommons.org/licenses/by/3.0/legalcode, https://spdx.org/licenses/CC-BY-3.0 + Files with this license: + README.md [5:5] + Scancode info: + Original SPDX id: MIT + Score : 19.35 + Match type : NOTICE + Links : http://opensource.org/licenses/mit-license.php, https://spdx.org/licenses/MIT + Files with this license: + README.md [5:5] + +KEEP MIT 58c44e43474457ea5d8a7dc9ed7c0b76 +BELONGS ya.make +FILE_INCLUDE LICENSE found in files: src/ftxui/component/input.cpp at line 3, src/ftxui/dom/flexbox.cpp at line 3, src/ftxui/dom/flexbox_helper.cpp at line 3 + License text: + // Use of this source code is governed by the MIT license that can be found in + // the LICENSE file. + Scancode info: + Original SPDX id: MIT + Score : 100.00 + Match type : NOTICE + Links : http://opensource.org/licenses/mit-license.php, https://spdx.org/licenses/MIT + Files with this license: + src/ftxui/component/input.cpp [2:3] + src/ftxui/dom/flexbox.cpp [2:3] + src/ftxui/dom/flexbox_helper.cpp [2:3] + +KEEP MIT 5debb370f50e1dfd24ff5144233a2ef6 +BELONGS ya.make + Note: matched license text is too long. Read it in the source files. + Scancode info: + Original SPDX id: MIT + Score : 100.00 + Match type : TEXT + Links : http://opensource.org/licenses/mit-license.php, https://spdx.org/licenses/MIT + Files with this license: + LICENSE [5:21] + +KEEP WTFPL 65d02264780d55f1b746f9954631cefd +BELONGS ya.make + Note: matched license text is too long. Read it in the source files. + Scancode info: + Original SPDX id: WTFPL + Score : 100.00 + Match type : NOTICE + Links : http://sam.zoy.org/wtfpl/, http://sam.zoy.org/wtfpl/COPYING, https://spdx.org/licenses/WTFPL + Files with this license: + src/ftxui/component/animation.cpp [22:26] + +KEEP MIT 87cbaeea422588c24f4fc7dc9f7422b1 +BELONGS ya.make +FILE_INCLUDE LICENSE found in files: src/ftxui/dom/box_helper.hpp at line 3 + License text: + // Use of this source code is governed by the MIT license that can be found in + // the LICENSE file.line. + Scancode info: + Original SPDX id: MIT + Score : 100.00 + Match type : NOTICE + Links : http://opensource.org/licenses/mit-license.php, https://spdx.org/licenses/MIT + Files with this license: + src/ftxui/dom/box_helper.hpp [2:3] + +KEEP MIT 8c6b397cbef46628bea401075e15b1d5 +BELONGS ya.make + License text: + The MIT License + Scancode info: + Original SPDX id: MIT + Score : 100.00 + Match type : REFERENCE + Links : http://opensource.org/licenses/mit-license.php, https://spdx.org/licenses/MIT + Files with this license: + LICENSE [1:1] + +KEEP MIT e45ac9ead97064977228638d2932ed62 +BELONGS ya.make +FILE_INCLUDE LICENSE found in files: include/ftxui/component/animation.hpp at line 3, include/ftxui/component/captured_mouse.hpp at line 3, include/ftxui/component/component.hpp at line 3, include/ftxui/component/component_base.hpp at line 3, include/ftxui/component/component_options.hpp at line 3, include/ftxui/component/event.hpp at line 3, include/ftxui/component/loop.hpp at line 3, include/ftxui/component/mouse.hpp at line 3, include/ftxui/component/receiver.hpp at line 3, include/ftxui/component/screen_interactive.hpp at line 3, include/ftxui/component/task.hpp at line 3, include/ftxui/dom/canvas.hpp at line 3, include/ftxui/dom/deprecated.hpp at line 3, include/ftxui/dom/direction.hpp at line 3, include/ftxui/dom/elements.hpp at line 3, include/ftxui/dom/flexbox_config.hpp at line 3, include/ftxui/dom/linear_gradient.hpp at line 3, include/ftxui/dom/node.hpp at line 3, include/ftxui/dom/requirement.hpp at line 3, include/ftxui/dom/selection.hpp at line 3, include/ftxui/dom/table.hpp at line 3, include/ftxui/dom/take_any_args.hpp at line 3, include/ftxui/screen/box.hpp at line 3, include/ftxui/screen/color.hpp at line 3, include/ftxui/screen/color_info.hpp at line 3, include/ftxui/screen/deprecated.hpp at line 3, include/ftxui/screen/image.hpp at line 3, include/ftxui/screen/pixel.hpp at line 3, include/ftxui/screen/screen.hpp at line 3, include/ftxui/screen/string.hpp at line 3, include/ftxui/screen/terminal.hpp at line 3, include/ftxui/util/autoreset.hpp at line 3, include/ftxui/util/ref.hpp at line 3, src/ftxui/component/button.cpp at line 3, src/ftxui/component/catch_event.cpp at line 3, src/ftxui/component/checkbox.cpp at line 3, src/ftxui/component/collapsible.cpp at line 3, src/ftxui/component/component.cpp at line 3, src/ftxui/component/component_options.cpp at line 3, src/ftxui/component/container.cpp at line 3, src/ftxui/component/dropdown.cpp at line 3, src/ftxui/component/event.cpp at line 3, src/ftxui/component/hoverable.cpp at line 3, src/ftxui/component/loop.cpp at line 3, src/ftxui/component/maybe.cpp at line 3, src/ftxui/component/menu.cpp at line 3, src/ftxui/component/modal.cpp at line 3, src/ftxui/component/radiobox.cpp at line 3, src/ftxui/component/renderer.cpp at line 3, src/ftxui/component/resizable_split.cpp at line 3, src/ftxui/component/screen_interactive.cpp at line 3, src/ftxui/component/slider.cpp at line 3, src/ftxui/component/terminal_input_parser.cpp at line 3, src/ftxui/component/terminal_input_parser.hpp at line 3, src/ftxui/component/util.cpp at line 3, src/ftxui/component/window.cpp at line 3, src/ftxui/dom/automerge.cpp at line 3, src/ftxui/dom/blink.cpp at line 3, src/ftxui/dom/bold.cpp at line 3, src/ftxui/dom/border.cpp at line 3, src/ftxui/dom/box_helper.cpp at line 3, src/ftxui/dom/canvas.cpp at line 3, src/ftxui/dom/clear_under.cpp at line 3, src/ftxui/dom/color.cpp at line 3, src/ftxui/dom/composite_decorator.cpp at line 3, src/ftxui/dom/dbox.cpp at line 3, src/ftxui/dom/dim.cpp at line 3, src/ftxui/dom/flex.cpp at line 3, src/ftxui/dom/flexbox_config.cpp at line 3, src/ftxui/dom/flexbox_helper.hpp at line 3, src/ftxui/dom/focus.cpp at line 3, src/ftxui/dom/frame.cpp at line 3, src/ftxui/dom/gauge.cpp at line 3, src/ftxui/dom/graph.cpp at line 3, src/ftxui/dom/gridbox.cpp at line 3, src/ftxui/dom/hbox.cpp at line 3, src/ftxui/dom/hyperlink.cpp at line 3, src/ftxui/dom/inverted.cpp at line 3, src/ftxui/dom/italic.cpp at line 3, src/ftxui/dom/linear_gradient.cpp at line 3, src/ftxui/dom/node.cpp at line 3, src/ftxui/dom/node_decorator.cpp at line 3, src/ftxui/dom/node_decorator.hpp at line 3, src/ftxui/dom/paragraph.cpp at line 3, src/ftxui/dom/reflect.cpp at line 3, src/ftxui/dom/scroll_indicator.cpp at line 3, src/ftxui/dom/selection.cpp at line 3, src/ftxui/dom/selection_style.cpp at line 3, src/ftxui/dom/separator.cpp at line 3, src/ftxui/dom/size.cpp at line 3, src/ftxui/dom/spinner.cpp at line 3, src/ftxui/dom/strikethrough.cpp at line 3, src/ftxui/dom/table.cpp at line 3, src/ftxui/dom/text.cpp at line 3, src/ftxui/dom/underlined.cpp at line 3, src/ftxui/dom/underlined_double.cpp at line 3, src/ftxui/dom/util.cpp at line 3, src/ftxui/dom/vbox.cpp at line 3, src/ftxui/screen/box.cpp at line 3, src/ftxui/screen/color.cpp at line 3, src/ftxui/screen/color_info.cpp at line 3, src/ftxui/screen/image.cpp at line 3, src/ftxui/screen/screen.cpp at line 3, src/ftxui/screen/string.cpp at line 3, src/ftxui/screen/string_internal.hpp at line 3, src/ftxui/screen/terminal.cpp at line 3, src/ftxui/screen/util.hpp at line 3 + License text: + // Use of this source code is governed by the MIT license that can be found in + // the LICENSE file. + Scancode info: + Original SPDX id: MIT + Score : 100.00 + Match type : NOTICE + Links : http://opensource.org/licenses/mit-license.php, https://spdx.org/licenses/MIT + Files with this license: + include/ftxui/component/animation.hpp [2:3] + include/ftxui/component/captured_mouse.hpp [2:3] + include/ftxui/component/component.hpp [2:3] + include/ftxui/component/component_base.hpp [2:3] + include/ftxui/component/component_options.hpp [2:3] + include/ftxui/component/event.hpp [2:3] + include/ftxui/component/loop.hpp [2:3] + include/ftxui/component/mouse.hpp [2:3] + include/ftxui/component/receiver.hpp [2:3] + include/ftxui/component/screen_interactive.hpp [2:3] + include/ftxui/component/task.hpp [2:3] + include/ftxui/dom/canvas.hpp [2:3] + include/ftxui/dom/deprecated.hpp [2:3] + include/ftxui/dom/direction.hpp [2:3] + include/ftxui/dom/elements.hpp [2:3] + include/ftxui/dom/flexbox_config.hpp [2:3] + include/ftxui/dom/linear_gradient.hpp [2:3] + include/ftxui/dom/node.hpp [2:3] + include/ftxui/dom/requirement.hpp [2:3] + include/ftxui/dom/selection.hpp [2:3] + include/ftxui/dom/table.hpp [2:3] + include/ftxui/dom/take_any_args.hpp [2:3] + include/ftxui/screen/box.hpp [2:3] + include/ftxui/screen/color.hpp [2:3] + include/ftxui/screen/color_info.hpp [2:3] + include/ftxui/screen/deprecated.hpp [2:3] + include/ftxui/screen/image.hpp [2:3] + include/ftxui/screen/pixel.hpp [2:3] + include/ftxui/screen/screen.hpp [2:3] + include/ftxui/screen/string.hpp [2:3] + include/ftxui/screen/terminal.hpp [2:3] + include/ftxui/util/autoreset.hpp [2:3] + include/ftxui/util/ref.hpp [2:3] + src/ftxui/component/button.cpp [2:3] + src/ftxui/component/catch_event.cpp [2:3] + src/ftxui/component/checkbox.cpp [2:3] + src/ftxui/component/collapsible.cpp [2:3] + src/ftxui/component/component.cpp [2:3] + src/ftxui/component/component_options.cpp [2:3] + src/ftxui/component/container.cpp [2:3] + src/ftxui/component/dropdown.cpp [2:3] + src/ftxui/component/event.cpp [2:3] + src/ftxui/component/hoverable.cpp [2:3] + src/ftxui/component/loop.cpp [2:3] + src/ftxui/component/maybe.cpp [2:3] + src/ftxui/component/menu.cpp [2:3] + src/ftxui/component/modal.cpp [2:3] + src/ftxui/component/radiobox.cpp [2:3] + src/ftxui/component/renderer.cpp [2:3] + src/ftxui/component/resizable_split.cpp [2:3] + src/ftxui/component/screen_interactive.cpp [2:3] + src/ftxui/component/slider.cpp [2:3] + src/ftxui/component/terminal_input_parser.cpp [2:3] + src/ftxui/component/terminal_input_parser.hpp [2:3] + src/ftxui/component/util.cpp [2:3] + src/ftxui/component/window.cpp [2:3] + src/ftxui/dom/automerge.cpp [2:3] + src/ftxui/dom/blink.cpp [2:3] + src/ftxui/dom/bold.cpp [2:3] + src/ftxui/dom/border.cpp [2:3] + src/ftxui/dom/box_helper.cpp [2:3] + src/ftxui/dom/canvas.cpp [2:3] + src/ftxui/dom/clear_under.cpp [2:3] + src/ftxui/dom/color.cpp [2:3] + src/ftxui/dom/composite_decorator.cpp [2:3] + src/ftxui/dom/dbox.cpp [2:3] + src/ftxui/dom/dim.cpp [2:3] + src/ftxui/dom/flex.cpp [2:3] + src/ftxui/dom/flexbox_config.cpp [2:3] + src/ftxui/dom/flexbox_helper.hpp [2:3] + src/ftxui/dom/focus.cpp [2:3] + src/ftxui/dom/frame.cpp [2:3] + src/ftxui/dom/gauge.cpp [2:3] + src/ftxui/dom/graph.cpp [2:3] + src/ftxui/dom/gridbox.cpp [2:3] + src/ftxui/dom/hbox.cpp [2:3] + src/ftxui/dom/hyperlink.cpp [2:3] + src/ftxui/dom/inverted.cpp [2:3] + src/ftxui/dom/italic.cpp [2:3] + src/ftxui/dom/linear_gradient.cpp [2:3] + src/ftxui/dom/node.cpp [2:3] + src/ftxui/dom/node_decorator.cpp [2:3] + src/ftxui/dom/node_decorator.hpp [2:3] + src/ftxui/dom/paragraph.cpp [2:3] + src/ftxui/dom/reflect.cpp [2:3] + src/ftxui/dom/scroll_indicator.cpp [2:3] + src/ftxui/dom/selection.cpp [2:3] + src/ftxui/dom/selection_style.cpp [2:3] + src/ftxui/dom/separator.cpp [2:3] + src/ftxui/dom/size.cpp [2:3] + src/ftxui/dom/spinner.cpp [2:3] + src/ftxui/dom/strikethrough.cpp [2:3] + src/ftxui/dom/table.cpp [2:3] + src/ftxui/dom/text.cpp [2:3] + src/ftxui/dom/underlined.cpp [2:3] + src/ftxui/dom/underlined_double.cpp [2:3] + src/ftxui/dom/util.cpp [2:3] + src/ftxui/dom/vbox.cpp [2:3] + src/ftxui/screen/box.cpp [2:3] + src/ftxui/screen/color.cpp [2:3] + src/ftxui/screen/color_info.cpp [2:3] + src/ftxui/screen/image.cpp [2:3] + src/ftxui/screen/screen.cpp [2:3] + src/ftxui/screen/string.cpp [2:3] + src/ftxui/screen/string_internal.hpp [2:3] + src/ftxui/screen/terminal.cpp [2:3] + src/ftxui/screen/util.hpp [2:3] diff --git a/contrib/libs/ftxui/.yandex_meta/licenses.list.txt b/contrib/libs/ftxui/.yandex_meta/licenses.list.txt new file mode 100644 index 00000000000..c79dee2c1a4 --- /dev/null +++ b/contrib/libs/ftxui/.yandex_meta/licenses.list.txt @@ -0,0 +1,97 @@ +====================COPYRIGHT==================== +// Copyright (c) 2011, Auerhaus Development, LLC + + +====================COPYRIGHT==================== +// Copyright 2020 Arthur Sonzogni. All rights reserved. + + +====================COPYRIGHT==================== +// Copyright 2021 Arthur Sonzogni. All rights reserved. + + +====================COPYRIGHT==================== +// Copyright 2022 Arthur Sonzogni. All rights reserved. + + +====================COPYRIGHT==================== +// Copyright 2023 Arthur Sonzogni. All rights reserved. + + +====================COPYRIGHT==================== +// Copyright 2024 Arthur Sonzogni. All rights reserved. + + +====================COPYRIGHT==================== +// Copyright 2025 Arthur Sonzogni. All rights reserved. + + +====================COPYRIGHT==================== +Copyright (c) 2019 Arthur Sonzogni. + + +====================File: LICENSE==================== +The MIT License + +Copyright (c) 2019 Arthur Sonzogni. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + + +====================MIT==================== +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + + +====================MIT==================== +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file.line. + + +====================MIT==================== +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +====================MIT==================== +The MIT License + + +====================WTFPL==================== +// This program is free software. It comes without any warranty, to +// the extent permitted by applicable law. You can redistribute it +// and/or modify it under the terms of the Do What The Fuck You Want +// To Public License, Version 2, as published by Sam Hocevar. See +// http://sam.zoy.org/wtfpl/COPYING for more details. diff --git a/contrib/libs/ftxui/.yandex_meta/override.nix b/contrib/libs/ftxui/.yandex_meta/override.nix new file mode 100644 index 00000000000..6bc8afd25e5 --- /dev/null +++ b/contrib/libs/ftxui/.yandex_meta/override.nix @@ -0,0 +1,12 @@ +pkgs: attrs: with pkgs; with attrs; rec { + version = "6.0.2"; + + src = fetchFromGitHub { + owner = "ArthurSonzogni"; + repo = "FTXUI"; + rev = "v${version}"; + hash = "sha256-VvP1ctFlkTDdrAGRERBxMRpFuM4mVpswR/HO9dzUSUo="; + }; + + patches = []; +} diff --git a/contrib/libs/ftxui/CHANGELOG.md b/contrib/libs/ftxui/CHANGELOG.md new file mode 100644 index 00000000000..6638c2712cc --- /dev/null +++ b/contrib/libs/ftxui/CHANGELOG.md @@ -0,0 +1,459 @@ +Changelog +========= + +6.0.2 (2025-03-30) +----- + +### Component +- BugFix: Fix major crash on Windows affecting all components. See #1020 +- BugFix: Fix focusRelative. + +6.0.1 (2025-03-28) +----- + +Same as v6.0.0. + +Due to a problem tag v6.0.0 was replaced. This isn't a good practice and affect +developers that started using it in the short timeframe. Submitting a new +release with the same content is the best way to fix this. + +See #1017 and #1019. + +6.0.0 (2025-03-23) +----- + +### Component +- Feature: Add support for raw input. Allowing more keys to be detected. +- Feature: Add `ScreenInteractive::ForceHandleCtrlC(false)` to allow component + to fully override the default `Ctrl+C` handler. +- Feature: Add `ScreenInteractive::ForceHandleCtrlZ(false)` to allow component + to fully override the default `Ctrl+Z` handler. +- Feature: Add `Mouse::WeelLeft` and `Mouse::WeelRight` events on supported + terminals. +- Feature: Add `Event::DebugString()`. +- Feature: Add support for `Input`'s insert mode. Add `InputOption::insert` + option. Added by @mingsheng13. +- Feature: Add `DropdownOption` to configure the dropdown. See #826. +- Feature: Add support for Selection. Thanks @clement-roblot. See #926. + - See `ScreenInteractive::GetSelection()`. + - See `ScreenInteractive::SelectionChange(...)` listener. +- Bugfix/Breaking change: `Mouse transition`: + - Detect when the mouse move, as opposed to being pressed. + The Mouse::Moved motion was added. + - Dragging the mouse with the left button pressed now avoids activating + multiple checkboxes. + - A couple of components are now activated when the mouse is pressed, + as opposed to being released. + This fixes: https://github.com/ArthurSonzogni/FTXUI/issues/773 + This fixes: https://github.com/ArthurSonzogni/FTXUI/issues/792 +- Bugfix: mouse.control is now reported correctly. +- Feature: Add `ScreenInteractive::FullscreenPrimaryScreen()`. This allows + displaying a fullscreen component on the primary screen, as opposed to the + alternate screen. +- Bugfix: `Input` `onchange` was not called on backspace or delete key. + Fixed by @chrysante in chrysante in PR #776. +- Bugfix: Propertly restore cursor shape on exit. See #792. +- Bugfix: Fix cursor position in when in the last column. See #831. +- Bugfix: Fix `ResizeableSplit` keyboard navigation. Fixed by #842. +- Bugfix: Fix `Menu` focus. See #841 +- Feature: Add `ComponentBase::Index()`. This allows to get the index of a + component in its parent. See #932 +- Feature: Add `EntryState::index`. This allows to get the index of a menu entry. + See #932 +- Feature: Add `SliderOption::on_change`. This allows to set a callback when the + slider value changes. See #938. +- Bugfix: Handle `Dropdown` with no entries. +- Bugfix: Fix crash in `LinearGradient` due to float precision and an off-by-one + mistake. See #998. + +### Dom +- Feature: Add `italic` decorator. For instance: + ```cpp + auto italic_text = text("Italic text") | italic; + ``` + ```cpp + auto italic_text = italic(text("Italic text")); + ``` + Proposed by @kenReneris in #1009. +- Feature: Add `hscroll_indicator`. It display an horizontal indicator + reflecting the current scroll position. Proposed by @ibrahimnasson in + [issue 752](https://github.com/ArthurSonzogni/FTXUI/issues/752) +- Feature: Add `extend_beyond_screen` option to `Dimension::Fit(..)`, allowing + the element to be larger than the screen. Proposed by @LordWhiro. See #572 and + #949. +- Feature: Add support for Selection. Thanks @clement-roblot. See #926. + - See `selectionColor` decorator. + - See `selectionBackgroundColor` decorator. + - See `selectionForegroundColor` decorator. + - See `selectionStyle(style)` decorator. + - See `selectionStyleReset` decorator. +- Breaking change: Change how "focus"/"select" are handled. This fixes the + behavior. +- Breaking change: `Component::OnRender()` becomes the method to override to + render a component. This replaces `Component::Render()` that is still in use + to call the rendering method on the children. This change allows to fix a + couple of issues around focus handling. + +### Screen +- Feature: Add `Box::IsEmpty()`. +- Feature: Color transparency + - Add `Color::RGBA(r,g,b,a)`. + - Add `Color::HSVA(r,g,b,a)`. + - Add `Color::Blend(Color)`. + - Add `Color::IsOpaque()` + +### Util +- Feature: Support arbitrary `Adapter` for `ConstStringListRef`. See #843. + +### Build +- Support for cmake's "unity/jumbo" builds. Fixed by @ClausKlein. + +5.0.0 +----- + +### Component +- Breaking: MenuDirection enum is renamed Direction +- Breaking: GaugeDirection enum is renamed Direction +- Breaking: Direction enum is renamed WidthOrHeight +- Breaking: Remove `ComponentBase` copy constructor/assignment. +- Breaking: MenuOption::entries is renamed MenuOption::entries_option. +- Breaking: `Ref<{Component}Option>` becomes `{Component}Option` in component constructors. +- Feature: `ResizeableSplit` now support arbitrary element as a separator. +- Feature: `input` is now supporting multiple lines. +- Feature: `input` style is now customizeable. +- Bugfix: Support F1-F5 from OS terminal. +- Feature: Add struct based constructor: + ```cpp + Component Button(ButtonOption options); + Component Checkbox(CheckboxOption options); + Component Input(InputOption options); + Component Menu(MenuOption options); + Component MenuEntry(MenuEntryOption options); + Component Radiobox(RadioboxOption options); + Component Slider(SliderOption<T> options); + Component ResizableSplit(ResizableSplitOption options); + ``` +- Feature: Add `ScreenInteractive::TrackMouse(false)` disable mouse support. + +### Dom +- Feature: Add `hyperlink` decorator. For instance: + ```cpp + auto link = text("Click here") | hyperlink("https://github.com/FTXUI") + ``` + See the [OSC 8 page](https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda). + FTXUI support proposed by @aaleino in [#662](https://github.com/ArthurSonzogni/FTXUI/issues/662). + +### Screen +- Breaking: `WordBreakProperty` becomes a uint8_t enum. This yields a 0.8% + performance improvement. +- Breaking: Remove user defined Pixel constructor and equality operator. +- Performance: 19% faster on benchmarks. + + +### Build +- Check version compatibility when using cmake find_package() +- Add `FTXUI_DEV_WARNING` options to turn on warnings when building FTXUI +- Turn OFF by default `FTXUI_BUILD_DOCS` +- Turn OFF by default `FTXUI_BUILD_EXAMPLE` + +4.1.1 +----- + +### Component +- Fix: Support arrow keys in application mode +- Fix: Remove useless new line when using an alternative screen. + +### Dom +- Feature: Add the dashed style for border and separator: + - See `DASHED` enum, and `separatorDashed()`, `borderDashed()` functions. +- Feature: Add colored borders. + - See functions: `borderStyled(BorderStyle, Color)` and `borderStyled(Color)`. +- Feature: Add `LinearGradient`. It can be used in `color` and `bgColor`. +- Improvement: Color::Interpolate() uses gamma correction. +- Fix: Check the `graph` area is positive. + +### Build/Install +- Use globally set CMAKE_CXX_STANDARD if it is set. +- Expose the pkg-config file +- Check version compatibility when using cmake find_package() + +4.1.0 (Abandonned) +----- +This version is abandonned and must not be used. It introduced a breaking change in the API. + +4.0.0 +----- + +### DOM +- Feature: more styles: + - `strikethrough` + - `underlinedDouble` +- Feature: Customize the cursor. Add the following decorators: + - `focusCursorBlock` + - `focusCursorBlockBlinking` + - `focusCursorBar` + - `focusCursorBarBlinking` + - `focusCursorUnderline` + - `focusCursorUnderlineBlinking` +- Bugfix: Fix `focus`/`select` when the `vbox`/`hbox`/`dbox` contains a + `flexbox` +- Bugfix: Fix the selected/focused area. It used to be 1 cell larger/longer than + requested +- Bugfix: Forward the selected/focused area from the child in gridbox. +- Bugfix: Fix incorrect Canvas computed dimensions. +- Bugfix: Support `vscroll_indicator` with a zero inner size. +- Bugfix: Fix `vscroll_indicator` hidding the last column. + +### Component: +- Feature: Add the `Modal` component. +- Feature: `Slider` supports taking references for all its arguments. +- Feature: `Slider` supports `SliderOption`. It supports: + - multiple directions. + - multiple colors. + - various values (value, min, max, increment). +- Feature: Define `ScreenInteractive::Exit()`. +- Feature: Add `Loop` to give developers a better control on the main loop. This + can be used to integrate FTXUI into another main loop, without taking the full + control. +- Feature: `Input` supports CTRL+Left and CTRL+Right +- Feature: Use a blinking bar in the `Input` component. +- Improvement: The `Menu` keeps the focus when an entry is selected with the + mouse. +- Bugfix: Add implementation of `ButtonOption::Border()`. It was missing. +- Bugfix: Provide the correct key for F1-F4 and F11. +- Feature: Add the `Hoverable` component decorators. + +### Screen +- Feature: add `Box::Union(a,b) -> Box` +- Bugfix: Fix resetting `dim` clashing with resetting of `bold`. +- Feature: Add emscripten screen resize support. +- Bugfix: Add unicode 13 support for full width characters. +- Bugfix: Fix MSVC treating codecvt C++17 deprecated function as an error. + +### Build +- Support using the google test version provided by the package manager. + +3.0.0 +----- + +### Build +- **breaking**: The library prefix is now back to "lib" (the default). This + means non-cmake users should not link against "libftxui-dom" for instance. + +### Component +- **Animations** module! Components can implement the `OnAnimation` method and + the animation::Animator to define some animated properties. + - `Menu` now support animations. + - `Button` now supports animations. +- Support SIGTSTP. (ctrl+z). +- Support task posting. `ScreenInteractive::Post(Task)`. +- `Menu` can now be used in the 4 directions, using `MenuOption.direction`. +- `Menu` can display an animated underline, using + `MenuOption.underline.enabled`. +- `Button` is now taking the focus in frame. +- **breaking** All the options are now using a transform function. +- **breaking** The `Toggle` component is now implemented using `Menu`. +- **bugfix** Container::Tab implements `Focusable()`. +- **bugfix** Improved default implementations of ComponentBase `Focusable()` and + `ActiveChild()` methods. +- **bugfix** Automatically convert '\r' keys into '\n' for Linux programs that + do not send the correct code for the return key, like the 'bind'. + https://github.com/ArthurSonzogni/FTXUI/issues/337 +- Add decorator for components: + - `operator|(Component, ComponentDecorator)` + - `operator|(Component, ElementDecorator)` + - `operator|=(Component, ComponentDecorator)` + - `operator|=(Component, ElementDecorator)` + - Add the `Maybe` decorator. + - Add the `CatchEvent` decorator. + - Add the `Renderer` decorator. +- **breaking** remove the "deprectated.hpp" header and Input support for wide + string. + +### DOM: +- **breaking**: The `inverted` decorator now toggle in the inverted attribute. +- Add `gauge` for the 4 directions. Expose the following API: +```cpp +Element gauge(float ratio); +Element gaugeLeft(float ratio); +Element gaugeRight(float ratio); +Element gaugeUp(float ratio); +Element gaugeDown(float ratio); +Element gaugeDirection(float ratio, GaugeDirection); +``` +- Add `separatorHSelector` and `separatorVSelector` elements. This can be used + to highlight an area. +- Add the `automerge` decorator. This makes separator characters to be merged + with others nearby. +- Fix the `Table` rendering function, to allow automerging characters. +- **Bugfix**: The `vscroll_indicator` now computes its offset and size + correctly. +- Add the `operator|=(Element, Decorator)` + +### Screen: +- Add: `Color::Interpolate(lambda, color_a, color_b)`. + +2.0.0 +----- + +### Features: + +#### Screen +- Add the `automerge` to the Pixel bit field. This now controls which pixels are + automatically merged. + +#### DOM: +- Add the `Canvas` class and `ElementFrom('canvas')` function. Together users of + the library can draw using braille and block characters. +- Support `flexbox` dom elements. This is build symmetrically to the HTML one. + All the following attributes are supported: direction, wrap, justify-content, + align-items, align-content, gap +- Add the dom elements helper based on `flexbox`: + - `paragraph` + - `paragraphAlignLeft` + - `paragraphAlignCenter` + - `paragraphAlignRight` + - `paragraphAlignJustify` +- Add the helper elements based on `flexbox`: `hflow()`, `vflow()`. +- Add: `focusPositionRelative` and `focusPosition` +- Add `Table` constructor from 2D vector of Element, instead of string. + +#### Component +- Add the `collapsible` component. +- Add the `ScreenInteractive::WithRestoredIO`. This decorates a callback. This + runs it with the terminal hooks temporarilly uninstalled. This is useful if + you want to execute command using directly stdin/stdout/sterr. + +### Bug + +#### Table +- The `table` horizontal and vertical separator are now correctly expanded. + +#### Component +- `Input` shouldn't take focus when hovered by the mouse. +- Modifying `Input`'s during on_enter/on_change event is now working correctly. + +### Breaking changes: +- The behavior of `paragraph` has been modified. It now returns en Element, + instead of a list of elements. + +0.11.1 +------ + +# Component +- Feature: Support for PageUp/PageDown/Home/End buttons. +- Bugfix: Check the selected element are within bounds for Dropdown. + +# Build +- Bugfix: Package library using the "Release config". Not debug. + +0.11 +---- + +## github workflow +- Add Windows ad MacOS artefacts. +- Merge all the workflows. + +## Bug +- On Unix system, fallback to {80,25} screen dimension on failure. + +## CMake +- Support for shared library, via `BUILD_SHARED_LIBS` option. +- Add library version and symlinks. + +0.10 (2021-09-30) +-------------------- + +## Bug +- Fix the automated merge of borders. + +### Dom +- `Table()` class to build stylised table. + See https://github.com/ArthurSonzogni/FTXUI/discussions/228 +- `vscroll_indicator`. Show a scrollbar indicator on the right. +- `separatorEmpty`. A separator drawing nothing. +- `separatorFixed`. A separator drawing the provided character. + +### Component +- `Maybe`: Display an component conditionnally based on a boolean. +- `Dropdown`: A dropdown select list. + +0.9 (2021-09-26) +---------------- + +The initial release where changelog where written. + +This version includes: + +### screen +- Style: + - Bold. + - Blink. + - Dim. + - Inverted. + - Underlined. + - Foreground color. + - Background color. +- Support for UTF8 unicode. + - Full wide character: 测试. + - Combining characters: a⃒ +- A Stencil buffer. +- Automatically merge box drawing characters. +- Detect terminal dimension. + +### DOM + +- Element: + - `text` & `vtext` + - `separator` and 5 variations. + - `gauge` + - `border` and 6 variations. + - `window` + - `spinner` + - `paragraph` and `hflow`. + +- Layout: + - `hbox` + - `vbox` + - `dbox` + - `gridbox` + - `frame`: Drawing inside a virtual area, potentially larger than the real + one. + - `focus`, `select`: scroll the inner view of a frame, to be in view. + - `flex` & 8 variations. `filler` + +- Decorators: + - `bold` + - `dim` + - `inverted` + - `blink` + - `color` + - `bgcolor` + - `clearunder` + +### Component + +- Container: + - `Container::Vertical` + - `Container::Horizontal` + - `Container::Tab` +- `Button` +- `Checkbox` +- `Input` +- `Menu` +- `MenuEntry` +- `Radiobox` +- `Toggle` +- `Slider` +- `Renderer` & variations +- `CatchEvent` + +### MISC + +- Fuzzer +- Tests using gtest. +- Doxygen documentation +- IWYU +- 52 examples. +- Support for WebAssembly. +- Support for Window and fallback for broken terminal. diff --git a/contrib/libs/ftxui/LICENSE b/contrib/libs/ftxui/LICENSE new file mode 100644 index 00000000000..0814b334296 --- /dev/null +++ b/contrib/libs/ftxui/LICENSE @@ -0,0 +1,22 @@ +The MIT License + +Copyright (c) 2019 Arthur Sonzogni. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/contrib/libs/ftxui/README.md b/contrib/libs/ftxui/README.md new file mode 100644 index 00000000000..e7d8f3193d3 --- /dev/null +++ b/contrib/libs/ftxui/README.md @@ -0,0 +1,404 @@ +<p align="center"> + <img src="https://github.com/ArthurSonzogni/FTXUI/assets/4759106/6925b6da-0a7e-49d9-883c-c890e1f36007" alt="Demo image"></img> + <br/> + <a href="#"><img src="https://img.shields.io/badge/c++-%2300599C.svg?style=flat&logo=c%2B%2B&logoColor=white"></img></a> + <a href="http://opensource.org/licenses/MIT"><img src="https://img.shields.io/github/license/arthursonzogni/FTXUI?color=black"></img></a> + <a href="#"><img src="https://img.shields.io/github/stars/ArthurSonzogni/FTXUI"></img></a> + <a href="#"><img src="https://img.shields.io/github/forks/ArthurSonzogni/FTXUI"></img></a> + <a href="#"><img src="https://img.shields.io/github/repo-size/ArthurSonzogni/FTXUI"></img></a> + <a href="https://github.com/ArthurSonzogni/FTXUI/graphs/contributors"><img src="https://img.shields.io/github/contributors/arthursonzogni/FTXUI?color=blue"></img></a> + <br/> + <a href="https://github.com/ArthurSonzogni/FTXUI/issues"><img src="https://img.shields.io/github/issues/ArthurSonzogni/FTXUI"></img></a> + <a href="https://repology.org/project/ftxui/versions"><img src="https://repology.org/badge/latest-versions/ftxui.svg" alt="latest packaged version(s)"></a> + <a href="https://codecov.io/gh/ArthurSonzogni/FTXUI"> + <img src="https://codecov.io/gh/ArthurSonzogni/FTXUI/branch/master/graph/badge.svg?token=C41FdRpNVA"/> + </a> + + + <br/> + <a href="https://arthursonzogni.github.io/FTXUI/">Documentation</a> · + <a href="https://github.com/ArthurSonzogni/FTXUI/issues">Report a Bug</a> · + <a href="https://arthursonzogni.github.io/FTXUI/examples.html">Examples</a> . + <a href="https://github.com/ArthurSonzogni/FTXUI/issues">Request Feature</a> · + <a href="https://github.com/ArthurSonzogni/FTXUI/pulls">Send a Pull Request</a> + +</p> + +# FTXUI + +<i>Functional Terminal (X) User interface</i> + +A simple cross-platform C++ library for terminal based user interfaces! + +## Feature + * Functional style. Inspired by + [1](https://hackernoon.com/building-reactive-terminal-interfaces-in-c-d392ce34e649?gi=d9fb9ce35901) + and [React](https://reactjs.org/) + * Simple and elegant syntax (in my opinion) + * Keyboard & mouse navigation. + * Support for [UTF8](https://en.wikipedia.org/wiki/UTF-8) and [fullwidth chars](https://en.wikipedia.org/wiki/Halfwidth_and_fullwidth_forms) (→ 测试) + * Support for animations. [Demo 1](https://arthursonzogni.github.io/FTXUI/examples/?file=component/menu_underline_animated_gallery), [Demo 2](https://arthursonzogni.github.io/FTXUI/examples/?file=component/button_style) + * Support for drawing. [Demo](https://arthursonzogni.github.io/FTXUI/examples/?file=component/canvas_animated) + * No dependencies + * **Cross platform**: Linux/MacOS (main target), WebAssembly, Windows (Thanks to contributors!). + * Learn by [examples](#documentation), and [tutorials](#documentation) + * Multiple packages: CMake [FetchContent]([https://bewagner.net/programming/2020/05/02/cmake-fetchcontent/](https://cmake.org/cmake/help/latest/module/FetchContent.html)) (preferred), vcpkg, pkgbuild, conan. + * Good practices: documentation, tests, fuzzers, performance tests, automated CI, automated packaging, etc... + +## Documentation + +- [Starter example project](https://github.com/ArthurSonzogni/ftxui-starter) +- [Documentation](https://arthursonzogni.github.io/FTXUI/) +- [Examples (WebAssembly)](https://arthursonzogni.github.io/FTXUI/examples/) +- [Build using CMake](https://arthursonzogni.github.io/FTXUI/#build-cmake) + +## Example +~~~cpp + vbox({ + hbox({ + text("one") | border, + text("two") | border | flex, + text("three") | border | flex, + }), + + gauge(0.25) | color(Color::Red), + gauge(0.50) | color(Color::White), + gauge(0.75) | color(Color::Blue), + }); +~~~ + + + +## Short gallery + +#### DOM + +This module defines a hierarchical set of Element. An Element manages layout and can be responsive to the terminal dimensions. + +They are declared in [<ftxui/dom/elements.hpp>](https://arthursonzogni.github.io/FTXUI/elements_8hpp_source.html +) + +<details><summary>Layout</summary> + +Element can be arranged together: + - horizontally with `hbox` + - vertically with `vbox` + - inside a grid with `gridbox` + - wrap along one direction using the `flexbox`. + +Element can become flexible using the the `flex` decorator. + +[Example](https://arthursonzogni.github.io/FTXUI/examples_2dom_2vbox_hbox_8cpp-example.html) using `hbox`, `vbox` and `filler`. + + + + +[Example](https://arthursonzogni.github.io/FTXUI/examples_2dom_2gridbox_8cpp-example.html) using gridbox: + + + +[Example](https://github.com/ArthurSonzogni/FTXUI/blob/master/examples/dom/hflow.cpp) using flexbox: + + + +[See](https://arthursonzogni.github.io/FTXUI/examples_2dom_2hflow_8cpp-example.html) also this [demo](https://arthursonzogni.github.io/FTXUI/examples/?file=component/flexbox). + +</details> + +<details><summary>Style</summary> + +An element can be decorated using the functions: + - `bold` + - `italic` + - `dim` + - `inverted` + - `underlined` + - `underlinedDouble` + - `blink` + - `strikethrough` + - `color` + - `bgcolor` + - `hyperlink` + +[Example](https://arthursonzogni.github.io/FTXUI/examples_2dom_2style_gallery_8cpp-example.html) + + + +FTXUI supports the pipe operator. It means: `decorator1(decorator2(element))` and `element | decorator1 | decorator2` can be used. + +</details> + +<details><summary>Colors</summary> + +FTXUI support every color palette: + +Color [gallery](https://arthursonzogni.github.io/FTXUI/examples_2dom_2color_gallery_8cpp-example.html): + + +</details> + +<details><summary>Border and separator</summary> + +Use decorator border and element separator() to subdivide your UI: + +```cpp +auto document = vbox({ + text("top"), + separator(), + text("bottom"), +}) | border; + +``` + +[Demo](https://arthursonzogni.github.io/FTXUI/examples_2dom_2separator_8cpp-example.html): + + + +</details> + +<details><summary>Text and paragraph</summary> + +A simple piece of text is represented using `text("content")`. + +To support text wrapping following spaces the following functions are provided: +```cpp +Element paragraph(std::string text); +Element paragraphAlignLeft(std::string text); +Element paragraphAlignRight(std::string text); +Element paragraphAlignCenter(std::string text); +Element paragraphAlignJustify(std::string text); +``` + +[Paragraph example](https://arthursonzogni.github.io/FTXUI/examples_2dom_2paragraph_8cpp-example.html) + + + +</details> + +<details><summary>Table</summary> + +A class to easily style a table of data. + +[Example](https://arthursonzogni.github.io/FTXUI/examples_2dom_2table_8cpp-example.html): + + + +</details> + +<details><summary>Canvas</summary> + +Drawing can be made on a Canvas, using braille, block, or simple characters: + +Simple [example](https://github.com/ArthurSonzogni/FTXUI/blob/master/examples/dom/canvas.cpp): + + + +Complex [examples](https://github.com/ArthurSonzogni/FTXUI/blob/master/examples/component/canvas_animated.cpp): + + +</details> + +#### Component + +ftxui/component produces dynamic UI, reactive to the user's input. It defines a set of ftxui::Component. A component reacts to Events (keyboard, mouse, resize, ...) and Renders as an Element (see previous section). + +Prebuilt components are declared in [<ftxui/component/component.hpp>](https://arthursonzogni.github.io/FTXUI/component_8hpp_source.html) + +<details><summary>Gallery</summary> + +[Gallery](https://arthursonzogni.github.io/FTXUI/examples_2component_2gallery_8cpp-example.html) of multiple components. ([demo](https://arthursonzogni.github.io/FTXUI/examples/?file=component/gallery)) + + + +</details> + +<details><summary>Radiobox</summary> + +[Example](https://arthursonzogni.github.io/FTXUI/examples_2component_2radiobox_8cpp-example.html): + + + +</details> + +<details><summary>Checkbox</summary> + +[Example](https://arthursonzogni.github.io/FTXUI/examples_2component_2checkbox_8cpp-example.html): + + + +</details> + +<details><summary>Input</summary> + +[Example](https://arthursonzogni.github.io/FTXUI/examples_2component_2input_8cpp-example.html): + + + +</details> + +<details><summary>Toggle</summary> + +[Example](https://arthursonzogni.github.io/FTXUI/examples_2component_2toggle_8cpp-example.html): + + + +</details> + + +<details><summary>Slider</summary> + +[Example](https://arthursonzogni.github.io/FTXUI/examples_2component_2slider_8cpp-example.html): + + + +</details> + + +<details><summary>Menu</summary> + +[Example](https://arthursonzogni.github.io/FTXUI/examples_2component_2menu_8cpp-example.html): + + + +</details> + + +<details><summary>ResizableSplit</summary> + +[Example](https://arthursonzogni.github.io/FTXUI/examples_2component_2resizable_split_8cpp-example.html): + + +</details> + + +<details><summary>Dropdown</summary> + +[Example](https://arthursonzogni.github.io/FTXUI/examples_2component_2dropdown_8cpp-example.html): + + + +</details> + +<details><summary>Tab</summary> + +[Vertical](https://arthursonzogni.github.io/FTXUI/examples_2component_2tab_vertical_8cpp-example.html): + + + +[Horizontal](https://arthursonzogni.github.io/FTXUI/examples_2component_2tab_horizontal_8cpp-example.html): + +  + + + +</details> + +## Libraries for FTXUI +- *Want to share a useful Component for FTXUI? Feel free to add yours here* +- [ftxui-grid-container](https://github.com/mingsheng13/grid-container-ftxui) +- [ftxui-ip-input](https://github.com/mingsheng13/ip-input-ftxui) +- [ftxui-image-view](https://github.com/ljrrjl/ftxui-image-view.git): For Image Display. + + +## Project using FTXUI + +Feel free to add your projects here: +- [json-tui](https://github.com/ArthurSonzogni/json-tui) +- [git-tui](https://github.com/ArthurSonzogni/git-tui) +- [ostree-tui](https://github.com/AP-Sensing/ostree-tui) +- [rgb-tui](https://github.com/ArthurSonzogni/rgb-tui) +- [chrome-log-beautifier](https://github.com/ArthurSonzogni/chrome-log-beautifier) +- [x86-64 CPU Architecture Simulation](https://github.com/AnisBdz/CPU) +- [ltuiny](https://github.com/adrianoviana87/ltuiny) +- [i3-termdialogs](https://github.com/mibli/i3-termdialogs) +- [simpPRU](https://github.com/VedantParanjape/simpPRU) +- [Pigeon ROS TUI](https://github.com/PigeonSensei/Pigeon_ros_tui) +- [hastur](https://github.com/robinlinden/hastur) +- [CryptoCalculator](https://github.com/brevis/CryptoCalculator) +- [todoman](https://github.com/aaleino/todoman) +- [TimeAccumulator](https://github.com/asari555/TimeAccumulator) +- [vantage](https://github.com/gokulmaxi/vantage) +- [tabdeeli](https://github.com/typon/tabdeeli) +- [tiles](https://github.com/tusharpm/tiles) +- [cachyos-cli-installer](https://github.com/cachyos/new-cli-installer) +- [beagle-config](https://github.com/SAtacker/beagle-config) +- [turing_cmd](https://github.com/DanArmor/turing_cmd) +- [StartUp](https://github.com/StubbornVegeta/StartUp) +- [eCAL monitor](https://github.com/eclipse-ecal/ecal) +- [Path Finder](https://github.com/Ruebled/Path_Finder) +- [rw-tui](https://github.com/LeeKyuHyuk/rw-tui) +- [resource-monitor](https://github.com/catalincd/resource-monitor) +- [ftxuiFileReader](https://github.com/J0sephDavis/ftxuiFileReader) +- [ftxui_CPUMeter](https://github.com/tzzzzzzzx/ftxui_CPUMeter) +- [Captain's log](https://github.com/nikoladucak/caps-log) +- [FTowerX](https://github.com/MhmRhm/FTowerX) +- [Caravan](https://github.com/r3w0p/caravan) +- [Step-Writer](https://github.com/BrianAnakPintar/step-writer) +- [XJ music](https://github.com/xjmusic/xjmusic) +- [UDP chat](https://github.com/Sergeydigl3/udp-chat-tui) +- [2048-cpp](https://github.com/Chessom/2048-cpp) +- [Memory game](https://github.com/mikolajlubiak/memory) +- [Terminal Animation](https://github.com/mikolajlubiak/terminal_animation) +- [pciex](https://github.com/s0nx/pciex) +- [Fallout terminal hacking](https://github.com/gshigin/yet-another-fallout-terminal-hacking-game) +- [Lazylist](https://github.com/zhuyongqi9/lazylist) +- [TUISIC](https://github.com/Dark-Kernel/tuisic) +- [inLimbo](https://github.com/nots1dd/inLimbo) +- [BestEdrOfTheMarket](https://github.com/Xacone/BestEdrOfTheMarket) +- [terminal-rain](https://github.com/Oakamoore/terminal-rain) +- [keywords](https://github.com/Oakamoore/keywords) ([Play web version :heart:](https://oakamoore.itch.io/keywords)) +- [FTB - tertminal file browser](https://github.com/Cyxuan0311/FTB) + +### [cpp-best-practices/game_jam](https://github.com/cpp-best-practices/game_jam) + +Several games using the FTXUI have been made during the Game Jam: +- [TermBreaker](https://github.com/ArthurSonzogni/termBreaker) [**[Play web version]**](https://arthursonzogni.com/TermBreaker/) +- [Minesweeper Marathon](https://github.com/cpp-best-practices/game_jam/blob/main/Jam1_April_2022/minesweeper_marathon.md) [**[Play web version]**](https://barlasgarden.com/minesweeper/index.html) +- [Grand Rounds](https://github.com/cpp-best-practices/game_jam/blob/main/Jam1_April_2022/grandrounds.md) +- [LightsRound](https://github.com/cpp-best-practices/game_jam/blob/main/Jam1_April_2022/LightsRound.v.0.1.0.md) +- [DanteO](https://github.com/cpp-best-practices/game_jam/blob/main/Jam1_April_2022/danteo.md) +- [Sumo](https://github.com/cpp-best-practices/game_jam/blob/main/Jam1_April_2022/sumo.md) +- [Drag Me aROUND](https://github.com/cpp-best-practices/game_jam/blob/main/Jam1_April_2022/drag_me_around.md) +- [DisarmSelfDestruct](https://github.com/cpp-best-practices/game_jam/blob/main/Jam1_April_2022/DisarmSelfDestruct.md) +- [TheWorld](https://github.com/cpp-best-practices/game_jam/blob/main/Jam1_April_2022/TheWorld.md) +- [smoothlife](https://github.com/cpp-best-practices/game_jam/blob/main/Jam1_April_2022/smoothlife.md) +- [Consu](https://github.com/cpp-best-practices/game_jam/blob/main/Jam1_April_2022/consu.md) + +## Utilization + +It is **highly** recommended to use CMake FetchContent to depend on FTXUI so you may specify which commit you would like to depend on. +```cmake +include(FetchContent) + +FetchContent_Declare(ftxui + GIT_REPOSITORY https://github.com/ArthurSonzogni/ftxui + GIT_TAG v6.0.2 +) + +FetchContent_GetProperties(ftxui) +if(NOT ftxui_POPULATED) + FetchContent_Populate(ftxui) + add_subdirectory(${ftxui_SOURCE_DIR} ${ftxui_BINARY_DIR} EXCLUDE_FROM_ALL) +endif() +``` + +If you don't, FTXUI may be used from the following packages: +- [vcpkg](https://vcpkgx.com/details.html?package=ftxui) +- [Arch Linux PKGBUILD](https://aur.archlinux.org/packages/ftxui-git/). +- [conan.io](https://conan.io/center/ftxui) +- [openSUSE](https://build.opensuse.org/package/show/devel:libraries:c_c++/ftxui) +- +[](https://repology.org/project/libftxui/versions) + +If you choose to build and link FTXUI yourself, `ftxui-component` must be first in the linking order relative to the other FTXUI libraries, i.e. +```bash +g++ . . . -lftxui-component -lftxui-dom -lftxui-screen . . . +``` + + + +## Contributors + +<a href="https://github.com/ArthurSonzogni/FTXUI/graphs/contributors"> + <img src="https://contrib.rocks/image?repo=ArthurSonzogni/FTXUI" /> +</a> diff --git a/contrib/libs/ftxui/include/ftxui/component/animation.hpp b/contrib/libs/ftxui/include/ftxui/component/animation.hpp new file mode 100644 index 00000000000..a04aea56a8d --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/component/animation.hpp @@ -0,0 +1,113 @@ +// Copyright 2022 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_ANIMATION_HPP +#define FTXUI_ANIMATION_HPP + +#include <chrono> // for milliseconds, duration, steady_clock, time_point +#include <functional> // for function + +namespace ftxui::animation { +// Components who haven't completed their animation can call this function to +// request a new frame to be drawn later. +// +// When there is no new events and no animations to complete, no new frame is +// drawn. +void RequestAnimationFrame(); + +using Clock = std::chrono::steady_clock; +using TimePoint = std::chrono::time_point<Clock>; +using Duration = std::chrono::duration<float>; + +// Parameter of Component::OnAnimation(param). +class Params { + public: + explicit Params(Duration duration) : duration_(duration) {} + + /// The duration this animation step represents. + Duration duration() const { return duration_; } + + private: + Duration duration_; +}; + +namespace easing { +using Function = std::function<float(float)>; +// Linear interpolation (no easing) +float Linear(float p); + +// Quadratic easing; p^2 +float QuadraticIn(float p); +float QuadraticOut(float p); +float QuadraticInOut(float p); + +// Cubic easing; p^3 +float CubicIn(float p); +float CubicOut(float p); +float CubicInOut(float p); + +// Quartic easing; p^4 +float QuarticIn(float p); +float QuarticOut(float p); +float QuarticInOut(float p); + +// Quintic easing; p^5 +float QuinticIn(float p); +float QuinticOut(float p); +float QuinticInOut(float p); + +// Sine wave easing; sin(p * PI/2) +float SineIn(float p); +float SineOut(float p); +float SineInOut(float p); + +// Circular easing; sqrt(1 - p^2) +float CircularIn(float p); +float CircularOut(float p); +float CircularInOut(float p); + +// Exponential easing, base 2 +float ExponentialIn(float p); +float ExponentialOut(float p); +float ExponentialInOut(float p); + +// Exponentially-damped sine wave easing +float ElasticIn(float p); +float ElasticOut(float p); +float ElasticInOut(float p); + +// Overshooting cubic easing; +float BackIn(float p); +float BackOut(float p); +float BackInOut(float p); + +// Exponentially-decaying bounce easing +float BounceIn(float p); +float BounceOut(float p); +float BounceInOut(float p); +} // namespace easing + +class Animator { + public: + explicit Animator(float* from, + float to = 0.f, + Duration duration = std::chrono::milliseconds(250), + easing::Function easing_function = easing::Linear, + Duration delay = std::chrono::milliseconds(0)); + + void OnAnimation(Params&); + + float to() const { return to_; } + + private: + float* value_; + float from_; + float to_; + Duration duration_; + easing::Function easing_function_; + Duration current_; +}; + +} // namespace ftxui::animation + +#endif /* end of include guard: FTXUI_ANIMATION_HPP */ diff --git a/contrib/libs/ftxui/include/ftxui/component/captured_mouse.hpp b/contrib/libs/ftxui/include/ftxui/component/captured_mouse.hpp new file mode 100644 index 00000000000..234e12f6eee --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/component/captured_mouse.hpp @@ -0,0 +1,22 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_CAPTURED_MOUSE_HPP +#define FTXUI_CAPTURED_MOUSE_HPP + +#include <memory> + +namespace ftxui { +class CapturedMouseInterface { + public: + CapturedMouseInterface() = default; + CapturedMouseInterface(const CapturedMouseInterface&) = default; + CapturedMouseInterface(CapturedMouseInterface&&) = delete; + CapturedMouseInterface& operator=(const CapturedMouseInterface&) = default; + CapturedMouseInterface& operator=(CapturedMouseInterface&&) = delete; + virtual ~CapturedMouseInterface() = default; +}; +using CapturedMouse = std::unique_ptr<CapturedMouseInterface>; +} // namespace ftxui + +#endif /* end of include guard: FTXUI_CAPTURED_MOUSE_HPP */ diff --git a/contrib/libs/ftxui/include/ftxui/component/component.hpp b/contrib/libs/ftxui/include/ftxui/component/component.hpp new file mode 100644 index 00000000000..f405d126ad7 --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/component/component.hpp @@ -0,0 +1,142 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_COMPONENT_HPP +#define FTXUI_COMPONENT_HPP + +#include <functional> // for function +#include <memory> // for make_shared, shared_ptr +#include <utility> // for forward + +#include "ftxui/component/component_base.hpp" // for Component, Components +#include "ftxui/component/component_options.hpp" // for ButtonOption, CheckboxOption, MenuOption +#include "ftxui/dom/elements.hpp" // for Element +#include "ftxui/util/ref.hpp" // for ConstRef, Ref, ConstStringRef, ConstStringListRef, StringRef + +namespace ftxui { +struct ButtonOption; +struct CheckboxOption; +struct Event; +struct InputOption; +struct MenuOption; +struct RadioboxOption; +struct MenuEntryOption; + +template <class T, class... Args> +std::shared_ptr<T> Make(Args&&... args) { + return std::make_shared<T>(std::forward<Args>(args)...); +} + +// Pipe operator to decorate components. +using ComponentDecorator = std::function<Component(Component)>; +using ElementDecorator = std::function<Element(Element)>; +Component operator|(Component component, ComponentDecorator decorator); +Component operator|(Component component, ElementDecorator decorator); +Component& operator|=(Component& component, ComponentDecorator decorator); +Component& operator|=(Component& component, ElementDecorator decorator); + +namespace Container { +Component Vertical(Components children); +Component Vertical(Components children, int* selector); +Component Horizontal(Components children); +Component Horizontal(Components children, int* selector); +Component Tab(Components children, int* selector); +Component Stacked(Components children); +} // namespace Container + +Component Button(ButtonOption options); +Component Button(ConstStringRef label, + std::function<void()> on_click, + ButtonOption options = ButtonOption::Simple()); + +Component Checkbox(CheckboxOption options); +Component Checkbox(ConstStringRef label, + bool* checked, + CheckboxOption options = CheckboxOption::Simple()); + +Component Input(InputOption options = {}); +Component Input(StringRef content, InputOption options = {}); +Component Input(StringRef content, + StringRef placeholder, + InputOption options = {}); + +Component Menu(MenuOption options); +Component Menu(ConstStringListRef entries, + int* selected_, + MenuOption options = MenuOption::Vertical()); +Component MenuEntry(MenuEntryOption options); +Component MenuEntry(ConstStringRef label, MenuEntryOption options = {}); + +Component Radiobox(RadioboxOption options); +Component Radiobox(ConstStringListRef entries, + int* selected_, + RadioboxOption options = {}); + +Component Dropdown(ConstStringListRef entries, int* selected); +Component Dropdown(DropdownOption options); + +Component Toggle(ConstStringListRef entries, int* selected); + +// General slider constructor: +template <typename T> +Component Slider(SliderOption<T> options); + +// Shorthand without the `SliderOption` constructor: +Component Slider(ConstStringRef label, + Ref<int> value, + ConstRef<int> min = 0, + ConstRef<int> max = 100, + ConstRef<int> increment = 5); +Component Slider(ConstStringRef label, + Ref<float> value, + ConstRef<float> min = 0.f, + ConstRef<float> max = 100.f, + ConstRef<float> increment = 5.f); +Component Slider(ConstStringRef label, + Ref<long> value, + ConstRef<long> min = 0L, + ConstRef<long> max = 100L, + ConstRef<long> increment = 5L); + +Component ResizableSplit(ResizableSplitOption options); +Component ResizableSplitLeft(Component main, Component back, int* main_size); +Component ResizableSplitRight(Component main, Component back, int* main_size); +Component ResizableSplitTop(Component main, Component back, int* main_size); +Component ResizableSplitBottom(Component main, Component back, int* main_size); + +Component Renderer(Component child, std::function<Element()>); +Component Renderer(std::function<Element()>); +Component Renderer(std::function<Element(bool /* focused */)>); +ComponentDecorator Renderer(ElementDecorator); + +Component CatchEvent(Component child, std::function<bool(Event)>); +ComponentDecorator CatchEvent(std::function<bool(Event)> on_event); + +Component Maybe(Component, const bool* show); +Component Maybe(Component, std::function<bool()>); +ComponentDecorator Maybe(const bool* show); +ComponentDecorator Maybe(std::function<bool()>); + +Component Modal(Component main, Component modal, const bool* show_modal); +ComponentDecorator Modal(Component modal, const bool* show_modal); + +Component Collapsible(ConstStringRef label, + Component child, + Ref<bool> show = false); + +Component Hoverable(Component component, bool* hover); +Component Hoverable(Component component, + std::function<void()> on_enter, + std::function<void()> on_leave); +Component Hoverable(Component component, // + std::function<void(bool)> on_change); +ComponentDecorator Hoverable(bool* hover); +ComponentDecorator Hoverable(std::function<void()> on_enter, + std::function<void()> on_leave); +ComponentDecorator Hoverable(std::function<void(bool)> on_change); + +Component Window(WindowOptions option); + +} // namespace ftxui + +#endif /* end of include guard: FTXUI_COMPONENT_HPP */ diff --git a/contrib/libs/ftxui/include/ftxui/component/component_base.hpp b/contrib/libs/ftxui/include/ftxui/component/component_base.hpp new file mode 100644 index 00000000000..fcc67e16db7 --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/component/component_base.hpp @@ -0,0 +1,105 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_COMPONENT_BASE_HPP +#define FTXUI_COMPONENT_BASE_HPP + +#include <memory> // for unique_ptr +#include <vector> // for vector + +#include "ftxui/component/captured_mouse.hpp" // for CaptureMouse +#include "ftxui/dom/elements.hpp" // for Element + +namespace ftxui { + +class Delegate; +class Focus; +struct Event; + +namespace animation { +class Params; +} // namespace animation + +class ComponentBase; +using Component = std::shared_ptr<ComponentBase>; +using Components = std::vector<Component>; + +/// @brief It implement rendering itself as ftxui::Element. It implement +/// keyboard navigation by responding to ftxui::Event. +/// @ingroup component +class ComponentBase { + public: + explicit ComponentBase(Components children) + : children_(std::move(children)) {} + virtual ~ComponentBase(); + ComponentBase() = default; + + // A component is not copyable/movable. + ComponentBase(const ComponentBase&) = delete; + ComponentBase(ComponentBase&&) = delete; + ComponentBase& operator=(const ComponentBase&) = delete; + ComponentBase& operator=(ComponentBase&&) = delete; + + // Component hierarchy: + ComponentBase* Parent() const; + Component& ChildAt(size_t i); + size_t ChildCount() const; + int Index() const; + void Add(Component children); + void Detach(); + void DetachAllChildren(); + + // Renders the component. + Element Render(); + + // Override this function modify how `Render` works. + virtual Element OnRender(); + + // Handles an event. + // By default, reduce on children with a lazy OR. + // + // Returns whether the event was handled or not. + virtual bool OnEvent(Event); + + // Handle an animation step. + virtual void OnAnimation(animation::Params& params); + + // Focus management ---------------------------------------------------------- + // + // If this component contains children, this indicates which one is active, + // nullptr if none is active. + // + // We say an element has the focus if the chain of ActiveChild() from the + // root component contains this object. + virtual Component ActiveChild(); + + // Return true when the component contains focusable elements. + // The non focusable Component will be skipped when navigating using the + // keyboard. + virtual bool Focusable() const; + + // Whether this is the active child of its parent. + bool Active() const; + // Whether all the ancestors are active. + bool Focused() const; + + // Make the |child| to be the "active" one. + virtual void SetActiveChild(ComponentBase* child); + void SetActiveChild(Component child); + + // Configure all the ancestors to give focus to this component. + void TakeFocus(); + + protected: + CapturedMouse CaptureMouse(const Event& event); + + Components children_; + + private: + ComponentBase* parent_ = nullptr; + bool in_render = false; +}; + +} // namespace ftxui + +#endif /* end of include guard: FTXUI_COMPONENT_BASE_HPP */ diff --git a/contrib/libs/ftxui/include/ftxui/component/component_options.hpp b/contrib/libs/ftxui/include/ftxui/component/component_options.hpp new file mode 100644 index 00000000000..e9dd0845365 --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/component/component_options.hpp @@ -0,0 +1,284 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_COMPONENT_COMPONENT_OPTIONS_HPP +#define FTXUI_COMPONENT_COMPONENT_OPTIONS_HPP + +#include <chrono> // for milliseconds +#include <ftxui/component/animation.hpp> // for Duration, QuadraticInOut, Function +#include <ftxui/dom/direction.hpp> // for Direction, Direction::Left, Direction::Right, Direction::Down +#include <ftxui/dom/elements.hpp> // for Element, separator +#include <ftxui/util/ref.hpp> // for Ref, ConstRef, StringRef +#include <functional> // for function +#include <string> // for string + +#include "ftxui/component/component_base.hpp" // for Component +#include "ftxui/screen/color.hpp" // for Color, Color::GrayDark, Color::White + +namespace ftxui { + +/// @brief arguments for |ButtonOption::transform|, |CheckboxOption::transform|, +/// |Radiobox::transform|, |MenuEntryOption::transform|, +/// |MenuOption::transform|. +struct EntryState { + std::string label; ///< The label to display. + bool state; ///< The state of the button/checkbox/radiobox + bool active; ///< Whether the entry is the active one. + bool focused; ///< Whether the entry is one focused by the user. + int index; ///< Index of the entry when applicable or -1. +}; + +struct UnderlineOption { + bool enabled = false; + + Color color_active = Color::White; + Color color_inactive = Color::GrayDark; + + animation::easing::Function leader_function = + animation::easing::QuadraticInOut; + animation::easing::Function follower_function = + animation::easing::QuadraticInOut; + + animation::Duration leader_duration = std::chrono::milliseconds(250); + animation::Duration leader_delay = std::chrono::milliseconds(0); + animation::Duration follower_duration = std::chrono::milliseconds(250); + animation::Duration follower_delay = std::chrono::milliseconds(0); + + void SetAnimation(animation::Duration d, animation::easing::Function f); + void SetAnimationDuration(animation::Duration d); + void SetAnimationFunction(animation::easing::Function f); + void SetAnimationFunction(animation::easing::Function f_leader, + animation::easing::Function f_follower); +}; + +/// @brief Option about a potentially animated color. +/// @ingroup component +struct AnimatedColorOption { + void Set( + Color inactive, + Color active, + animation::Duration duration = std::chrono::milliseconds(250), + animation::easing::Function function = animation::easing::QuadraticInOut); + + bool enabled = false; + Color inactive; + Color active; + animation::Duration duration = std::chrono::milliseconds(250); + animation::easing::Function function = animation::easing::QuadraticInOut; +}; + +struct AnimatedColorsOption { + AnimatedColorOption background; + AnimatedColorOption foreground; +}; + +/// @brief Option for the MenuEntry component. +/// @ingroup component +struct MenuEntryOption { + ConstStringRef label = "MenuEntry"; + std::function<Element(const EntryState& state)> transform; + AnimatedColorsOption animated_colors; +}; + +/// @brief Option for the Menu component. +/// @ingroup component +struct MenuOption { + // Standard constructors: + static MenuOption Horizontal(); + static MenuOption HorizontalAnimated(); + static MenuOption Vertical(); + static MenuOption VerticalAnimated(); + static MenuOption Toggle(); + + ConstStringListRef entries; ///> The list of entries. + Ref<int> selected = 0; ///> The index of the selected entry. + + // Style: + UnderlineOption underline; + MenuEntryOption entries_option; + Direction direction = Direction::Down; + std::function<Element()> elements_prefix; + std::function<Element()> elements_infix; + std::function<Element()> elements_postfix; + + // Observers: + std::function<void()> on_change; ///> Called when the selected entry changes. + std::function<void()> on_enter; ///> Called when the user presses enter. + Ref<int> focused_entry = 0; +}; + +/// @brief Option for the AnimatedButton component. +/// @ingroup component +struct ButtonOption { + // Standard constructors: + static ButtonOption Ascii(); + static ButtonOption Simple(); + static ButtonOption Border(); + static ButtonOption Animated(); + static ButtonOption Animated(Color color); + static ButtonOption Animated(Color background, Color foreground); + static ButtonOption Animated(Color background, + Color foreground, + Color background_active, + Color foreground_active); + + ConstStringRef label = "Button"; + std::function<void()> on_click = [] {}; + + // Style: + std::function<Element(const EntryState&)> transform; + AnimatedColorsOption animated_colors; +}; + +/// @brief Option for the Checkbox component. +/// @ingroup component +struct CheckboxOption { + // Standard constructors: + static CheckboxOption Simple(); + + ConstStringRef label = "Checkbox"; + + Ref<bool> checked = false; + + // Style: + std::function<Element(const EntryState&)> transform; + + // Observer: + /// Called when the user change the state. + std::function<void()> on_change = [] {}; +}; + +/// @brief Used to define style for the Input component. +struct InputState { + Element element; + bool hovered; ///< Whether the input is hovered by the mouse. + bool focused; ///< Whether the input is focused by the user. + bool is_placeholder; ///< Whether the input is empty and displaying the + ///< placeholder. +}; + +/// @brief Option for the Input component. +/// @ingroup component +struct InputOption { + // A set of predefined styles: + + /// @brief Create the default input style: + static InputOption Default(); + /// @brief A white on black style with high margins: + static InputOption Spacious(); + + /// The content of the input. + StringRef content = ""; + + /// The content of the input when it's empty. + StringRef placeholder = ""; + + // Style: + std::function<Element(InputState)> transform; + Ref<bool> password = false; ///< Obscure the input content using '*'. + Ref<bool> multiline = true; ///< Whether the input can be multiline. + Ref<bool> insert = true; ///< Insert or overtype character mode. + + /// Called when the content changes. + std::function<void()> on_change = [] {}; + /// Called when the user presses enter. + std::function<void()> on_enter = [] {}; + + // The char position of the cursor: + Ref<int> cursor_position = 0; +}; + +/// @brief Option for the Radiobox component. +/// @ingroup component +struct RadioboxOption { + // Standard constructors: + static RadioboxOption Simple(); + + // Content: + ConstStringListRef entries; + Ref<int> selected = 0; + + // Style: + std::function<Element(const EntryState&)> transform; + + // Observers: + /// Called when the selected entry changes. + std::function<void()> on_change = [] {}; + Ref<int> focused_entry = 0; +}; + +struct ResizableSplitOption { + Component main; + Component back; + Ref<Direction> direction = Direction::Left; + Ref<int> main_size = + (direction() == Direction::Left || direction() == Direction::Right) ? 20 + : 10; + std::function<Element()> separator_func = [] { return ::ftxui::separator(); }; +}; + +// @brief Option for the `Slider` component. +// @ingroup component +template <typename T> +struct SliderOption { + Ref<T> value; + ConstRef<T> min = T(0); + ConstRef<T> max = T(100); + ConstRef<T> increment = (max() - min()) / 20; + Direction direction = Direction::Right; + Color color_active = Color::White; + Color color_inactive = Color::GrayDark; + std::function<void()> on_change; ///> Called when `value` is updated. +}; + +// Parameter pack used by `WindowOptions::render`. +struct WindowRenderState { + Element inner; ///< The element wrapped inside this window. + const std::string& title; ///< The title of the window. + bool active = false; ///< Whether the window is the active one. + bool drag = false; ///< Whether the window is being dragged. + bool resize = false; ///< Whether the window is being resized. + bool hover_left = false; ///< Whether the resizeable left side is hovered. + bool hover_right = false; ///< Whether the resizeable right side is hovered. + bool hover_top = false; ///< Whether the resizeable top side is hovered. + bool hover_down = false; ///< Whether the resizeable down side is hovered. +}; + +// @brief Option for the `Window` component. +// @ingroup component +struct WindowOptions { + Component inner; ///< The component wrapped by this window. + ConstStringRef title = ""; ///< The title displayed by this window. + + Ref<int> left = 0; ///< The left side position of the window. + Ref<int> top = 0; ///< The top side position of the window. + Ref<int> width = 20; ///< The width of the window. + Ref<int> height = 10; ///< The height of the window. + + Ref<bool> resize_left = true; ///< Can the left side be resized? + Ref<bool> resize_right = true; ///< Can the right side be resized? + Ref<bool> resize_top = true; ///< Can the top side be resized? + Ref<bool> resize_down = true; ///< Can the down side be resized? + + /// An optional function to customize how the window looks like: + std::function<Element(const WindowRenderState&)> render; +}; + +/// @brief Option for the Dropdown component. +/// @ingroup component +/// A dropdown menu is a checkbox opening/closing a radiobox. +struct DropdownOption { + /// Whether the dropdown is open or closed: + Ref<bool> open = false; + // The options for the checkbox: + CheckboxOption checkbox; + // The options for the radiobox: + RadioboxOption radiobox; + // The transformation function: + std::function<Element(bool open, Element checkbox, Element radiobox)> + transform; +}; + +} // namespace ftxui + +#endif /* end of include guard: FTXUI_COMPONENT_COMPONENT_OPTIONS_HPP */ diff --git a/contrib/libs/ftxui/include/ftxui/component/event.hpp b/contrib/libs/ftxui/include/ftxui/component/event.hpp new file mode 100644 index 00000000000..6ce20e06aad --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/component/event.hpp @@ -0,0 +1,152 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_COMPONENT_EVENT_HPP +#define FTXUI_COMPONENT_EVENT_HPP + +#include <ftxui/component/mouse.hpp> // for Mouse +#include <string> // for string, operator== + +namespace ftxui { + +class ScreenInteractive; +class ComponentBase; + +/// @brief Represent an event. It can be key press event, a terminal resize, or +/// more ... +/// +/// For example: +/// - Printable character can be created using Event::Character('a'). +/// - Some special are predefined, like Event::ArrowLeft. +/// - One can find arbitrary code for special Events using: +/// ./example/util/print_key_press +/// For instance, CTLR+A maps to Event::Special({1}); +/// +/// Useful documentation about xterm specification: +/// https://invisible-island.net/xterm/ctlseqs/ctlseqs.html +struct Event { + // --- Constructor section --------------------------------------------------- + static Event Character(std::string); + static Event Character(char); + static Event Character(wchar_t); + static Event Special(std::string); + static Event Mouse(std::string, Mouse mouse); + static Event CursorPosition(std::string, int x, int y); // Internal + static Event CursorShape(std::string, int shape); // Internal + + // --- Arrow --- + static const Event ArrowLeft; + static const Event ArrowRight; + static const Event ArrowUp; + static const Event ArrowDown; + + static const Event ArrowLeftCtrl; + static const Event ArrowRightCtrl; + static const Event ArrowUpCtrl; + static const Event ArrowDownCtrl; + + // --- Other --- + static const Event Backspace; + static const Event Delete; + static const Event Return; + static const Event Escape; + static const Event Tab; + static const Event TabReverse; + + // --- Navigation keys --- + static const Event Insert; + static const Event Home; + static const Event End; + static const Event PageUp; + static const Event PageDown; + + // --- Function keys --- + static const Event F1, F2, F3, F4, F5, F6, F7, F8, F9, F10, F11, F12; + + // --- Control keys --- + static const Event a, A, CtrlA, AltA, CtrlAltA; + static const Event b, B, CtrlB, AltB, CtrlAltB; + static const Event c, C, CtrlC, AltC, CtrlAltC; + static const Event d, D, CtrlD, AltD, CtrlAltD; + static const Event e, E, CtrlE, AltE, CtrlAltE; + static const Event f, F, CtrlF, AltF, CtrlAltF; + static const Event g, G, CtrlG, AltG, CtrlAltG; + static const Event h, H, CtrlH, AltH, CtrlAltH; + static const Event i, I, CtrlI, AltI, CtrlAltI; + static const Event j, J, CtrlJ, AltJ, CtrlAltJ; + static const Event k, K, CtrlK, AltK, CtrlAltK; + static const Event l, L, CtrlL, AltL, CtrlAltL; + static const Event m, M, CtrlM, AltM, CtrlAltM; + static const Event n, N, CtrlN, AltN, CtrlAltN; + static const Event o, O, CtrlO, AltO, CtrlAltO; + static const Event p, P, CtrlP, AltP, CtrlAltP; + static const Event q, Q, CtrlQ, AltQ, CtrlAltQ; + static const Event r, R, CtrlR, AltR, CtrlAltR; + static const Event s, S, CtrlS, AltS, CtrlAltS; + static const Event t, T, CtrlT, AltT, CtrlAltT; + static const Event u, U, CtrlU, AltU, CtrlAltU; + static const Event v, V, CtrlV, AltV, CtrlAltV; + static const Event w, W, CtrlW, AltW, CtrlAltW; + static const Event x, X, CtrlX, AltX, CtrlAltX; + static const Event y, Y, CtrlY, AltY, CtrlAltY; + static const Event z, Z, CtrlZ, AltZ, CtrlAltZ; + + // --- Custom --- + static const Event Custom; + + //--- Method section --------------------------------------------------------- + bool operator==(const Event& other) const { return input_ == other.input_; } + bool operator!=(const Event& other) const { return !operator==(other); } + bool operator<(const Event& other) const { return input_ < other.input_; } + + const std::string& input() const { return input_; } + + bool is_character() const { return type_ == Type::Character; } + std::string character() const { return input_; } + + bool is_mouse() const { return type_ == Type::Mouse; } + struct Mouse& mouse() { return data_.mouse; } + + // --- Internal Method section ----------------------------------------------- + bool is_cursor_position() const { return type_ == Type::CursorPosition; } + int cursor_x() const { return data_.cursor.x; } + int cursor_y() const { return data_.cursor.y; } + + bool is_cursor_shape() const { return type_ == Type::CursorShape; } + int cursor_shape() const { return data_.cursor_shape; } + + // Debug + std::string DebugString() const; + + //--- State section ---------------------------------------------------------- + ScreenInteractive* screen_ = nullptr; + + private: + friend ComponentBase; + friend ScreenInteractive; + enum class Type { + Unknown, + Character, + Mouse, + CursorPosition, + CursorShape, + }; + Type type_ = Type::Unknown; + + struct Cursor { + int x = 0; + int y = 0; + }; + + union { + struct Mouse mouse; + struct Cursor cursor; + int cursor_shape; + } data_ = {}; + + std::string input_; +}; + +} // namespace ftxui + +#endif /* end of include guard: FTXUI_COMPONENT_EVENT_HPP */ diff --git a/contrib/libs/ftxui/include/ftxui/component/loop.hpp b/contrib/libs/ftxui/include/ftxui/component/loop.hpp new file mode 100644 index 00000000000..faef475c65d --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/component/loop.hpp @@ -0,0 +1,41 @@ +// Copyright 2022 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_COMPONENT_LOOP_HPP +#define FTXUI_COMPONENT_LOOP_HPP + +#include <memory> // for shared_ptr + +#include "ftxui/component/component_base.hpp" // for ComponentBase + +namespace ftxui { +class ComponentBase; + +using Component = std::shared_ptr<ComponentBase>; +class ScreenInteractive; + +class Loop { + public: + Loop(ScreenInteractive* screen, Component component); + ~Loop(); + + bool HasQuitted(); + void RunOnce(); + void RunOnceBlocking(); + void Run(); + + // This class is non copyable/movable. + Loop(const Loop&) = default; + Loop(Loop&&) = delete; + Loop& operator=(Loop&&) = delete; + Loop(const ScreenInteractive&) = delete; + Loop& operator=(const Loop&) = delete; + + private: + ScreenInteractive* screen_; + Component component_; +}; + +} // namespace ftxui + +#endif // FTXUI_COMPONENT_LOOP_HPP diff --git a/contrib/libs/ftxui/include/ftxui/component/mouse.hpp b/contrib/libs/ftxui/include/ftxui/component/mouse.hpp new file mode 100644 index 00000000000..cc6c3f1ccc7 --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/component/mouse.hpp @@ -0,0 +1,47 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_COMPONENT_MOUSE_HPP +#define FTXUI_COMPONENT_MOUSE_HPP +namespace ftxui { + +/// @brief A mouse event. It contains the coordinate of the mouse, the button +/// pressed and the modifier (shift, ctrl, meta). +/// @ingroup component +struct Mouse { + enum Button { + Left = 0, + Middle = 1, + Right = 2, + None = 3, + WheelUp = 4, + WheelDown = 5, + WheelLeft = 6, /// Supported terminal only. + WheelRight = 7, /// Supported terminal only. + }; + + enum Motion { + Released = 0, + Pressed = 1, + Moved = 2, + }; + + // Button + Button button = Button::None; + + // Motion + Motion motion = Motion::Pressed; + + // Modifiers: + bool shift = false; + bool meta = false; + bool control = false; + + // Coordinates: + int x = 0; + int y = 0; +}; + +} // namespace ftxui + +#endif /* end of include guard: FTXUI_COMPONENT_MOUSE_HPP */ diff --git a/contrib/libs/ftxui/include/ftxui/component/receiver.hpp b/contrib/libs/ftxui/include/ftxui/component/receiver.hpp new file mode 100644 index 00000000000..55189cf9f6f --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/component/receiver.hpp @@ -0,0 +1,145 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_COMPONENT_RECEIVER_HPP_ +#define FTXUI_COMPONENT_RECEIVER_HPP_ + +#include <algorithm> // for copy, max +#include <atomic> // for atomic, __atomic_base +#include <condition_variable> // for condition_variable +#include <memory> // for unique_ptr, make_unique +#include <mutex> // for mutex, unique_lock +#include <queue> // for queue +#include <utility> // for move + +namespace ftxui { + +// Usage: +// +// Initialization: +// --------------- +// +// auto receiver = MakeReceiver<std:string>(); +// auto sender_1= receiver->MakeSender(); +// auto sender_2 = receiver->MakeSender(); +// +// Then move the senders elsewhere, potentially in a different thread. +// +// On the producer side: +// ---------------------- +// [thread 1] sender_1->Send("hello"); +// [thread 2] sender_2->Send("world"); +// +// On the consumer side: +// --------------------- +// char c; +// while(receiver->Receive(&c)) // Return true as long as there is a producer. +// print(c) +// +// Receiver::Receive() returns true when there are no more senders. + +// clang-format off +template<class T> class SenderImpl; +template<class T> class ReceiverImpl; + +template<class T> using Sender = std::unique_ptr<SenderImpl<T>>; +template<class T> using Receiver = std::unique_ptr<ReceiverImpl<T>>; +template<class T> Receiver<T> MakeReceiver(); +// clang-format on + +// ---- Implementation part ---- + +template <class T> +class SenderImpl { + public: + SenderImpl(const SenderImpl&) = delete; + SenderImpl(SenderImpl&&) = delete; + SenderImpl& operator=(const SenderImpl&) = delete; + SenderImpl& operator=(SenderImpl&&) = delete; + void Send(T t) { receiver_->Receive(std::move(t)); } + ~SenderImpl() { receiver_->ReleaseSender(); } + + Sender<T> Clone() { return receiver_->MakeSender(); } + + private: + friend class ReceiverImpl<T>; + explicit SenderImpl(ReceiverImpl<T>* consumer) : receiver_(consumer) {} + ReceiverImpl<T>* receiver_; +}; + +template <class T> +class ReceiverImpl { + public: + Sender<T> MakeSender() { + std::unique_lock<std::mutex> lock(mutex_); + senders_++; + return std::unique_ptr<SenderImpl<T>>(new SenderImpl<T>(this)); + } + ReceiverImpl() = default; + + bool Receive(T* t) { + while (senders_ || !queue_.empty()) { + std::unique_lock<std::mutex> lock(mutex_); + if (queue_.empty()) { + notifier_.wait(lock); + } + if (queue_.empty()) { + continue; + } + *t = std::move(queue_.front()); + queue_.pop(); + return true; + } + return false; + } + + bool ReceiveNonBlocking(T* t) { + std::unique_lock<std::mutex> lock(mutex_); + if (queue_.empty()) { + return false; + } + *t = queue_.front(); + queue_.pop(); + return true; + } + + bool HasPending() { + std::unique_lock<std::mutex> lock(mutex_); + return !queue_.empty(); + } + + bool HasQuitted() { + std::unique_lock<std::mutex> lock(mutex_); + return queue_.empty() && !senders_; + } + + private: + friend class SenderImpl<T>; + + void Receive(T t) { + { + std::unique_lock<std::mutex> lock(mutex_); + queue_.push(std::move(t)); + } + notifier_.notify_one(); + } + + void ReleaseSender() { + senders_--; + notifier_.notify_one(); + } + + std::mutex mutex_; + std::queue<T> queue_; + std::condition_variable notifier_; + std::atomic<int> senders_{0}; +}; + +template <class T> +Receiver<T> MakeReceiver() { + return std::make_unique<ReceiverImpl<T>>(); +} + +} // namespace ftxui + +#endif // FTXUI_COMPONENT_RECEIVER_HPP_ diff --git a/contrib/libs/ftxui/include/ftxui/component/screen_interactive.hpp b/contrib/libs/ftxui/include/ftxui/component/screen_interactive.hpp new file mode 100644 index 00000000000..a2909fca89e --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/component/screen_interactive.hpp @@ -0,0 +1,167 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_COMPONENT_SCREEN_INTERACTIVE_HPP +#define FTXUI_COMPONENT_SCREEN_INTERACTIVE_HPP + +#include <atomic> // for atomic +#include <ftxui/component/receiver.hpp> // for Receiver, Sender +#include <functional> // for function +#include <memory> // for shared_ptr +#include <string> // for string +#include <thread> // for thread +#include <variant> // for variant + +#include "ftxui/component/animation.hpp" // for TimePoint +#include "ftxui/component/captured_mouse.hpp" // for CapturedMouse +#include "ftxui/component/event.hpp" // for Event +#include "ftxui/component/task.hpp" // for Task, Closure +#include "ftxui/dom/selection.hpp" // for SelectionOption +#include "ftxui/screen/screen.hpp" // for Screen + +namespace ftxui { +class ComponentBase; +class Loop; +struct Event; + +using Component = std::shared_ptr<ComponentBase>; +class ScreenInteractivePrivate; + +class ScreenInteractive : public Screen { + public: + // Constructors: + static ScreenInteractive FixedSize(int dimx, int dimy); + static ScreenInteractive Fullscreen(); + static ScreenInteractive FullscreenPrimaryScreen(); + static ScreenInteractive FullscreenAlternateScreen(); + static ScreenInteractive FitComponent(); + static ScreenInteractive TerminalOutput(); + + // Options. Must be called before Loop(). + void TrackMouse(bool enable = true); + + // Return the currently active screen, nullptr if none. + static ScreenInteractive* Active(); + + // Start/Stop the main loop. + void Loop(Component); + void Exit(); + Closure ExitLoopClosure(); + + // Post tasks to be executed by the loop. + void Post(Task task); + void PostEvent(Event event); + void RequestAnimationFrame(); + + CapturedMouse CaptureMouse(); + + // Decorate a function. The outputted one will execute similarly to the + // inputted one, but with the currently active screen terminal hooks + // temporarily uninstalled. + Closure WithRestoredIO(Closure); + + // FTXUI implements handlers for Ctrl-C and Ctrl-Z. By default, these handlers + // are executed, even if the component catches the event. This avoid users + // handling every event to be trapped in the application. However, in some + // cases, the application may want to handle these events itself. In this + // case, the application can force FTXUI to not handle these events by calling + // the following functions with force=true. + void ForceHandleCtrlC(bool force); + void ForceHandleCtrlZ(bool force); + + // Selection API. + std::string GetSelection(); + void SelectionChange(std::function<void()> callback); + + private: + void ExitNow(); + + void Install(); + void Uninstall(); + + void PreMain(); + void PostMain(); + + bool HasQuitted(); + void RunOnce(Component component); + void RunOnceBlocking(Component component); + + void HandleTask(Component component, Task& task); + bool HandleSelection(bool handled, Event event); + void RefreshSelection(); + void Draw(Component component); + void ResetCursorPosition(); + + void Signal(int signal); + + ScreenInteractive* suspended_screen_ = nullptr; + enum class Dimension { + FitComponent, + Fixed, + Fullscreen, + TerminalOutput, + }; + Dimension dimension_ = Dimension::Fixed; + bool use_alternative_screen_ = false; + ScreenInteractive(int dimx, + int dimy, + Dimension dimension, + bool use_alternative_screen); + + bool track_mouse_ = true; + + Sender<Task> task_sender_; + Receiver<Task> task_receiver_; + + std::string set_cursor_position; + std::string reset_cursor_position; + + std::atomic<bool> quit_{false}; + std::thread event_listener_; + std::thread animation_listener_; + bool animation_requested_ = false; + animation::TimePoint previous_animation_time_; + + int cursor_x_ = 1; + int cursor_y_ = 1; + + bool mouse_captured = false; + bool previous_frame_resized_ = false; + + bool frame_valid_ = false; + + bool force_handle_ctrl_c_ = true; + bool force_handle_ctrl_z_ = true; + + // The style of the cursor to restore on exit. + int cursor_reset_shape_ = 1; + + // Selection API: + CapturedMouse selection_pending_; + struct SelectionData { + int start_x = -1; + int start_y = -1; + int end_x = -2; + int end_y = -2; + bool empty = true; + bool operator==(const SelectionData& other) const; + bool operator!=(const SelectionData& other) const; + }; + SelectionData selection_data_; + SelectionData selection_data_previous_; + std::unique_ptr<Selection> selection_; + std::function<void()> selection_on_change_; + + friend class Loop; + + public: + class Private { + public: + static void Signal(ScreenInteractive& s, int signal) { s.Signal(signal); } + }; + friend Private; +}; + +} // namespace ftxui + +#endif /* end of include guard: FTXUI_COMPONENT_SCREEN_INTERACTIVE_HPP */ diff --git a/contrib/libs/ftxui/include/ftxui/component/task.hpp b/contrib/libs/ftxui/include/ftxui/component/task.hpp new file mode 100644 index 00000000000..6a770e86e4d --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/component/task.hpp @@ -0,0 +1,17 @@ +// Copyright 2022 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_COMPONENT_ANIMATION_HPP +#define FTXUI_COMPONENT_ANIMATION_HPP + +#include <functional> +#include <variant> +#include "ftxui/component/event.hpp" + +namespace ftxui { +class AnimationTask {}; +using Closure = std::function<void()>; +using Task = std::variant<Event, Closure, AnimationTask>; +} // namespace ftxui + +#endif // FTXUI_COMPONENT_ANIMATION_HPP diff --git a/contrib/libs/ftxui/include/ftxui/dom/canvas.hpp b/contrib/libs/ftxui/include/ftxui/dom/canvas.hpp new file mode 100644 index 00000000000..5d33d67a92a --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/dom/canvas.hpp @@ -0,0 +1,147 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_DOM_CANVAS_HPP +#define FTXUI_DOM_CANVAS_HPP + +#include <cstddef> // for size_t +#include <functional> // for function +#include <string> // for string +#include <unordered_map> // for unordered_map + +#include "ftxui/screen/color.hpp" // for Color +#include "ftxui/screen/image.hpp" // for Pixel, Image + +#ifdef DrawText +// Workaround for WinUsr.h (via Windows.h) defining macros that break things. +// https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-drawtext +#undef DrawText +#endif + +namespace ftxui { + +struct Canvas { + public: + Canvas() = default; + Canvas(int width, int height); + + // Getters: + int width() const { return width_; } + int height() const { return height_; } + Pixel GetPixel(int x, int y) const; + + using Stylizer = std::function<void(Pixel&)>; + + // Draws using braille characters -------------------------------------------- + void DrawPointOn(int x, int y); + void DrawPointOff(int x, int y); + void DrawPointToggle(int x, int y); + void DrawPoint(int x, int y, bool value); + void DrawPoint(int x, int y, bool value, const Stylizer& s); + void DrawPoint(int x, int y, bool value, const Color& color); + void DrawPointLine(int x1, int y1, int x2, int y2); + void DrawPointLine(int x1, int y1, int x2, int y2, const Stylizer& s); + void DrawPointLine(int x1, int y1, int x2, int y2, const Color& color); + void DrawPointCircle(int x, int y, int radius); + void DrawPointCircle(int x, int y, int radius, const Stylizer& s); + void DrawPointCircle(int x, int y, int radius, const Color& color); + void DrawPointCircleFilled(int x, int y, int radius); + void DrawPointCircleFilled(int x, int y, int radius, const Stylizer& s); + void DrawPointCircleFilled(int x, int y, int radius, const Color& color); + void DrawPointEllipse(int x, int y, int r1, int r2); + void DrawPointEllipse(int x, int y, int r1, int r2, const Color& color); + void DrawPointEllipse(int x, int y, int r1, int r2, const Stylizer& s); + void DrawPointEllipseFilled(int x, int y, int r1, int r2); + void DrawPointEllipseFilled(int x, int y, int r1, int r2, const Color& color); + void DrawPointEllipseFilled(int x, int y, int r1, int r2, const Stylizer& s); + + // Draw using box characters ------------------------------------------------- + // Block are of size 1x2. y is considered to be a multiple of 2. + void DrawBlockOn(int x, int y); + void DrawBlockOff(int x, int y); + void DrawBlockToggle(int x, int y); + void DrawBlock(int x, int y, bool value); + void DrawBlock(int x, int y, bool value, const Stylizer& s); + void DrawBlock(int x, int y, bool value, const Color& color); + void DrawBlockLine(int x1, int y1, int x2, int y2); + void DrawBlockLine(int x1, int y1, int x2, int y2, const Stylizer& s); + void DrawBlockLine(int x1, int y1, int x2, int y2, const Color& color); + void DrawBlockCircle(int x1, int y1, int radius); + void DrawBlockCircle(int x1, int y1, int radius, const Stylizer& s); + void DrawBlockCircle(int x1, int y1, int radius, const Color& color); + void DrawBlockCircleFilled(int x1, int y1, int radius); + void DrawBlockCircleFilled(int x1, int y1, int radius, const Stylizer& s); + void DrawBlockCircleFilled(int x1, int y1, int radius, const Color& color); + void DrawBlockEllipse(int x1, int y1, int r1, int r2); + void DrawBlockEllipse(int x1, int y1, int r1, int r2, const Stylizer& s); + void DrawBlockEllipse(int x1, int y1, int r1, int r2, const Color& color); + void DrawBlockEllipseFilled(int x1, int y1, int r1, int r2); + void DrawBlockEllipseFilled(int x1, + int y1, + int r1, + int r2, + const Stylizer& s); + void DrawBlockEllipseFilled(int x1, + int y1, + int r1, + int r2, + const Color& color); + + // Draw using normal characters ---------------------------------------------- + // Draw using character of size 2x4 at position (x,y) + // x is considered to be a multiple of 2. + // y is considered to be a multiple of 4. + void DrawText(int x, int y, const std::string& value); + void DrawText(int x, int y, const std::string& value, const Color& color); + void DrawText(int x, int y, const std::string& value, const Stylizer& style); + + // Draw using directly pixels or images -------------------------------------- + // x is considered to be a multiple of 2. + // y is considered to be a multiple of 4. + void DrawPixel(int x, int y, const Pixel&); + void DrawImage(int x, int y, const Image&); + + // Decorator: + // x is considered to be a multiple of 2. + // y is considered to be a multiple of 4. + void Style(int x, int y, const Stylizer& style); + + private: + bool IsIn(int x, int y) const { + return x >= 0 && x < width_ && y >= 0 && y < height_; + } + + enum CellType { + kCell, // Units of size 2x4 + kBlock, // Units of size 2x2 + kBraille, // Units of size 1x1 + }; + + struct Cell { + CellType type = kCell; + Pixel content; + }; + + struct XY { + int x; + int y; + bool operator==(const XY& other) const { + return x == other.x && y == other.y; + } + }; + + struct XYHash { + size_t operator()(const XY& xy) const { + constexpr size_t shift = 1024; + return size_t(xy.x) * shift + size_t(xy.y); + } + }; + + int width_ = 0; + int height_ = 0; + std::unordered_map<XY, Cell, XYHash> storage_; +}; + +} // namespace ftxui + +#endif // FTXUI_DOM_CANVAS_HPP diff --git a/contrib/libs/ftxui/include/ftxui/dom/deprecated.hpp b/contrib/libs/ftxui/include/ftxui/dom/deprecated.hpp new file mode 100644 index 00000000000..78c66f189fd --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/dom/deprecated.hpp @@ -0,0 +1,16 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_DOM_DEPRECATED_HPP +#define FTXUI_DOM_DEPRECATED_HPP + +#include <ftxui/dom/node.hpp> +#include <string> + +namespace ftxui { +Element text(std::wstring text); +Element vtext(std::wstring text); +Elements paragraph(std::wstring text); +} // namespace ftxui + +#endif // FTXUI_DOM_DEPRECATED_HPP diff --git a/contrib/libs/ftxui/include/ftxui/dom/direction.hpp b/contrib/libs/ftxui/include/ftxui/dom/direction.hpp new file mode 100644 index 00000000000..2a38f0924d5 --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/dom/direction.hpp @@ -0,0 +1,17 @@ +// Copyright 2023 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_DOM_DIRECTION_HPP +#define FTXUI_DOM_DIRECTION_HPP + +namespace ftxui { +enum class Direction { + Up = 0, + Down = 1, + Left = 2, + Right = 3, +}; + +} // namespace ftxui + +#endif /* end of include guard: FTXUI_DOM_DIRECTION_HPP */ diff --git a/contrib/libs/ftxui/include/ftxui/dom/elements.hpp b/contrib/libs/ftxui/include/ftxui/dom/elements.hpp new file mode 100644 index 00000000000..4978ded6330 --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/dom/elements.hpp @@ -0,0 +1,202 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_DOM_ELEMENTS_HPP +#define FTXUI_DOM_ELEMENTS_HPP + +#include <functional> +#include <memory> + +#include "ftxui/dom/canvas.hpp" +#include "ftxui/dom/direction.hpp" +#include "ftxui/dom/flexbox_config.hpp" +#include "ftxui/dom/linear_gradient.hpp" +#include "ftxui/dom/node.hpp" +#include "ftxui/screen/box.hpp" +#include "ftxui/screen/color.hpp" +#include "ftxui/screen/terminal.hpp" +#include "ftxui/util/ref.hpp" + +namespace ftxui { +class Node; +using Element = std::shared_ptr<Node>; +using Elements = std::vector<Element>; +using Decorator = std::function<Element(Element)>; +using GraphFunction = std::function<std::vector<int>(int, int)>; + +enum BorderStyle { + LIGHT, + DASHED, + HEAVY, + DOUBLE, + ROUNDED, + EMPTY, +}; + +// Pipe elements into decorator togethers. +// For instance the next lines are equivalents: +// -> text("ftxui") | bold | underlined +// -> underlined(bold(text("FTXUI"))) +Element operator|(Element, Decorator); +Element& operator|=(Element&, Decorator); +Elements operator|(Elements, Decorator); +Decorator operator|(Decorator, Decorator); + +// --- Widget --- +Element text(std::string text); +Element vtext(std::string text); +Element separator(); +Element separatorLight(); +Element separatorDashed(); +Element separatorHeavy(); +Element separatorDouble(); +Element separatorEmpty(); +Element separatorStyled(BorderStyle); +Element separator(Pixel); +Element separatorCharacter(std::string); +Element separatorHSelector(float left, + float right, + Color unselected_color, + Color selected_color); +Element separatorVSelector(float up, + float down, + Color unselected_color, + Color selected_color); +Element gauge(float progress); +Element gaugeLeft(float progress); +Element gaugeRight(float progress); +Element gaugeUp(float progress); +Element gaugeDown(float progress); +Element gaugeDirection(float progress, Direction direction); +Element border(Element); +Element borderLight(Element); +Element borderDashed(Element); +Element borderHeavy(Element); +Element borderDouble(Element); +Element borderRounded(Element); +Element borderEmpty(Element); +Decorator borderStyled(BorderStyle); +Decorator borderStyled(BorderStyle, Color); +Decorator borderStyled(Color); +Decorator borderWith(const Pixel&); +Element window(Element title, Element content, BorderStyle border = ROUNDED); +Element spinner(int charset_index, size_t image_index); +Element paragraph(const std::string& text); +Element paragraphAlignLeft(const std::string& text); +Element paragraphAlignRight(const std::string& text); +Element paragraphAlignCenter(const std::string& text); +Element paragraphAlignJustify(const std::string& text); +Element graph(GraphFunction); +Element emptyElement(); +Element canvas(ConstRef<Canvas>); +Element canvas(int width, int height, std::function<void(Canvas&)>); +Element canvas(std::function<void(Canvas&)>); + +// -- Decorator --- +Element bold(Element); +Element dim(Element); +Element italic(Element); +Element inverted(Element); +Element underlined(Element); +Element underlinedDouble(Element); +Element blink(Element); +Element strikethrough(Element); +Decorator color(Color); +Decorator bgcolor(Color); +Decorator color(const LinearGradient&); +Decorator bgcolor(const LinearGradient&); +Element color(Color, Element); +Element bgcolor(Color, Element); +Element color(const LinearGradient&, Element); +Element bgcolor(const LinearGradient&, Element); +Decorator focusPosition(int x, int y); +Decorator focusPositionRelative(float x, float y); +Element automerge(Element child); +Decorator hyperlink(std::string link); +Element hyperlink(std::string link, Element child); +Element selectionStyleReset(Element); +Decorator selectionColor(Color foreground); +Decorator selectionBackgroundColor(Color foreground); +Decorator selectionForegroundColor(Color foreground); +Decorator selectionStyle(std::function<void(Pixel&)> style); + +// --- Layout is +// Horizontal, Vertical or stacked set of elements. +Element hbox(Elements); +Element vbox(Elements); +Element dbox(Elements); +Element flexbox(Elements, FlexboxConfig config = FlexboxConfig()); +Element gridbox(std::vector<Elements> lines); + +Element hflow(Elements); // Helper: default flexbox with row direction. +Element vflow(Elements); // Helper: default flexbox with column direction. + +// -- Flexibility --- +// Define how to share the remaining space when not all of it is used inside a +// container. +Element flex(Element); // Expand/Minimize if possible/needed. +Element flex_grow(Element); // Expand element if possible. +Element flex_shrink(Element); // Minimize element if needed. + +Element xflex(Element); // Expand/Minimize if possible/needed on X axis. +Element xflex_grow(Element); // Expand element if possible on X axis. +Element xflex_shrink(Element); // Minimize element if needed on X axis. + +Element yflex(Element); // Expand/Minimize if possible/needed on Y axis. +Element yflex_grow(Element); // Expand element if possible on Y axis. +Element yflex_shrink(Element); // Minimize element if needed on Y axis. + +Element notflex(Element); // Reset the flex attribute. +Element filler(); // A blank expandable element. + +// -- Size override; +enum WidthOrHeight { WIDTH, HEIGHT }; +enum Constraint { LESS_THAN, EQUAL, GREATER_THAN }; +Decorator size(WidthOrHeight, Constraint, int value); + +// --- Frame --- +// A frame is a scrollable area. The internal area is potentially larger than +// the external one. The internal area is scrolled in order to make visible the +// focused element. +Element frame(Element); +Element xframe(Element); +Element yframe(Element); +Element focus(Element); +Element select(Element e); // Deprecated - Alias for focus. + +// --- Cursor --- +// Those are similar to `focus`, but also change the shape of the cursor. +Element focusCursorBlock(Element); +Element focusCursorBlockBlinking(Element); +Element focusCursorBar(Element); +Element focusCursorBarBlinking(Element); +Element focusCursorUnderline(Element); +Element focusCursorUnderlineBlinking(Element); + +// --- Misc --- +Element vscroll_indicator(Element); +Element hscroll_indicator(Element); +Decorator reflect(Box& box); +// Before drawing the |element| clear the pixel below. This is useful in +// combinaison with dbox. +Element clear_under(Element element); + +// --- Util -------------------------------------------------------------------- +Element hcenter(Element); +Element vcenter(Element); +Element center(Element); +Element align_right(Element); +Element nothing(Element element); + +namespace Dimension { +Dimensions Fit(Element&, bool extend_beyond_screen = false); +} // namespace Dimension + +} // namespace ftxui + +// Make container able to take any number of children as input. +#include "ftxui/dom/take_any_args.hpp" + +// Include old definitions using wstring. +#include "ftxui/dom/deprecated.hpp" +#endif // FTXUI_DOM_ELEMENTS_HPP diff --git a/contrib/libs/ftxui/include/ftxui/dom/flexbox_config.hpp b/contrib/libs/ftxui/include/ftxui/dom/flexbox_config.hpp new file mode 100644 index 00000000000..8ae26779c8f --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/dom/flexbox_config.hpp @@ -0,0 +1,114 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_DOM_FLEXBOX_CONFIG_HPP +#define FTXUI_DOM_FLEXBOX_CONFIG_HPP + +/* + This replicate the CSS flexbox model. + See guide for documentation: + https://css-tricks.com/snippets/css/a-guide-to-flexbox/ +*/ + +namespace ftxui { + +struct FlexboxConfig { + /// This establishes the main-axis, thus defining the direction flex items are + /// placed in the flex container. Flexbox is (aside wrapping) single-direction + /// layout concept. Think of flex items as primarily laying out either in + /// horizontal rows or vertical columns. + enum class Direction { + Row, ///< Flex items are laid out in a row. + RowInversed, ///< Flex items are laid out in a row, but in reverse order. + Column, ///< Flex items are laid out in a column. + ColumnInversed ///< Flex items are laid out in a column, but in reverse + ///< order. + }; + Direction direction = Direction::Row; + + /// By default, flex items will all try to fit onto one line. You can change + /// that and allow the items to wrap as needed with this property. + enum class Wrap { + NoWrap, ///< Flex items will all try to fit onto one line. + Wrap, ///< Flex items will wrap onto multiple lines. + WrapInversed, ///< Flex items will wrap onto multiple lines, but in reverse + ///< order. + }; + Wrap wrap = Wrap::Wrap; + + /// This defines the alignment along the main axis. It helps distribute extra + /// free space leftover when either all the flex items on a line are + /// inflexible, or are flexible but have reached their maximum size. It also + /// exerts some control over the alignment of items when they overflow the + /// line. + enum class JustifyContent { + /// Items are aligned to the start of flexbox's direction. + FlexStart, + /// Items are aligned to the end of flexbox's direction. + FlexEnd, + /// Items are centered along the line. + Center, + /// Items are stretched to fill the line. + Stretch, + /// Items are evenly distributed in the line; first item is on the start + // line, last item on the end line + SpaceBetween, + /// Items are evenly distributed in the line with equal space around them. + /// Note that visually the spaces aren’t equal, since all the items have + /// equal space on both sides. The first item will have one unit of space + /// against the container edge, but two units of space between the next item + /// because that next item has its own spacing that applies. + SpaceAround, + /// Items are distributed so that the spacing between any two items (and the + /// space to the edges) is equal. + SpaceEvenly, + }; + JustifyContent justify_content = JustifyContent::FlexStart; + + /// This defines the default behavior for how flex items are laid out along + /// the cross axis on the current line. Think of it as the justify-content + /// version for the cross-axis (perpendicular to the main-axis). + enum class AlignItems { + FlexStart, ///< items are placed at the start of the cross axis. + FlexEnd, ///< items are placed at the end of the cross axis. + Center, ///< items are centered along the cross axis. + Stretch, ///< items are stretched to fill the cross axis. + }; + AlignItems align_items = AlignItems::FlexStart; + + // This aligns a flex container’s lines within when there is extra space in + // the cross-axis, similar to how justify-content aligns individual items + // within the main-axis. + enum class AlignContent { + FlexStart, ///< items are placed at the start of the cross axis. + FlexEnd, ///< items are placed at the end of the cross axis. + Center, ///< items are centered along the cross axis. + Stretch, ///< items are stretched to fill the cross axis. + SpaceBetween, ///< items are evenly distributed in the cross axis. + SpaceAround, ///< tems evenly distributed with equal space around each + ///< line. + SpaceEvenly, ///< items are evenly distributed in the cross axis with equal + ///< space around them. + }; + AlignContent align_content = AlignContent::FlexStart; + + int gap_x = 0; + int gap_y = 0; + + // Constructor pattern. For chained use like: + // ``` + // FlexboxConfig() + // .Set(FlexboxConfig::Direction::Row) + // .Set(FlexboxConfig::Wrap::Wrap); + // ``` + FlexboxConfig& Set(FlexboxConfig::Direction); + FlexboxConfig& Set(FlexboxConfig::Wrap); + FlexboxConfig& Set(FlexboxConfig::JustifyContent); + FlexboxConfig& Set(FlexboxConfig::AlignItems); + FlexboxConfig& Set(FlexboxConfig::AlignContent); + FlexboxConfig& SetGap(int gap_x, int gap_y); +}; + +} // namespace ftxui + +#endif // FTXUI_DOM_FLEXBOX_CONFIG_HPP diff --git a/contrib/libs/ftxui/include/ftxui/dom/linear_gradient.hpp b/contrib/libs/ftxui/include/ftxui/dom/linear_gradient.hpp new file mode 100644 index 00000000000..da7e77b893a --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/dom/linear_gradient.hpp @@ -0,0 +1,51 @@ +// Copyright 2023 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_DOM_LINEAR_GRADIENT_HPP +#define FTXUI_DOM_LINEAR_GRADIENT_HPP + +#include <optional> +#include <vector> + +#include "ftxui/screen/color.hpp" // for Colors + +namespace ftxui { + +/// @brief A class representing the settings for linear-gradient color effect. +/// +/// Example: +/// ```cpp +/// LinearGradient() +/// .Angle(45) +/// .Stop(Color::Red, 0.0) +/// .Stop(Color::Green, 0.5) +/// .Stop(Color::Blue, 1.0); +/// ``` +/// +/// There are also shorthand constructors: +/// ```cpp +/// LinearGradient(Color::Red, Color::Blue); +/// LinearGradient(45, Color::Red, Color::Blue); +/// ``` +struct LinearGradient { + float angle = 0.f; + struct Stop { + Color color = Color::Default; + std::optional<float> position; + }; + std::vector<Stop> stops; + + // Simple constructor + LinearGradient(); + LinearGradient(Color begin, Color end); + LinearGradient(float angle, Color begin, Color end); + + // Modifier using the builder pattern. + LinearGradient& Angle(float angle); + LinearGradient& Stop(Color color, float position); + LinearGradient& Stop(Color color); +}; + +} // namespace ftxui + +#endif // FTXUI_DOM_LINEAR_GRADIENT_HPP diff --git a/contrib/libs/ftxui/include/ftxui/dom/node.hpp b/contrib/libs/ftxui/include/ftxui/dom/node.hpp new file mode 100644 index 00000000000..6357ee65482 --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/dom/node.hpp @@ -0,0 +1,79 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_DOM_NODE_HPP +#define FTXUI_DOM_NODE_HPP + +#include <memory> // for shared_ptr +#include <vector> // for vector + +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/dom/selection.hpp" // for Selection +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" + +namespace ftxui { + +class Node; +class Screen; + +using Element = std::shared_ptr<Node>; +using Elements = std::vector<Element>; + +class Node { + public: + Node(); + explicit Node(Elements children); + Node(const Node&) = delete; + Node(const Node&&) = delete; + Node& operator=(const Node&) = delete; + Node& operator=(const Node&&) = delete; + + virtual ~Node(); + + // Step 1: Compute layout requirement. Tell parent what dimensions this + // element wants to be. + // Propagated from Children to Parents. + virtual void ComputeRequirement(); + Requirement requirement() { return requirement_; } + + // Step 2: Assign this element its final dimensions. + // Propagated from Parents to Children. + virtual void SetBox(Box box); + + // Step 3: (optional) Selection + // Propagated from Parents to Children. + virtual void Select(Selection& selection); + + // Step 4: Draw this element. + virtual void Render(Screen& screen); + + virtual std::string GetSelectedContent(Selection& selection); + + // Layout may not resolve within a single iteration for some elements. This + // allows them to request additionnal iterations. This signal must be + // forwarded to children at least once. + struct Status { + int iteration = 0; + bool need_iteration = false; + }; + virtual void Check(Status* status); + + friend void Render(Screen& screen, Node* node, Selection& selection); + + protected: + Elements children_; + Requirement requirement_; + Box box_; +}; + +void Render(Screen& screen, const Element& element); +void Render(Screen& screen, Node* node); +void Render(Screen& screen, Node* node, Selection& selection); +std::string GetNodeSelectedContent(Screen& screen, + Node* node, + Selection& selection); + +} // namespace ftxui + +#endif // FTXUI_DOM_NODE_HPP diff --git a/contrib/libs/ftxui/include/ftxui/dom/requirement.hpp b/contrib/libs/ftxui/include/ftxui/dom/requirement.hpp new file mode 100644 index 00000000000..b8a5c573bca --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/dom/requirement.hpp @@ -0,0 +1,51 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_DOM_REQUIREMENT_HPP +#define FTXUI_DOM_REQUIREMENT_HPP + +#include "ftxui/screen/box.hpp" +#include "ftxui/screen/screen.hpp" + +namespace ftxui { +class Node; + +struct Requirement { + // The required size to fully draw the element. + int min_x = 0; + int min_y = 0; + + // How much flexibility is given to the component. + int flex_grow_x = 0; + int flex_grow_y = 0; + int flex_shrink_x = 0; + int flex_shrink_y = 0; + + // Focus management to support the frame/focus/select element. + struct Focused { + bool enabled = false; + Box box; + Node* node = nullptr; + Screen::Cursor::Shape cursor_shape = Screen::Cursor::Shape::Hidden; + + // Internal for interactions with components. + bool component_active = false; + + // Return whether this requirement should be preferred over the other. + bool Prefer(const Focused& other) const { + if (!other.enabled) { + return false; + } + if (!enabled) { + return true; + } + + return other.component_active && !component_active; + } + }; + Focused focused; +}; + +} // namespace ftxui + +#endif // FTXUI_DOM_REQUIREMENT_HPP diff --git a/contrib/libs/ftxui/include/ftxui/dom/selection.hpp b/contrib/libs/ftxui/include/ftxui/dom/selection.hpp new file mode 100644 index 00000000000..912d9e44752 --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/dom/selection.hpp @@ -0,0 +1,50 @@ +// Copyright 2024 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +#ifndef FTXUI_DOM_SELECTION_HPP +#define FTXUI_DOM_SELECTION_HPP + +#include <functional> + +#include <sstream> +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/pixel.hpp" // for Pixel + +namespace ftxui { + +/// @brief Represent a selection in the terminal. +class Selection { + public: + Selection(); // Empty selection. + Selection(int start_x, int start_y, int end_x, int end_y); + + const Box& GetBox() const; + + Selection SaturateHorizontal(Box box); + Selection SaturateVertical(Box box); + bool IsEmpty() const { return empty_; } + + void AddPart(const std::string& part, int y, int left, int right); + std::string GetParts() { return parts_.str(); } + + private: + Selection(int start_x, int start_y, int end_x, int end_y, Selection* parent); + + const int start_x_ = 0; + const int start_y_ = 0; + const int end_x_ = 0; + const int end_y_ = 0; + const Box box_ = {}; + Selection* const parent_ = this; + const bool empty_ = true; + std::stringstream parts_; + + // The position of the last inserted part. + int x_ = 0; + int y_ = 0; +}; + +} // namespace ftxui + +#endif /* end of include guard: FTXUI_DOM_SELECTION_HPP */ diff --git a/contrib/libs/ftxui/include/ftxui/dom/table.hpp b/contrib/libs/ftxui/include/ftxui/dom/table.hpp new file mode 100644 index 00000000000..5460237dc7c --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/dom/table.hpp @@ -0,0 +1,95 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_DOM_TABLE +#define FTXUI_DOM_TABLE + +#include <string> // for string +#include <vector> // for vector + +#include "ftxui/dom/elements.hpp" // for Element, BorderStyle, LIGHT, Decorator + +namespace ftxui { + +// Usage: +// +// Initialization: +// --------------- +// +// auto table = Table({ +// {"X", "Y"}, +// {"-1", "1"}, +// {"+0", "0"}, +// {"+1", "1"}, +// }); +// +// table.SelectAll().Border(LIGHT); +// +// table.SelectRow(1).Border(DOUBLE); +// table.SelectRow(1).SeparatorInternal(Light); +// +// std::move(table).Element(); + +class Table; +class TableSelection; + +class Table { + public: + Table(); + explicit Table(std::vector<std::vector<std::string>>); + explicit Table(std::vector<std::vector<Element>>); + Table(std::initializer_list<std::vector<std::string>> init); + TableSelection SelectAll(); + TableSelection SelectCell(int column, int row); + TableSelection SelectRow(int row_index); + TableSelection SelectRows(int row_min, int row_max); + TableSelection SelectColumn(int column_index); + TableSelection SelectColumns(int column_min, int column_max); + TableSelection SelectRectangle(int column_min, + int column_max, + int row_min, + int row_max); + Element Render(); + + private: + void Initialize(std::vector<std::vector<Element>>); + friend TableSelection; + std::vector<std::vector<Element>> elements_; + int input_dim_x_ = 0; + int input_dim_y_ = 0; + int dim_x_ = 0; + int dim_y_ = 0; +}; + +class TableSelection { + public: + void Decorate(Decorator); + void DecorateAlternateRow(Decorator, int modulo = 2, int shift = 0); + void DecorateAlternateColumn(Decorator, int modulo = 2, int shift = 0); + + void DecorateCells(Decorator); + void DecorateCellsAlternateColumn(Decorator, int modulo = 2, int shift = 0); + void DecorateCellsAlternateRow(Decorator, int modulo = 2, int shift = 0); + + void Border(BorderStyle border = LIGHT); + void BorderLeft(BorderStyle border = LIGHT); + void BorderRight(BorderStyle border = LIGHT); + void BorderTop(BorderStyle border = LIGHT); + void BorderBottom(BorderStyle border = LIGHT); + + void Separator(BorderStyle border = LIGHT); + void SeparatorVertical(BorderStyle border = LIGHT); + void SeparatorHorizontal(BorderStyle border = LIGHT); + + private: + friend Table; + Table* table_; + int x_min_; + int x_max_; + int y_min_; + int y_max_; +}; + +} // namespace ftxui + +#endif /* end of include guard: FTXUI_DOM_TABLE */ diff --git a/contrib/libs/ftxui/include/ftxui/dom/take_any_args.hpp b/contrib/libs/ftxui/include/ftxui/dom/take_any_args.hpp new file mode 100644 index 00000000000..52c83248d47 --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/dom/take_any_args.hpp @@ -0,0 +1,48 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_DOM_TAKE_ANY_ARGS_HPP +#define FTXUI_DOM_TAKE_ANY_ARGS_HPP + +// IWYU pragma: private, include "ftxui/dom/elements.hpp" +#include <ftxui/dom/node.hpp> + +namespace ftxui { + +template <class T> +void Merge(Elements& /*container*/, T /*element*/) {} + +template <> +inline void Merge(Elements& container, Element element) { + container.push_back(std::move(element)); +} + +template <> +inline void Merge(Elements& container, Elements elements) { + for (auto& element : elements) { + container.push_back(std::move(element)); + } +} + +// Turn a set of arguments into a vector. +template <class... Args> +Elements unpack(Args... args) { + std::vector<Element> vec; + (Merge(vec, std::move(args)), ...); + return vec; +} + +// Make |container| able to take any number of argments. +#define TAKE_ANY_ARGS(container) \ + template <class... Args> \ + Element container(Args... children) { \ + return container(unpack(std::forward<Args>(children)...)); \ + } + +TAKE_ANY_ARGS(vbox) +TAKE_ANY_ARGS(hbox) +TAKE_ANY_ARGS(dbox) +TAKE_ANY_ARGS(hflow) +} // namespace ftxui + +#endif // FTXUI_DOM_TAKE_ANY_ARGS_HPP diff --git a/contrib/libs/ftxui/include/ftxui/screen/box.hpp b/contrib/libs/ftxui/include/ftxui/screen/box.hpp new file mode 100644 index 00000000000..128362d9bae --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/screen/box.hpp @@ -0,0 +1,26 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_SCREEN_BOX_HPP +#define FTXUI_SCREEN_BOX_HPP + +namespace ftxui { + +struct Box { + int x_min = 0; + int x_max = 0; + int y_min = 0; + int y_max = 0; + + static auto Intersection(Box a, Box b) -> Box; + static auto Union(Box a, Box b) -> Box; + void Shift(int x, int y); + bool Contain(int x, int y) const; + bool IsEmpty() const; + bool operator==(const Box& other) const; + bool operator!=(const Box& other) const; +}; + +} // namespace ftxui + +#endif // FTXUI_SCREEN_BOX_HPP diff --git a/contrib/libs/ftxui/include/ftxui/screen/color.hpp b/contrib/libs/ftxui/include/ftxui/screen/color.hpp new file mode 100644 index 00000000000..1489dca6349 --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/screen/color.hpp @@ -0,0 +1,345 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_SCREEN_COLOR_HPP +#define FTXUI_SCREEN_COLOR_HPP + +#include <cstdint> // for uint8_t +#include <string> // for string + +#ifdef RGB +// Workaround for wingdi.h (via Windows.h) defining macros that break things. +// https://docs.microsoft.com/en-us/windows/win32/api/wingdi/nf-wingdi-rgb +#undef RGB +#endif + +namespace ftxui { + +/// @brief A class representing terminal colors. +/// @ingroup screen +class Color { + public: + enum Palette1 : uint8_t; + enum Palette16 : uint8_t; + enum Palette256 : uint8_t; + + // NOLINTBEGIN + Color(); // Transparent. + Color(Palette1 index); // Transparent. + Color(Palette16 index); // Implicit conversion from index to Color. + Color(Palette256 index); // Implicit conversion from index to Color. + // NOLINTEND + Color(uint8_t red, uint8_t green, uint8_t blue, uint8_t alpha = 255); + static Color RGB(uint8_t red, uint8_t green, uint8_t blue); + static Color HSV(uint8_t hue, uint8_t saturation, uint8_t value); + static Color RGBA(uint8_t red, uint8_t green, uint8_t blue, uint8_t alpha); + static Color HSVA(uint8_t hue, + uint8_t saturation, + uint8_t value, + uint8_t alpha); + static Color Interpolate(float t, const Color& a, const Color& b); + static Color Blend(const Color& lhs, const Color& rhs); + + //--------------------------- + // List of colors: + //--------------------------- + // clang-format off + enum Palette1 : uint8_t{ + Default, // Transparent + }; + + enum Palette16 : uint8_t { + Black = 0, + Red = 1, + Green = 2, + Yellow = 3, + Blue = 4, + Magenta = 5, + Cyan = 6, + GrayLight = 7, + GrayDark = 8, + RedLight = 9, + GreenLight = 10, + YellowLight = 11, + BlueLight = 12, + MagentaLight = 13, + CyanLight = 14, + White = 15, + }; + + enum Palette256 : uint8_t { + Aquamarine1 = 122, + Aquamarine1Bis = 86, + Aquamarine3 = 79, + Blue1 = 21, + Blue3 = 19, + Blue3Bis = 20, + BlueViolet = 57, + CadetBlue = 72, + CadetBlueBis = 73, + Chartreuse1 = 118, + Chartreuse2 = 112, + Chartreuse2Bis = 82, + Chartreuse3 = 70, + Chartreuse3Bis = 76, + Chartreuse4 = 64, + CornflowerBlue = 69, + Cornsilk1 = 230, + Cyan1 = 51, + Cyan2 = 50, + Cyan3 = 43, + DarkBlue = 18, + DarkCyan = 36, + DarkGoldenrod = 136, + DarkGreen = 22, + DarkKhaki = 143, + DarkMagenta = 90, + DarkMagentaBis = 91, + DarkOliveGreen1 = 191, + DarkOliveGreen1Bis = 192, + DarkOliveGreen2 = 155, + DarkOliveGreen3 = 107, + DarkOliveGreen3Bis = 113, + DarkOliveGreen3Ter = 149, + DarkOrange = 208, + DarkOrange3 = 130, + DarkOrange3Bis = 166, + DarkRed = 52, + DarkRedBis = 88, + DarkSeaGreen = 108, + DarkSeaGreen1 = 158, + DarkSeaGreen1Bis = 193, + DarkSeaGreen2 = 151, + DarkSeaGreen2Bis = 157, + DarkSeaGreen3 = 115, + DarkSeaGreen3Bis = 150, + DarkSeaGreen4 = 65, + DarkSeaGreen4Bis = 71, + DarkSlateGray1 = 123, + DarkSlateGray2 = 87, + DarkSlateGray3 = 116, + DarkTurquoise = 44, + DarkViolet = 128, + DarkVioletBis = 92, + DeepPink1 = 198, + DeepPink1Bis = 199, + DeepPink2 = 197, + DeepPink3 = 161, + DeepPink3Bis = 162, + DeepPink4 = 125, + DeepPink4Bis = 89, + DeepPink4Ter = 53, + DeepSkyBlue1 = 39, + DeepSkyBlue2 = 38, + DeepSkyBlue3 = 31, + DeepSkyBlue3Bis = 32, + DeepSkyBlue4 = 23, + DeepSkyBlue4Bis = 24, + DeepSkyBlue4Ter = 25, + DodgerBlue1 = 33, + DodgerBlue2 = 27, + DodgerBlue3 = 26, + Gold1 = 220, + Gold3 = 142, + Gold3Bis = 178, + Green1 = 46, + Green3 = 34, + Green3Bis = 40, + Green4 = 28, + GreenYellow = 154, + Grey0 = 16, + Grey100 = 231, + Grey11 = 234, + Grey15 = 235, + Grey19 = 236, + Grey23 = 237, + Grey27 = 238, + Grey3 = 232, + Grey30 = 239, + Grey35 = 240, + Grey37 = 59, + Grey39 = 241, + Grey42 = 242, + Grey46 = 243, + Grey50 = 244, + Grey53 = 102, + Grey54 = 245, + Grey58 = 246, + Grey62 = 247, + Grey63 = 139, + Grey66 = 248, + Grey69 = 145, + Grey7 = 233, + Grey70 = 249, + Grey74 = 250, + Grey78 = 251, + Grey82 = 252, + Grey84 = 188, + Grey85 = 253, + Grey89 = 254, + Grey93 = 255, + Honeydew2 = 194, + HotPink = 205, + HotPink2 = 169, + HotPink3 = 132, + HotPink3Bis = 168, + HotPinkBis = 206, + IndianRed = 131, + IndianRed1 = 203, + IndianRed1Bis = 204, + IndianRedBis = 167, + Khaki1 = 228, + Khaki3 = 185, + LightCoral = 210, + LightCyan1Bis = 195, + LightCyan3 = 152, + LightGoldenrod1 = 227, + LightGoldenrod2 = 186, + LightGoldenrod2Bis = 221, + LightGoldenrod2Ter = 222, + LightGoldenrod3 = 179, + LightGreen = 119, + LightGreenBis = 120, + LightPink1 = 217, + LightPink3 = 174, + LightPink4 = 95, + LightSalmon1 = 216, + LightSalmon3 = 137, + LightSalmon3Bis = 173, + LightSeaGreen = 37, + LightSkyBlue1 = 153, + LightSkyBlue3 = 109, + LightSkyBlue3Bis = 110, + LightSlateBlue = 105, + LightSlateGrey = 103, + LightSteelBlue = 147, + LightSteelBlue1 = 189, + LightSteelBlue3 = 146, + LightYellow3 = 187, + Magenta1 = 201, + Magenta2 = 165, + Magenta2Bis = 200, + Magenta3 = 127, + Magenta3Bis = 163, + Magenta3Ter = 164, + MediumOrchid = 134, + MediumOrchid1 = 171, + MediumOrchid1Bis = 207, + MediumOrchid3 = 133, + MediumPurple = 104, + MediumPurple1 = 141, + MediumPurple2 = 135, + MediumPurple2Bis = 140, + MediumPurple3 = 97, + MediumPurple3Bis = 98, + MediumPurple4 = 60, + MediumSpringGreen = 49, + MediumTurquoise = 80, + MediumVioletRed = 126, + MistyRose1 = 224, + MistyRose3 = 181, + NavajoWhite1 = 223, + NavajoWhite3 = 144, + NavyBlue = 17, + Orange1 = 214, + Orange3 = 172, + Orange4 = 58, + Orange4Bis = 94, + OrangeRed1 = 202, + Orchid = 170, + Orchid1 = 213, + Orchid2 = 212, + PaleGreen1 = 121, + PaleGreen1Bis = 156, + PaleGreen3 = 114, + PaleGreen3Bis = 77, + PaleTurquoise1 = 159, + PaleTurquoise4 = 66, + PaleVioletRed1 = 211, + Pink1 = 218, + Pink3 = 175, + Plum1 = 219, + Plum2 = 183, + Plum3 = 176, + Plum4 = 96, + Purple = 129, + Purple3 = 56, + Purple4 = 54, + Purple4Bis = 55, + PurpleBis = 93, + Red1 = 196, + Red3 = 124, + Red3Bis = 160, + RosyBrown = 138, + RoyalBlue1 = 63, + Salmon1 = 209, + SandyBrown = 215, + SeaGreen1 = 84, + SeaGreen1Bis = 85, + SeaGreen2 = 83, + SeaGreen3 = 78, + SkyBlue1 = 117, + SkyBlue2 = 111, + SkyBlue3 = 74, + SlateBlue1 = 99, + SlateBlue3 = 61, + SlateBlue3Bis = 62, + SpringGreen1 = 48, + SpringGreen2 = 42, + SpringGreen2Bis = 47, + SpringGreen3 = 35, + SpringGreen3Bis = 41, + SpringGreen4 = 29, + SteelBlue = 67, + SteelBlue1 = 75, + SteelBlue1Bis = 81, + SteelBlue3 = 68, + Tan = 180, + Thistle1 = 225, + Thistle3 = 182, + Turquoise2 = 45, + Turquoise4 = 30, + Violet = 177, + Wheat1 = 229, + Wheat4 = 101, + Yellow1 = 226, + Yellow2 = 190, + Yellow3 = 148, + Yellow3Bis = 184, + Yellow4 = 100, + Yellow4Bis = 106, + }; + // clang-format on + + // --- Operators ------ + bool operator==(const Color& rhs) const; + bool operator!=(const Color& rhs) const; + + std::string Print(bool is_background_color) const; + bool IsOpaque() const { return alpha_ == 255; } + + private: + enum class ColorType : uint8_t { + Palette1, + Palette16, + Palette256, + TrueColor, + }; + ColorType type_ = ColorType::Palette1; + uint8_t red_ = 0; + uint8_t green_ = 0; + uint8_t blue_ = 0; + uint8_t alpha_ = 0; +}; + +inline namespace literals { + +/// @brief Creates a color from a combined hex RGB representation, +/// e.g. 0x808000_rgb +Color operator""_rgb(unsigned long long int combined); + +} // namespace literals + +} // namespace ftxui + +#endif // FTXUI_SCREEN_COLOR_HPP diff --git a/contrib/libs/ftxui/include/ftxui/screen/color_info.hpp b/contrib/libs/ftxui/include/ftxui/screen/color_info.hpp new file mode 100644 index 00000000000..8e625e81b9d --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/screen/color_info.hpp @@ -0,0 +1,29 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_SCREEN_COLOR_INFO_HPP +#define FTXUI_SCREEN_COLOR_INFO_HPP + +#include <cstdint> +#include <ftxui/screen/color.hpp> + +namespace ftxui { + +struct ColorInfo { + const char* name; + uint8_t index_256; + uint8_t index_16; + uint8_t red; + uint8_t green; + uint8_t blue; + uint8_t hue; + uint8_t saturation; + uint8_t value; +}; + +ColorInfo GetColorInfo(Color::Palette256 index); +ColorInfo GetColorInfo(Color::Palette16 index); + +} // namespace ftxui + +#endif // FTXUI_SCREEN_COLOR_INFO_HPP diff --git a/contrib/libs/ftxui/include/ftxui/screen/deprecated.hpp b/contrib/libs/ftxui/include/ftxui/screen/deprecated.hpp new file mode 100644 index 00000000000..e3a51f8deee --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/screen/deprecated.hpp @@ -0,0 +1,14 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_SCREEN_DEPRECATED_HPP +#define FTXUI_SCREEN_DEPRECATED_HPP + +#include <string> + +namespace ftxui { +int wchar_width(wchar_t); +int wstring_width(const std::wstring&); +} // namespace ftxui + +#endif // FTXUI_SCREEN_DEPRECATED_HPP diff --git a/contrib/libs/ftxui/include/ftxui/screen/image.hpp b/contrib/libs/ftxui/include/ftxui/screen/image.hpp new file mode 100644 index 00000000000..77e471992b0 --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/screen/image.hpp @@ -0,0 +1,48 @@ +// Copyright 2024 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_SCREEN_IMAGE_HPP +#define FTXUI_SCREEN_IMAGE_HPP + +#include <string> // for string, basic_string, allocator +#include <vector> // for vector + +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/pixel.hpp" // for Pixel + +namespace ftxui { + +/// @brief A rectangular grid of Pixel. +/// @ingroup screen +class Image { + public: + // Constructors: + Image() = delete; + Image(int dimx, int dimy); + + // Access a character in the grid at a given position. + std::string& at(int x, int y); + const std::string& at(int x, int y) const; + + // Access a cell (Pixel) in the grid at a given position. + Pixel& PixelAt(int x, int y); + const Pixel& PixelAt(int x, int y) const; + + // Get screen dimensions. + int dimx() const { return dimx_; } + int dimy() const { return dimy_; } + + // Fill the image with space and default style + void Clear(); + + Box stencil; + + protected: + int dimx_; + int dimy_; + std::vector<std::vector<Pixel>> pixels_; +}; + +} // namespace ftxui + +#endif // FTXUI_SCREEN_IMAGE_HPP diff --git a/contrib/libs/ftxui/include/ftxui/screen/pixel.hpp b/contrib/libs/ftxui/include/ftxui/screen/pixel.hpp new file mode 100644 index 00000000000..b9b66a97abf --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/screen/pixel.hpp @@ -0,0 +1,54 @@ +// Copyright 2024 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_SCREEN_PIXEL_HPP +#define FTXUI_SCREEN_PIXEL_HPP + +#include <cstdint> // for uint8_t +#include <string> // for string, basic_string, allocator +#include "ftxui/screen/color.hpp" // for Color, Color::Default + +namespace ftxui { + +/// @brief A Unicode character and its associated style. +/// @ingroup screen +struct Pixel { + Pixel() + : blink(false), + bold(false), + dim(false), + italic(false), + inverted(false), + underlined(false), + underlined_double(false), + strikethrough(false), + automerge(false) {} + + // A bit field representing the style: + bool blink : 1; + bool bold : 1; + bool dim : 1; + bool italic : 1; + bool inverted : 1; + bool underlined : 1; + bool underlined_double : 1; + bool strikethrough : 1; + bool automerge : 1; + + // The hyperlink associated with the pixel. + // 0 is the default value, meaning no hyperlink. + // It's an index for accessing Screen meta data + uint8_t hyperlink = 0; + + // The graphemes stored into the pixel. To support combining characters, + // like: a?, this can potentially contain multiple codepoints. + std::string character = ""; + + // Colors: + Color background_color = Color::Default; + Color foreground_color = Color::Default; +}; + +} // namespace ftxui + +#endif // FTXUI_SCREEN_PIXEL_HPP diff --git a/contrib/libs/ftxui/include/ftxui/screen/screen.hpp b/contrib/libs/ftxui/include/ftxui/screen/screen.hpp new file mode 100644 index 00000000000..aa2f4f8ccfd --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/screen/screen.hpp @@ -0,0 +1,88 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_SCREEN_SCREEN_HPP +#define FTXUI_SCREEN_SCREEN_HPP + +#include <cstdint> // for uint8_t +#include <functional> // for function +#include <string> // for string, basic_string, allocator +#include <vector> // for vector + +#include "ftxui/screen/image.hpp" // for Pixel, Image +#include "ftxui/screen/terminal.hpp" // for Dimensions +#include "ftxui/util/autoreset.hpp" // for AutoReset + +namespace ftxui { + +/// @brief Define how the Screen's dimensions should look like. +/// @ingroup screen +namespace Dimension { +Dimensions Fixed(int); +Dimensions Full(); +} // namespace Dimension + +/// @brief A rectangular grid of Pixel. +/// @ingroup screen +class Screen : public Image { + public: + // Constructors: + Screen(int dimx, int dimy); + static Screen Create(Dimensions dimension); + static Screen Create(Dimensions width, Dimensions height); + + std::string ToString() const; + + // Print the Screen on to the terminal. + void Print() const; + + // Fill the screen with space and reset any screen state, like hyperlinks, and + // cursor + void Clear(); + + // Move the terminal cursor n-lines up with n = dimy(). + std::string ResetPosition(bool clear = false) const; + + void ApplyShader(); + + struct Cursor { + int x = 0; + int y = 0; + + enum Shape { + Hidden = 0, + BlockBlinking = 1, + Block = 2, + UnderlineBlinking = 3, + Underline = 4, + BarBlinking = 5, + Bar = 6, + }; + Shape shape; + }; + + Cursor cursor() const { return cursor_; } + void SetCursor(Cursor cursor) { cursor_ = cursor; } + + // Store an hyperlink in the screen. Return the id of the hyperlink. The id is + // used to identify the hyperlink when the user click on it. + uint8_t RegisterHyperlink(const std::string& link); + const std::string& Hyperlink(uint8_t id) const; + + using SelectionStyle = std::function<void(Pixel&)>; + const SelectionStyle& GetSelectionStyle() const; + void SetSelectionStyle(SelectionStyle decorator); + + protected: + Cursor cursor_; + std::vector<std::string> hyperlinks_ = {""}; + + // The current selection style. This is overridden by various dom elements. + SelectionStyle selection_style_ = [](Pixel& pixel) { + pixel.inverted ^= true; + }; +}; + +} // namespace ftxui + +#endif // FTXUI_SCREEN_SCREEN_HPP diff --git a/contrib/libs/ftxui/include/ftxui/screen/string.hpp b/contrib/libs/ftxui/include/ftxui/screen/string.hpp new file mode 100644 index 00000000000..ca5397bc135 --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/screen/string.hpp @@ -0,0 +1,31 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_SCREEN_STRING_HPP +#define FTXUI_SCREEN_STRING_HPP + +#include <string> // for string, wstring, to_string +#include <vector> // for vector + +namespace ftxui { +std::string to_string(const std::wstring& s); +std::wstring to_wstring(const std::string& s); + +template <typename T> +std::wstring to_wstring(T s) { + return to_wstring(std::to_string(s)); +} + +int string_width(const std::string&); + +// Split the string into a its glyphs. An empty one is inserted ater fullwidth +// ones. +std::vector<std::string> Utf8ToGlyphs(const std::string& input); + +// Map every cells drawn by |input| to their corresponding Glyphs. Half-size +// Glyphs takes one cell, full-size Glyphs take two cells. +std::vector<int> CellToGlyphIndex(const std::string& input); + +} // namespace ftxui + +#endif /* end of include guard: FTXUI_SCREEN_STRING_HPP */ diff --git a/contrib/libs/ftxui/include/ftxui/screen/terminal.hpp b/contrib/libs/ftxui/include/ftxui/screen/terminal.hpp new file mode 100644 index 00000000000..051aed08b1e --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/screen/terminal.hpp @@ -0,0 +1,30 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_SCREEN_TERMINAL_HPP +#define FTXUI_SCREEN_TERMINAL_HPP + +namespace ftxui { +struct Dimensions { + int dimx; + int dimy; +}; + +namespace Terminal { +Dimensions Size(); +void SetFallbackSize(const Dimensions& fallbackSize); + +enum Color { + Palette1, + Palette16, + Palette256, + TrueColor, +}; +Color ColorSupport(); +void SetColorSupport(Color color); + +} // namespace Terminal + +} // namespace ftxui + +#endif // FTXUI_SCREEN_TERMINAL_HPP diff --git a/contrib/libs/ftxui/include/ftxui/util/autoreset.hpp b/contrib/libs/ftxui/include/ftxui/util/autoreset.hpp new file mode 100644 index 00000000000..ab33293d1c2 --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/util/autoreset.hpp @@ -0,0 +1,32 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_UTIL_AUTORESET_HPP +#define FTXUI_UTIL_AUTORESET_HPP + +#include <utility> + +namespace ftxui { + +/// Assign a value to a variable, reset its old value when going out of scope. +template <typename T> +class AutoReset { + public: + AutoReset(T* variable, T new_value) + : variable_(variable), previous_value_(std::move(*variable)) { + *variable_ = std::move(new_value); + } + AutoReset(const AutoReset&) = delete; + AutoReset(AutoReset&&) = delete; + AutoReset& operator=(const AutoReset&) = delete; + AutoReset& operator=(AutoReset&&) = delete; + ~AutoReset() { *variable_ = std::move(previous_value_); } + + private: + T* variable_; + T previous_value_; +}; + +} // namespace ftxui + +#endif /* end of include guard: FTXUI_UTIL_AUTORESET_HPP */ diff --git a/contrib/libs/ftxui/include/ftxui/util/ref.hpp b/contrib/libs/ftxui/include/ftxui/util/ref.hpp new file mode 100644 index 00000000000..42a5938149b --- /dev/null +++ b/contrib/libs/ftxui/include/ftxui/util/ref.hpp @@ -0,0 +1,216 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_UTIL_REF_HPP +#define FTXUI_UTIL_REF_HPP + +#include <ftxui/screen/string.hpp> +#include <memory> +#include <string> +#include <variant> +#include <vector> + +namespace ftxui { + +/// @brief An adapter. Own or reference an immutable object. +template <typename T> +class ConstRef { + public: + ConstRef() = default; + ConstRef(T t) : variant_(std::move(t)) {} // NOLINT + ConstRef(const T* t) : variant_(t) {} // NOLINT + ConstRef& operator=(ConstRef&&) noexcept = default; + ConstRef(const ConstRef<T>&) = default; + ConstRef(ConstRef<T>&&) noexcept = default; + ~ConstRef() = default; + + // Make a "reseatable" reference + ConstRef<T>& operator=(const ConstRef<T>&) = default; + + // Accessors: + const T& operator()() const { return *Address(); } + const T& operator*() const { return *Address(); } + const T* operator->() const { return Address(); } + + private: + std::variant<T, const T*> variant_ = T{}; + + const T* Address() const { + return std::holds_alternative<T>(variant_) ? &std::get<T>(variant_) + : std::get<const T*>(variant_); + } +}; + +/// @brief An adapter. Own or reference an mutable object. +template <typename T> +class Ref { + public: + Ref() = default; + Ref(T t) : variant_(std::move(t)) {} // NOLINT + Ref(T* t) : variant_(t) {} // NOLINT + ~Ref() = default; + Ref& operator=(Ref&&) noexcept = default; + Ref(const Ref<T>&) = default; + Ref(Ref<T>&&) noexcept = default; + + // Make a "reseatable" reference. + Ref<T>& operator=(const Ref<T>&) = default; + + // Accessors: + T& operator()() { return *Address(); } + T& operator*() { return *Address(); } + T* operator->() { return Address(); } + const T& operator()() const { return *Address(); } + const T& operator*() const { return *Address(); } + const T* operator->() const { return Address(); } + + private: + std::variant<T, T*> variant_ = T{}; + + const T* Address() const { + return std::holds_alternative<T>(variant_) ? &std::get<T>(variant_) + : std::get<T*>(variant_); + } + T* Address() { + return std::holds_alternative<T>(variant_) ? &std::get<T>(variant_) + : std::get<T*>(variant_); + } +}; + +/// @brief An adapter. Own or reference a constant string. For convenience, this +/// class convert multiple mutable string toward a shared representation. +class StringRef : public Ref<std::string> { + public: + using Ref<std::string>::Ref; + + StringRef(const wchar_t* ref) // NOLINT + : StringRef(to_string(std::wstring(ref))) {} + StringRef(const char* ref) // NOLINT + : StringRef(std::string(ref)) {} +}; + +/// @brief An adapter. Own or reference a constant string. For convenience, this +/// class convert multiple immutable string toward a shared representation. +class ConstStringRef : public ConstRef<std::string> { + public: + using ConstRef<std::string>::ConstRef; + + ConstStringRef(const std::wstring* ref) // NOLINT + : ConstStringRef(to_string(*ref)) {} + ConstStringRef(const std::wstring ref) // NOLINT + : ConstStringRef(to_string(ref)) {} + ConstStringRef(const wchar_t* ref) // NOLINT + : ConstStringRef(to_string(std::wstring(ref))) {} + ConstStringRef(const char* ref) // NOLINT + : ConstStringRef(std::string(ref)) {} +}; + +/// @brief An adapter. Reference a list of strings. +/// +/// Supported input: +/// - `std::vector<std::string>` +/// - `std::vector<std::string>*` +/// - `std::vector<std::wstring>*` +/// - `Adapter*` +/// - `std::unique_ptr<Adapter>` +class ConstStringListRef { + public: + // Bring your own adapter: + class Adapter { + public: + Adapter() = default; + Adapter(const Adapter&) = default; + Adapter& operator=(const Adapter&) = default; + Adapter(Adapter&&) = default; + Adapter& operator=(Adapter&&) = default; + virtual ~Adapter() = default; + virtual size_t size() const = 0; + virtual std::string operator[](size_t i) const = 0; + }; + using Variant = std::variant<const std::vector<std::string>, // + const std::vector<std::string>*, // + const std::vector<std::wstring>*, // + Adapter*, // + std::unique_ptr<Adapter> // + >; + + ConstStringListRef() = default; + ~ConstStringListRef() = default; + ConstStringListRef& operator=(const ConstStringListRef&) = default; + ConstStringListRef& operator=(ConstStringListRef&&) = default; + ConstStringListRef(ConstStringListRef&&) = default; + ConstStringListRef(const ConstStringListRef&) = default; + + ConstStringListRef(std::vector<std::string> value) // NOLINT + { + variant_ = std::make_shared<Variant>(value); + } + ConstStringListRef(const std::vector<std::string>* value) // NOLINT + { + variant_ = std::make_shared<Variant>(value); + } + ConstStringListRef(const std::vector<std::wstring>* value) // NOLINT + { + variant_ = std::make_shared<Variant>(value); + } + ConstStringListRef(Adapter* adapter) // NOLINT + { + variant_ = std::make_shared<Variant>(adapter); + } + template <typename AdapterType> + ConstStringListRef(std::unique_ptr<AdapterType> adapter) // NOLINT + { + variant_ = std::make_shared<Variant>( + static_cast<std::unique_ptr<Adapter>>(std::move(adapter))); + } + + size_t size() const { + return variant_ ? std::visit(SizeVisitor(), *variant_) : 0; + } + + std::string operator[](size_t i) const { + return variant_ ? std::visit(IndexedGetter(i), *variant_) : ""; + } + + private: + struct SizeVisitor { + size_t operator()(const std::vector<std::string>& v) const { + return v.size(); + } + size_t operator()(const std::vector<std::string>* v) const { + return v->size(); + } + size_t operator()(const std::vector<std::wstring>* v) const { + return v->size(); + } + size_t operator()(const Adapter* v) const { return v->size(); } + size_t operator()(const std::unique_ptr<Adapter>& v) const { + return v->size(); + } + }; + + struct IndexedGetter { + IndexedGetter(size_t index) // NOLINT + : index_(index) {} + size_t index_; + std::string operator()(const std::vector<std::string>& v) const { + return v[index_]; + } + std::string operator()(const std::vector<std::string>* v) const { + return (*v)[index_]; + } + std::string operator()(const std::vector<std::wstring>* v) const { + return to_string((*v)[index_]); + } + std::string operator()(const Adapter* v) const { return (*v)[index_]; } + std::string operator()(const std::unique_ptr<Adapter>& v) const { + return (*v)[index_]; + } + }; + + std::shared_ptr<Variant> variant_; +}; + +} // namespace ftxui + +#endif /* end of include guard: FTXUI_UTIL_REF_HPP */ diff --git a/contrib/libs/ftxui/src/ftxui/component/animation.cpp b/contrib/libs/ftxui/src/ftxui/component/animation.cpp new file mode 100644 index 00000000000..6b6a1a4d2b7 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/animation.cpp @@ -0,0 +1,286 @@ +#include <cmath> // for sin, pow, sqrt, cos +#include <utility> // for move + +#include "ftxui/component/animation.hpp" + +// NOLINTBEGIN(*-magic-numbers) +namespace ftxui::animation { + +namespace easing { + +namespace { +constexpr float kPi = 3.14159265358979323846f; +constexpr float kPi2 = kPi / 2.f; +} // namespace + +// Easing function have been taken out of: +// https://github.com/warrenm/AHEasing/blob/master/AHEasing/easing.c +// +// Corresponding license: +// Copyright (c) 2011, Auerhaus Development, LLC +// +// This program is free software. It comes without any warranty, to +// the extent permitted by applicable law. You can redistribute it +// and/or modify it under the terms of the Do What The Fuck You Want +// To Public License, Version 2, as published by Sam Hocevar. See +// http://sam.zoy.org/wtfpl/COPYING for more details. + +/// @brief Modeled after the line y = x +float Linear(float p) { + return p; +} + +/// @brief Modeled after the parabola y = x^2 +float QuadraticIn(float p) { + return p * p; +} + +// @brief Modeled after the parabola y = -x^2 + 2x +float QuadraticOut(float p) { + return -(p * (p - 2.f)); +} + +// @brief Modeled after the piecewise quadratic +// y = (1/2)((2x)^2) ; [0, 0.5) +// y = -(1/2)((2x-1)*(2x-3) - 1) ; [0.5, 1] +float QuadraticInOut(float p) { + return p < 0.5f ? 2.f * p * p : (-2.f * p * p) + (4.f * p) - 1.f; +} + +// @brief Modeled after the cubic y = x^3 +float CubicIn(float p) { + return p * p * p; +} + +// @brief Modeled after the cubic y = (x - 1)^3 + 1 +float CubicOut(float p) { + const float f = (p - 1.f); + return f * f * f + 1.f; +} + +// @brief Modeled after the piecewise cubic +// y = (1/2)((2x)^3) ; [0, 0.5) +// y = (1/2)((2x-2)^3 + 2) ; [0.5, 1] +float CubicInOut(float p) { + if (p < 0.5f) { + return 4.f * p * p * p; + } + const float f = ((2.f * p) - 2.f); + return 0.5f * f * f * f + 1.f; +} + +// @brief Modeled after the quartic x^4 +float QuarticIn(float p) { + return p * p * p * p; +} + +// @brief Modeled after the quartic y = 1 - (x - 1)^4 +float QuarticOut(float p) { + const float f = (p - 1.f); + return f * f * f * (1.f - p) + 1.f; +} + +// @brief Modeled after the piecewise quartic +// y = (1/2)((2x)^4) ; [0, 0.5) +// y = -(1/2)((2x-2)^4 - 2) ; [0.5, 1] +float QuarticInOut(float p) { + if (p < 0.5f) { + return 8.f * p * p * p * p; + } + const float f = (p - 1.f); + return -8.f * f * f * f * f + 1.f; +} + +// @brief Modeled after the quintic y = x^5 +float QuinticIn(float p) { + return p * p * p * p * p; +} + +// @brief Modeled after the quintic y = (x - 1)^5 + 1 +float QuinticOut(float p) { + const float f = (p - 1.f); + return f * f * f * f * f + 1.f; +} + +// @brief Modeled after the piecewise quintic +// y = (1/2)((2x)^5) ; [0, 0.5) +// y = (1/2)((2x-2)^5 + 2) ; [0.5, 1] +float QuinticInOut(float p) { + if (p < 0.5f) { + return 16.f * p * p * p * p * p; + } + const float f = ((2.f * p) - 2.f); + return 0.5f * f * f * f * f * f + 1.f; +} + +// @brief Modeled after quarter-cycle of sine wave +float SineIn(float p) { + return std::sin((p - 1.f) * kPi2) + 1.f; +} + +// @brief Modeled after quarter-cycle of sine wave (different phase) +float SineOut(float p) { + return std::sin(p * kPi2); +} + +// @brief Modeled after half sine wave +float SineInOut(float p) { + return 0.5f * (1.f - std::cos(p * kPi)); +} + +// @brief Modeled after shifted quadrant IV of unit circle +float CircularIn(float p) { + return 1.f - std::sqrt(1.f - (p * p)); +} + +// @brief Modeled after shifted quadrant II of unit circle +float CircularOut(float p) { + return std::sqrt((2.f - p) * p); +} + +// @brief Modeled after the piecewise circular function +// y = (1/2)(1 - sqrt(1 - 4x^2)) ; [0, 0.5) +// y = (1/2)(sqrt(-(2x - 3)*(2x - 1)) + 1) ; [0.5, 1] +float CircularInOut(float p) { + if (p < 0.5f) { + return 0.5f * (1.f - std::sqrt(1.f - 4.f * (p * p))); + } + return 0.5f * (std::sqrt(-((2.f * p) - 3.f) * ((2.f * p) - 1.f)) + 1.f); +} + +// @brief Modeled after the exponential function y = 2^(10(x - 1)) +float ExponentialIn(float p) { + return (p == 0.f) ? p : std::pow(2.f, 10.f * (p - 1.f)); +} + +// @brief Modeled after the exponential function y = -2^(-10x) + 1 +float ExponentialOut(float p) { + return (p == 1.f) ? p : 1.f - std::pow(2.f, -10.f * p); +} + +// @brief Modeled after the piecewise exponential +// y = (1/2)2^(10(2x - 1)) ; [0,0.5) +// y = -(1/2)*2^(-10(2x - 1))) + 1 ; [0.5,1] +float ExponentialInOut(float p) { + if (p == 0.f || p == 1.f) { + return p; + } + + if (p < 0.5f) { + return 0.5f * std::pow(2.f, (20.f * p) - 10.f); + } + return -0.5f * std::pow(2.f, (-20.f * p) + 10.f) + 1.f; +} + +// @brief Modeled after the damped sine wave y = sin(13pi/2*x)*pow(2, 10 * (x - +// 1)) +float ElasticIn(float p) { + return std::sin(13.f * kPi2 * p) * std::pow(2.f, 10.f * (p - 1.f)); +} + +// @brief Modeled after the damped sine wave y = sin(-13pi/2*(x + 1))*pow(2, +// -10x) + +// 1 +float ElasticOut(float p) { + return std::sin(-13.f * kPi2 * (p + 1.f)) * std::pow(2.f, -10.f * p) + 1.f; +} + +// @brief Modeled after the piecewise exponentially-damped sine wave: +// y = (1/2)*sin(13pi/2*(2*x))*pow(2, 10 * ((2*x) - 1)) ; [0,0.5) +// y = (1/2)*(sin(-13pi/2*((2x-1)+1))*pow(2,-10(2*x-1)) + 2) ; [0.5, 1] +float ElasticInOut(float p) { + if (p < 0.5f) { + return 0.5f * std::sin(13.f * kPi2 * (2.f * p)) * + std::pow(2.f, 10.f * ((2.f * p) - 1.f)); + } + return 0.5f * (std::sin(-13.f * kPi2 * ((2.f * p - 1.f) + 1.f)) * + std::pow(2.f, -10.f * (2.f * p - 1.f)) + + 2.f); +} + +// @brief Modeled after the overshooting cubic y = x^3-x*sin(x*pi) +float BackIn(float p) { + return p * p * p - p * std::sin(p * kPi); +} + +// @brief Modeled after overshooting cubic y = 1-((1-x)^3-(1-x)*sin((1-x)*pi)) +float BackOut(float p) { + const float f = (1.f - p); + return 1.f - (f * f * f - f * std::sin(f * kPi)); +} + +// @brief Modeled after the piecewise overshooting cubic function: +// y = (1/2)*((2x)^3-(2x)*sin(2*x*pi)) ; [0, 0.5) +// y = (1/2)*(1-((1-x)^3-(1-x)*sin((1-x)*pi))+1) ; [0.5, 1] +float BackInOut(float p) { + if (p < 0.5f) { + const float f = 2.f * p; + return 0.5f * (f * f * f - f * std::sin(f * kPi)); + } + const float f = (1.f - (2.f * p - 1.f)); + return 0.5f * (1.f - (f * f * f - f * std::sin(f * kPi))) + 0.5f; +} + +float BounceIn(float p) { + return 1.f - BounceOut(1.f - p); +} + +float BounceOut(float p) { + if (p < 4.f / 11.f) { + return (121.f * p * p) / 16.f; + } + + if (p < 8.f / 11.f) { + return (363.f / 40.f * p * p) - (99.f / 10.f * p) + 17.f / 5.f; + } + + if (p < 9.f / 10.f) { + return (4356.f / 361.f * p * p) - (35442.f / 1805.f * p) + 16061.f / 1805.f; + } + + return (54.f / 5.f * p * p) - (513 / 25.f * p) + 268 / 25.f; +} + +float BounceInOut(float p) { + if (p < 0.5f) { + return 0.5f * BounceIn(p * 2.f); + } + return 0.5f * BounceOut(p * 2.f - 1.f) + 0.5f; +} + +} // namespace easing + +Animator::Animator(float* from, + float to, + Duration duration, + easing::Function easing_function, + Duration delay) + : value_(from), + from_(*from), + to_(to), + duration_(duration), + easing_function_(std::move(easing_function)), + current_(-delay) { + RequestAnimationFrame(); +} + +void Animator::OnAnimation(Params& params) { + current_ += params.duration(); + + if (current_ >= duration_) { + *value_ = to_; + return; + } + + if (current_ <= Duration()) { + *value_ = from_; + } else { + *value_ = from_ + (to_ - from_) * easing_function_(current_ / duration_); + } + + RequestAnimationFrame(); +} + +} // namespace ftxui::animation + +// NOLINTEND(*-magic-numbers) diff --git a/contrib/libs/ftxui/src/ftxui/component/button.cpp b/contrib/libs/ftxui/src/ftxui/component/button.cpp new file mode 100644 index 00000000000..79ad78acb5c --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/button.cpp @@ -0,0 +1,212 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +#include <functional> // for function +#include <utility> // for move + +#include "ftxui/component/animation.hpp" // for Animator, Params (ptr only) +#include "ftxui/component/component.hpp" // for Make, Button +#include "ftxui/component/component_base.hpp" // for ComponentBase +#include "ftxui/component/component_options.hpp" // for ButtonOption, AnimatedColorOption, AnimatedColorsOption, EntryState +#include "ftxui/component/event.hpp" // for Event, Event::Return +#include "ftxui/component/mouse.hpp" // for Mouse, Mouse::Left, Mouse::Pressed +#include "ftxui/component/screen_interactive.hpp" // for Component +#include "ftxui/dom/elements.hpp" // for operator|, Decorator, Element, operator|=, bgcolor, color, reflect, text, bold, border, inverted, nothing +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/color.hpp" // for Color +#include "ftxui/util/ref.hpp" // for Ref, ConstStringRef + +namespace ftxui { + +namespace { + +Element DefaultTransform(EntryState params) { // NOLINT + auto element = text(params.label) | border; + if (params.active) { + element |= bold; + } + if (params.focused) { + element |= inverted; + } + return element; +} + +class ButtonBase : public ComponentBase, public ButtonOption { + public: + explicit ButtonBase(ButtonOption option) : ButtonOption(std::move(option)) {} + + // Component implementation: + Element OnRender() override { + const bool active = Active(); + const bool focused = Focused(); + const bool focused_or_hover = focused || mouse_hover_; + + float target = focused_or_hover ? 1.f : 0.f; // NOLINT + if (target != animator_background_.to()) { + SetAnimationTarget(target); + } + + const EntryState state{ + *label, false, active, focused_or_hover, Index(), + }; + + auto element = (transform ? transform : DefaultTransform) // + (state); + element |= AnimatedColorStyle(); + element |= focus; + element |= reflect(box_); + return element; + } + + Decorator AnimatedColorStyle() { + Decorator style = nothing; + if (animated_colors.background.enabled) { + style = style | + bgcolor(Color::Interpolate(animation_foreground_, // + animated_colors.background.inactive, + animated_colors.background.active)); + } + if (animated_colors.foreground.enabled) { + style = + style | color(Color::Interpolate(animation_foreground_, // + animated_colors.foreground.inactive, + animated_colors.foreground.active)); + } + return style; + } + + void SetAnimationTarget(float target) { + if (animated_colors.foreground.enabled) { + animator_foreground_ = animation::Animator( + &animation_foreground_, target, animated_colors.foreground.duration, + animated_colors.foreground.function); + } + if (animated_colors.background.enabled) { + animator_background_ = animation::Animator( + &animation_background_, target, animated_colors.background.duration, + animated_colors.background.function); + } + } + + void OnAnimation(animation::Params& p) override { + animator_background_.OnAnimation(p); + animator_foreground_.OnAnimation(p); + } + + void OnClick() { + animation_background_ = 0.5F; // NOLINT + animation_foreground_ = 0.5F; // NOLINT + SetAnimationTarget(1.F); // NOLINT + + // TODO(arthursonzogni): Consider posting the task to the main loop, instead + // of invoking it immediately. + on_click(); // May delete this. + } + + bool OnEvent(Event event) override { + if (event.is_mouse()) { + return OnMouseEvent(event); + } + + if (event == Event::Return) { + OnClick(); // May delete this. + return true; + } + return false; + } + + bool OnMouseEvent(Event event) { + mouse_hover_ = + box_.Contain(event.mouse().x, event.mouse().y) && CaptureMouse(event); + + if (!mouse_hover_) { + return false; + } + + if (event.mouse().button == Mouse::Left && + event.mouse().motion == Mouse::Pressed) { + TakeFocus(); + OnClick(); // May delete this. + return true; + } + + return false; + } + + bool Focusable() const final { return true; } + + private: + bool mouse_hover_ = false; + Box box_; + ButtonOption option_; + float animation_background_ = 0; + float animation_foreground_ = 0; + animation::Animator animator_background_ = + animation::Animator(&animation_background_); + animation::Animator animator_foreground_ = + animation::Animator(&animation_foreground_); +}; + +} // namespace + +/// @brief Draw a button. Execute a function when clicked. +/// @param option Additional optional parameters. +/// @ingroup component +/// @see ButtonBase +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::FitComponent(); +/// Component button = Button({ +/// .label = "Click to quit", +/// .on_click = screen.ExitLoopClosure(), +/// }); +/// screen.Loop(button) +/// ``` +/// +/// ### Output +/// +/// ```bash +/// ┌─────────────┐ +/// │Click to quit│ +/// └─────────────┘ +/// ``` +Component Button(ButtonOption option) { + return Make<ButtonBase>(std::move(option)); +} + +/// @brief Draw a button. Execute a function when clicked. +/// @param label The label of the button. +/// @param on_click The action to execute when clicked. +/// @param option Additional optional parameters. +/// @ingroup component +/// @see ButtonBase +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::FitComponent(); +/// std::string label = "Click to quit"; +/// Component button = Button(&label, screen.ExitLoopClosure()); +/// screen.Loop(button) +/// ``` +/// +/// ### Output +/// +/// ```bash +/// ┌─────────────┐ +/// │Click to quit│ +/// └─────────────┘ +/// ``` +// NOLINTNEXTLINE +Component Button(ConstStringRef label, + std::function<void()> on_click, + ButtonOption option) { + option.label = std::move(label); + option.on_click = std::move(on_click); + return Make<ButtonBase>(std::move(option)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/catch_event.cpp b/contrib/libs/ftxui/src/ftxui/component/catch_event.cpp new file mode 100644 index 00000000000..a20b6cfdbed --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/catch_event.cpp @@ -0,0 +1,88 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <functional> // for function +#include <utility> // for move + +#include "ftxui/component/component.hpp" // for Make, CatchEvent, ComponentDecorator +#include "ftxui/component/component_base.hpp" // for Component, ComponentBase +#include "ftxui/component/event.hpp" // for Event + +namespace ftxui { + +class CatchEventBase : public ComponentBase { + public: + // Constructor. + explicit CatchEventBase(std::function<bool(Event)> on_event) + : on_event_(std::move(on_event)) {} + + // Component implementation. + bool OnEvent(Event event) override { + if (on_event_(event)) { + return true; + } else { + return ComponentBase::OnEvent(event); + } + } + + protected: + std::function<bool(Event)> on_event_; +}; + +/// @brief Return a component, using |on_event| to catch events. This function +/// must returns true when the event has been handled, false otherwise. +/// @param child The wrapped component. +/// @param on_event The function drawing the interface. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// auto renderer = Renderer([] { +/// return text("My interface"); +/// }); +/// auto component = CatchEvent(renderer, [&](Event event) { +/// if (event == Event::Character('q')) { +/// screen.ExitLoopClosure()(); +/// return true; +/// } +/// return false; +/// }); +/// screen.Loop(component); +/// ``` +Component CatchEvent(Component child, + std::function<bool(Event event)> on_event) { + auto out = Make<CatchEventBase>(std::move(on_event)); + out->Add(std::move(child)); + return out; +} + +/// @brief Decorate a component, using |on_event| to catch events. This function +/// must returns true when the event has been handled, false otherwise. +/// @param on_event The function drawing the interface. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// auto renderer = Renderer([] { return text("Hello world"); }); +/// renderer |= CatchEvent([&](Event event) { +/// if (event == Event::Character('q')) { +/// screen.ExitLoopClosure()(); +/// return true; +/// } +/// return false; +/// }); +/// screen.Loop(renderer); +/// ``` +ComponentDecorator CatchEvent(std::function<bool(Event)> on_event) { + return [on_event = std::move(on_event)](Component child) { + return CatchEvent(std::move(child), [on_event = on_event](Event event) { + return on_event(std::move(event)); + }); + }; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/checkbox.cpp b/contrib/libs/ftxui/src/ftxui/component/checkbox.cpp new file mode 100644 index 00000000000..41a4e9579bf --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/checkbox.cpp @@ -0,0 +1,141 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <functional> // for function +#include <utility> // for move + +#include "ftxui/component/component.hpp" // for Make, Checkbox +#include "ftxui/component/component_base.hpp" // for Component, ComponentBase +#include "ftxui/component/component_options.hpp" // for CheckboxOption, EntryState +#include "ftxui/component/event.hpp" // for Event, Event::Return +#include "ftxui/component/mouse.hpp" // for Mouse, Mouse::Left, Mouse::Pressed +#include "ftxui/dom/elements.hpp" // for operator|, Element, reflect, focus, nothing, select +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/util/ref.hpp" // for Ref, ConstStringRef + +namespace ftxui { + +namespace { +class CheckboxBase : public ComponentBase, public CheckboxOption { + public: + explicit CheckboxBase(CheckboxOption option) + : CheckboxOption(std::move(option)) {} + + private: + // Component implementation. + Element OnRender() override { + const bool is_focused = Focused(); + const bool is_active = Active(); + auto entry_state = EntryState{ + *label, *checked, is_active, is_focused || hovered_, -1, + }; + auto element = (transform ? transform : CheckboxOption::Simple().transform)( + entry_state); + element |= focus; + element |= reflect(box_); + return element; + } + + bool OnEvent(Event event) override { + if (!CaptureMouse(event)) { + return false; + } + + if (event.is_mouse()) { + return OnMouseEvent(event); + } + + hovered_ = false; + if (event == Event::Character(' ') || event == Event::Return) { + *checked = !*checked; + on_change(); + TakeFocus(); + return true; + } + return false; + } + + bool OnMouseEvent(Event event) { + hovered_ = box_.Contain(event.mouse().x, event.mouse().y); + + if (!CaptureMouse(event)) { + return false; + } + + if (!hovered_) { + return false; + } + + if (event.mouse().button == Mouse::Left && + event.mouse().motion == Mouse::Pressed) { + *checked = !*checked; + on_change(); + return true; + } + + return false; + } + + bool Focusable() const final { return true; } + + bool hovered_ = false; + Box box_; +}; +} // namespace + +/// @brief Draw checkable element. +/// @param option Additional optional parameters. +/// @ingroup component +/// @see CheckboxBase +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::FitComponent(); +/// CheckboxOption option; +/// option.label = "Make a sandwidth"; +/// option.checked = false; +/// Component checkbox = Checkbox(option); +/// screen.Loop(checkbox) +/// ``` +/// +/// ### Output +/// +/// ```bash +/// ☐ Make a sandwitch +/// ``` +// NOLINTNEXTLINE +Component Checkbox(CheckboxOption option) { + return Make<CheckboxBase>(std::move(option)); +} + +/// @brief Draw checkable element. +/// @param label The label of the checkbox. +/// @param checked Whether the checkbox is checked or not. +/// @param option Additional optional parameters. +/// @ingroup component +/// @see CheckboxBase +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::FitComponent(); +/// std::string label = "Make a sandwidth"; +/// bool checked = false; +/// Component checkbox = Checkbox(&label, &checked); +/// screen.Loop(checkbox) +/// ``` +/// +/// ### Output +/// +/// ```bash +/// ☐ Make a sandwitch +/// ``` +// NOLINTNEXTLINE +Component Checkbox(ConstStringRef label, bool* checked, CheckboxOption option) { + option.label = std::move(label); + option.checked = checked; + return Make<CheckboxBase>(std::move(option)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/collapsible.cpp b/contrib/libs/ftxui/src/ftxui/component/collapsible.cpp new file mode 100644 index 00000000000..a79c855066d --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/collapsible.cpp @@ -0,0 +1,60 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <functional> // for function +#include <utility> // for move + +#include "ftxui/component/component.hpp" // for Checkbox, Maybe, Make, Vertical, Collapsible +#include "ftxui/component/component_base.hpp" // for Component, ComponentBase +#include "ftxui/component/component_options.hpp" // for CheckboxOption, EntryState +#include "ftxui/dom/elements.hpp" // for operator|=, text, hbox, Element, bold, inverted +#include "ftxui/util/ref.hpp" // for Ref, ConstStringRef + +namespace ftxui { + +/// @brief A collapsible component. It display a checkbox with an arrow. Once +/// activated, the children is displayed. +/// @param label The label of the checkbox. +/// @param child The children to display. +/// @param show Hold the state about whether the children is displayed or not. +/// +/// ### Example +/// ```cpp +/// auto component = Collapsible("Show details", details); +/// ``` +/// +/// ### Output +/// ``` +/// +/// ▼ Show details +/// <details component> +/// ``` +// NOLINTNEXTLINE +Component Collapsible(ConstStringRef label, Component child, Ref<bool> show) { + class Impl : public ComponentBase { + public: + Impl(ConstStringRef label, Component child, Ref<bool> show) : show_(show) { + CheckboxOption opt; + opt.transform = [](EntryState s) { // NOLINT + auto prefix = text(s.state ? "▼ " : "▶ "); // NOLINT + auto t = text(s.label); + if (s.active) { + t |= bold; + } + if (s.focused) { + t |= inverted; + } + return hbox({prefix, t}); + }; + Add(Container::Vertical({ + Checkbox(std::move(label), show_.operator->(), opt), + Maybe(std::move(child), show_.operator->()), + })); + } + Ref<bool> show_; + }; + + return Make<Impl>(std::move(label), std::move(child), show); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/component.cpp b/contrib/libs/ftxui/src/ftxui/component/component.cpp new file mode 100644 index 00000000000..825865ae6a8 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/component.cpp @@ -0,0 +1,257 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for find_if +#include <cassert> // for assert +#include <cstddef> // for size_t +#include <iterator> // for begin, end +#include <memory> // for unique_ptr, make_unique +#include <utility> // for move +#include <vector> // for vector, __alloc_traits<>::value_type + +#include "ftxui/component/captured_mouse.hpp" // for CapturedMouse, CapturedMouseInterface +#include "ftxui/component/component.hpp" +#include "ftxui/component/component_base.hpp" // for ComponentBase, Components +#include "ftxui/component/event.hpp" // for Event +#include "ftxui/component/screen_interactive.hpp" // for Component, ScreenInteractive +#include "ftxui/dom/elements.hpp" // for text, Element +#include "ftxui/dom/node.hpp" // for Node, Elements +#include "ftxui/screen/box.hpp" // for Box + +namespace ftxui::animation { +class Params; +} // namespace ftxui::animation + +namespace ftxui { + +namespace { +class CaptureMouseImpl : public CapturedMouseInterface {}; +} // namespace + +ComponentBase::~ComponentBase() { + DetachAllChildren(); +} + +/// @brief Return the parent ComponentBase, or nul if any. +/// @see Detach +/// @see Parent +/// @ingroup component +ComponentBase* ComponentBase::Parent() const { + return parent_; +} + +/// @brief Access the child at index `i`. +/// @ingroup component +Component& ComponentBase::ChildAt(size_t i) { + assert(i < ChildCount()); // NOLINT + return children_[i]; +} + +/// @brief Returns the number of children. +/// @ingroup component +size_t ComponentBase::ChildCount() const { + return children_.size(); +} + +/// @brief Return index of the component in its parent. -1 if no parent. +/// @ingroup component +int ComponentBase::Index() const { + if (parent_ == nullptr) { + return -1; + } + int index = 0; + for (const Component& child : parent_->children_) { + if (child.get() == this) { + return index; + } + index++; + } + return -1; // Not reached. +} + +/// @brief Add a child. +/// @@param child The child to be attached. +/// @ingroup component +void ComponentBase::Add(Component child) { + child->Detach(); + child->parent_ = this; + children_.push_back(std::move(child)); +} + +/// @brief Detach this child from its parent. +/// @see Detach +/// @see Parent +/// @ingroup component +void ComponentBase::Detach() { + if (parent_ == nullptr) { + return; + } + auto it = std::find_if(std::begin(parent_->children_), // + std::end(parent_->children_), // + [this](const Component& that) { // + return this == that.get(); + }); + ComponentBase* parent = parent_; + parent_ = nullptr; + parent->children_.erase(it); // Might delete |this|. +} + +/// @brief Remove all children. +/// @ingroup component +void ComponentBase::DetachAllChildren() { + while (!children_.empty()) { + children_[0]->Detach(); + } +} + +/// @brief Draw the component. +/// Build a ftxui::Element to be drawn on the ftxui::Screen representing this +/// ftxui::ComponentBase. Please override OnRender() to modify the rendering. +/// @ingroup component +Element ComponentBase::Render() { + // Some users might call `ComponentBase::Render()` from + // `T::OnRender()`. To avoid infinite recursion, we use a flag. + if (in_render) { + return ComponentBase::OnRender(); + } + + in_render = true; + Element element = OnRender(); + in_render = false; + + class Wrapper : public Node { + public: + bool active_ = false; + + Wrapper(Element child, bool active) + : Node({std::move(child)}), active_(active) {} + + void SetBox(Box box) override { + Node::SetBox(box); + children_[0]->SetBox(box); + } + + void ComputeRequirement() override { + Node::ComputeRequirement(); + requirement_.focused.component_active = active_; + } + }; + + return std::make_shared<Wrapper>(std::move(element), Active()); +} + +/// @brief Draw the component. +/// Build a ftxui::Element to be drawn on the ftxi::Screen representing this +/// ftxui::ComponentBase. This function is means to be overridden. +/// @ingroup component +Element ComponentBase::OnRender() { + if (children_.size() == 1) { + return children_.front()->Render(); + } + + return text("Not implemented component"); +} + +/// @brief Called in response to an event. +/// @param event The event. +/// @return True when the event has been handled. +/// The default implementation called OnEvent on every child until one return +/// true. If none returns true, return false. +/// @ingroup component +bool ComponentBase::OnEvent(Event event) { // NOLINT + for (Component& child : children_) { // NOLINT + if (child->OnEvent(event)) { + return true; + } + } + return false; +} + +/// @brief Called in response to an animation event. +/// @param params the parameters of the animation +/// The default implementation dispatch the event to every child. +/// @ingroup component +void ComponentBase::OnAnimation(animation::Params& params) { + for (const Component& child : children_) { + child->OnAnimation(params); + } +} + +/// @brief Return the currently Active child. +/// @return the currently Active child. +/// @ingroup component +Component ComponentBase::ActiveChild() { + for (auto& child : children_) { + if (child->Focusable()) { + return child; + } + } + return nullptr; +} + +/// @brief Return true when the component contains focusable elements. +/// The non focusable Components will be skipped when navigating using the +/// keyboard. +/// @ingroup component +bool ComponentBase::Focusable() const { + for (const Component& child : children_) { // NOLINT + if (child->Focusable()) { + return true; + } + } + return false; +} + +/// @brief Returns if the element if the currently active child of its parent. +/// @ingroup component +bool ComponentBase::Active() const { + return parent_ == nullptr || parent_->ActiveChild().get() == this; +} + +/// @brief Returns if the elements if focused by the user. +/// True when the ComponentBase is focused by the user. An element is Focused +/// when it is with all its ancestors the ActiveChild() of their parents, and it +/// Focusable(). +/// @ingroup component +bool ComponentBase::Focused() const { + const auto* current = this; + while (current && current->Active()) { + current = current->parent_; + } + return !current && Focusable(); +} + +/// @brief Make the |child| to be the "active" one. +/// @param child the child to become active. +/// @ingroup component +void ComponentBase::SetActiveChild([[maybe_unused]] ComponentBase* child) {} + +/// @brief Make the |child| to be the "active" one. +/// @param child the child to become active. +/// @ingroup component +void ComponentBase::SetActiveChild(Component child) { // NOLINT + SetActiveChild(child.get()); +} + +/// @brief Configure all the ancestors to give focus to this component. +/// @ingroup component +void ComponentBase::TakeFocus() { + ComponentBase* child = this; + while (ComponentBase* parent = child->parent_) { + parent->SetActiveChild(child); + child = parent; + } +} + +/// @brief Take the CapturedMouse if available. There is only one component of +/// them. It represents a component taking priority over others. +/// @param event The event +/// @ingroup component +CapturedMouse ComponentBase::CaptureMouse(const Event& event) { // NOLINT + if (event.screen_) { + return event.screen_->CaptureMouse(); + } + return std::make_unique<CaptureMouseImpl>(); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/component_options.cpp b/contrib/libs/ftxui/src/ftxui/component/component_options.cpp new file mode 100644 index 00000000000..77b439f36c2 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/component_options.cpp @@ -0,0 +1,358 @@ +// Copyright 2022 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include "ftxui/component/component_options.hpp" + +#include <ftxui/screen/color.hpp> // for Color, Color::White, Color::Black, Color::GrayDark, Color::Blue, Color::GrayLight, Color::Red +#include <memory> // for shared_ptr +#include <utility> // for move +#include "ftxui/component/animation.hpp" // for Function, Duration +#include "ftxui/dom/direction.hpp" +#include "ftxui/dom/elements.hpp" // for operator|=, Element, text, bgcolor, inverted, bold, dim, operator|, color, borderEmpty, hbox, automerge, border, borderLight + +namespace ftxui { + +/// @brief A color option that can be animated. +/// @params _inactive The color when the component is inactive. +/// @params _active The color when the component is active. +/// @params _duration The duration of the animation. +/// @params _function The easing function of the animation. +/// @ingroup component +void AnimatedColorOption::Set(Color _inactive, + Color _active, + animation::Duration _duration, + animation::easing::Function _function) { + enabled = true; + inactive = _inactive; + active = _active; + duration = _duration; + function = std::move(_function); +} + +/// @brief Set how the underline should animate. +/// @param d The duration of the animation. +/// @param f The easing function of the animation. +/// @ingroup component +void UnderlineOption::SetAnimation(animation::Duration d, + animation::easing::Function f) { + SetAnimationDuration(d); + SetAnimationFunction(std::move(f)); +} + +/// @brief Set how the underline should animate. +/// @param d The duration of the animation. +/// @ingroup component +void UnderlineOption::SetAnimationDuration(animation::Duration d) { + leader_duration = d; + follower_duration = d; +} + +/// @brief Set how the underline should animate. +/// @param f The easing function of the animation. +/// @ingroup component +void UnderlineOption::SetAnimationFunction(animation::easing::Function f) { + leader_function = f; + follower_function = std::move(f); +} + +/// @brief Set how the underline should animate. +/// This is useful to desynchronize the animation of the leader and the +/// follower. +/// @param f_leader The duration of the animation for the leader. +/// @param f_follower The duration of the animation for the follower. +/// @ingroup component +void UnderlineOption::SetAnimationFunction( + animation::easing::Function f_leader, + animation::easing::Function f_follower) { + leader_function = std::move(f_leader); + follower_function = std::move(f_follower); +} + +/// @brief Standard options for an horizontal menu. +/// This can be useful to implement a tab bar. +/// @ingroup component +// static +MenuOption MenuOption::Horizontal() { + MenuOption option; + option.direction = Direction::Right; + option.entries_option.transform = [](const EntryState& state) { + Element e = text(state.label); + if (state.focused) { + e |= inverted; + } + if (state.active) { + e |= bold; + } + if (!state.focused && !state.active) { + e |= dim; + } + return e; + }; + option.elements_infix = [] { return text(" "); }; + + return option; +} + +/// @brief Standard options for an animated horizontal menu. +/// This can be useful to implement a tab bar. +/// @ingroup component +// static +MenuOption MenuOption::HorizontalAnimated() { + auto option = Horizontal(); + option.underline.enabled = true; + return option; +} + +/// @brief Standard options for a vertical menu. +/// This can be useful to implement a list of selectable items. +/// @ingroup component +// static +MenuOption MenuOption::Vertical() { + MenuOption option; + option.entries_option.transform = [](const EntryState& state) { + Element e = text((state.active ? "> " : " ") + state.label); // NOLINT + if (state.focused) { + e |= inverted; + } + if (state.active) { + e |= bold; + } + if (!state.focused && !state.active) { + e |= dim; + } + return e; + }; + return option; +} + +/// @brief Standard options for an animated vertical menu. +/// This can be useful to implement a list of selectable items. +/// @ingroup component +// static +MenuOption MenuOption::VerticalAnimated() { + auto option = MenuOption::Vertical(); + option.entries_option.transform = [](const EntryState& state) { + Element e = text(state.label); + if (state.focused) { + e |= inverted; + } + if (state.active) { + e |= bold; + } + if (!state.focused && !state.active) { + e |= dim; + } + return e; + }; + option.underline.enabled = true; + return option; +} + +/// @brief Standard options for a horitontal menu with some separator. +/// This can be useful to implement a tab bar. +/// @ingroup component +// static +MenuOption MenuOption::Toggle() { + auto option = MenuOption::Horizontal(); + option.elements_infix = [] { return text("│") | automerge; }; + return option; +} + +/// @brief Create a ButtonOption, highlighted using [] characters. +/// @ingroup component +// static +ButtonOption ButtonOption::Ascii() { + ButtonOption option; + option.transform = [](const EntryState& s) { + const std::string t = s.focused ? "[" + s.label + "]" // + : " " + s.label + " "; + return text(t); + }; + return option; +} + +/// @brief Create a ButtonOption, inverted when focused. +/// @ingroup component +// static +ButtonOption ButtonOption::Simple() { + ButtonOption option; + option.transform = [](const EntryState& s) { + auto element = text(s.label) | borderLight; + if (s.focused) { + element |= inverted; + } + return element; + }; + return option; +} + +/// @brief Create a ButtonOption. The button is shown using a border, inverted +/// when focused. This is the current default. +/// @ingroup component +ButtonOption ButtonOption::Border() { + ButtonOption option; + option.transform = [](const EntryState& s) { + auto element = text(s.label) | border; + if (s.active) { + element |= bold; + } + if (s.focused) { + element |= inverted; + } + return element; + }; + return option; +} + +/// @brief Create a ButtonOption, using animated colors. +/// @ingroup component +// static +ButtonOption ButtonOption::Animated() { + return Animated(Color::Black, Color::GrayLight, // + Color::GrayDark, Color::White); +} + +/// @brief Create a ButtonOption, using animated colors. +/// @ingroup component +// static +ButtonOption ButtonOption::Animated(Color color) { + return ButtonOption::Animated( + Color::Interpolate(0.85F, color, Color::Black), // NOLINT + Color::Interpolate(0.10F, color, Color::White), // NOLINT + Color::Interpolate(0.10F, color, Color::Black), // NOLINT + Color::Interpolate(0.85F, color, Color::White)); // NOLINT +} + +/// @brief Create a ButtonOption, using animated colors. +/// @ingroup component +// static +ButtonOption ButtonOption::Animated(Color background, Color foreground) { + // NOLINTBEGIN + return ButtonOption::Animated( + /*bakground=*/background, + /*foreground=*/foreground, + /*background_active=*/foreground, + /*foreground_active=*/background); + // NOLINTEND +} + +/// @brief Create a ButtonOption, using animated colors. +/// @ingroup component +// static +ButtonOption ButtonOption::Animated(Color background, + Color foreground, + Color background_active, + Color foreground_active) { + ButtonOption option; + option.transform = [](const EntryState& s) { + auto element = text(s.label) | borderEmpty; + if (s.focused) { + element |= bold; + } + return element; + }; + option.animated_colors.foreground.Set(foreground, foreground_active); + option.animated_colors.background.Set(background, background_active); + return option; +} + +/// @brief Option for standard Checkbox. +/// @ingroup component +// static +CheckboxOption CheckboxOption::Simple() { + auto option = CheckboxOption(); + option.transform = [](const EntryState& s) { +#if defined(FTXUI_MICROSOFT_TERMINAL_FALLBACK) + // Microsoft terminal do not use fonts able to render properly the default + // radiobox glyph. + auto prefix = text(s.state ? "[X] " : "[ ] "); // NOLINT +#else + auto prefix = text(s.state ? "▣ " : "☐ "); // NOLINT +#endif + auto t = text(s.label); + if (s.active) { + t |= bold; + } + if (s.focused) { + t |= inverted; + } + return hbox({prefix, t}); + }; + return option; +} + +/// @brief Option for standard Radiobox +/// @ingroup component +// static +RadioboxOption RadioboxOption::Simple() { + auto option = RadioboxOption(); + option.transform = [](const EntryState& s) { +#if defined(FTXUI_MICROSOFT_TERMINAL_FALLBACK) + // Microsoft terminal do not use fonts able to render properly the default + // radiobox glyph. + auto prefix = text(s.state ? "(*) " : "( ) "); // NOLINT +#else + auto prefix = text(s.state ? "◉ " : "○ "); // NOLINT +#endif + auto t = text(s.label); + if (s.active) { + t |= bold; + } + if (s.focused) { + t |= inverted; + } + return hbox({prefix, t}); + }; + return option; +} + +/// @brief Standard options for the input component. +/// @ingroup component +// static +InputOption InputOption::Default() { + InputOption option; + option.transform = [](InputState state) { + state.element |= color(Color::White); + + if (state.is_placeholder) { + state.element |= dim; + } + + if (state.focused) { + state.element |= inverted; + } else if (state.hovered) { + state.element |= bgcolor(Color::GrayDark); + } + + return state.element; + }; + return option; +} + +/// @brief Standard options for a more beautiful input component. +/// @ingroup component +// static +InputOption InputOption::Spacious() { + InputOption option; + option.transform = [](InputState state) { + state.element |= borderEmpty; + state.element |= color(Color::White); + + if (state.is_placeholder) { + state.element |= dim; + } + + if (state.focused) { + state.element |= bgcolor(Color::Black); + } + + if (state.hovered) { + state.element |= bgcolor(Color::GrayDark); + } + + return state.element; + }; + return option; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/container.cpp b/contrib/libs/ftxui/src/ftxui/component/container.cpp new file mode 100644 index 00000000000..68de8b43b1b --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/container.cpp @@ -0,0 +1,438 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for max, min +#include <cstddef> // for size_t +#include <memory> // for make_shared, __shared_ptr_access, allocator, shared_ptr, allocator_traits<>::value_type +#include <utility> // for move + +#include "ftxui/component/component.hpp" // for Horizontal, Vertical, Tab +#include "ftxui/component/component_base.hpp" // for Components, Component, ComponentBase +#include "ftxui/component/event.hpp" // for Event, Event::Tab, Event::TabReverse, Event::ArrowDown, Event::ArrowLeft, Event::ArrowRight, Event::ArrowUp, Event::End, Event::Home, Event::PageDown, Event::PageUp +#include "ftxui/component/mouse.hpp" // for Mouse, Mouse::WheelDown, Mouse::WheelUp +#include "ftxui/dom/elements.hpp" // for text, Elements, operator|, reflect, Element, hbox, vbox +#include "ftxui/screen/box.hpp" // for Box + +namespace ftxui { + +class ContainerBase : public ComponentBase { + public: + ContainerBase(Components children, int* selector) + : selector_(selector ? selector : &selected_) { + for (Component& child : children) { + Add(std::move(child)); + } + } + + // Component override. + bool OnEvent(Event event) override { + if (event.is_mouse()) { + return OnMouseEvent(event); + } + + if (!Focused()) { + return false; + } + + if (ActiveChild() && ActiveChild()->OnEvent(event)) { + return true; + } + + return EventHandler(event); + } + + Component ActiveChild() override { + if (children_.empty()) { + return nullptr; + } + + return children_[static_cast<size_t>(*selector_) % children_.size()]; + } + + void SetActiveChild(ComponentBase* child) override { + for (size_t i = 0; i < children_.size(); ++i) { + if (children_[i].get() == child) { + *selector_ = static_cast<int>(i); + return; + } + } + } + + protected: + // Handlers + virtual bool EventHandler(Event /*unused*/) { return false; } // NOLINT + + virtual bool OnMouseEvent(Event event) { + return ComponentBase::OnEvent(std::move(event)); + } + + int selected_ = 0; + int* selector_ = nullptr; + + void MoveSelector(int dir) { + for (int i = *selector_ + dir; i >= 0 && i < int(children_.size()); + i += dir) { + if (children_[i]->Focusable()) { + *selector_ = i; + return; + } + } + } + + void MoveSelectorWrap(int dir) { + if (children_.empty()) { + return; + } + for (size_t offset = 1; offset < children_.size(); ++offset) { + const size_t i = + (*selector_ + offset * dir + children_.size()) % children_.size(); + if (children_[i]->Focusable()) { + *selector_ = int(i); + return; + } + } + } +}; + +class VerticalContainer : public ContainerBase { + public: + using ContainerBase::ContainerBase; + + Element OnRender() override { + Elements elements; + elements.reserve(children_.size()); + for (auto& it : children_) { + elements.push_back(it->Render()); + } + if (elements.empty()) { + return text("Empty container") | reflect(box_); + } + return vbox(std::move(elements)) | reflect(box_); + } + + bool EventHandler(Event event) override { + const int old_selected = *selector_; + if (event == Event::ArrowUp || event == Event::Character('k')) { + MoveSelector(-1); + } + if (event == Event::ArrowDown || event == Event::Character('j')) { + MoveSelector(+1); + } + if (event == Event::PageUp) { + for (int i = 0; i < box_.y_max - box_.y_min; ++i) { + MoveSelector(-1); + } + } + if (event == Event::PageDown) { + for (int i = 0; i < box_.y_max - box_.y_min; ++i) { + MoveSelector(1); + } + } + if (event == Event::Home) { + for (size_t i = 0; i < children_.size(); ++i) { + MoveSelector(-1); + } + } + if (event == Event::End) { + for (size_t i = 0; i < children_.size(); ++i) { + MoveSelector(1); + } + } + if (event == Event::Tab) { + MoveSelectorWrap(+1); + } + if (event == Event::TabReverse) { + MoveSelectorWrap(-1); + } + + *selector_ = std::max(0, std::min(int(children_.size()) - 1, *selector_)); + return old_selected != *selector_; + } + + bool OnMouseEvent(Event event) override { + if (ContainerBase::OnMouseEvent(event)) { + return true; + } + + if (event.mouse().button != Mouse::WheelUp && + event.mouse().button != Mouse::WheelDown) { + return false; + } + + if (!box_.Contain(event.mouse().x, event.mouse().y)) { + return false; + } + + const int old_selected = *selector_; + if (event.mouse().button == Mouse::WheelUp) { + MoveSelector(-1); + } + if (event.mouse().button == Mouse::WheelDown) { + MoveSelector(+1); + } + *selector_ = std::max(0, std::min(int(children_.size()) - 1, *selector_)); + + return old_selected != *selector_; + } + + Box box_; +}; + +class HorizontalContainer : public ContainerBase { + public: + using ContainerBase::ContainerBase; + + Element OnRender() override { + Elements elements; + elements.reserve(children_.size()); + for (auto& it : children_) { + elements.push_back(it->Render()); + } + if (elements.empty()) { + return text("Empty container"); + } + return hbox(std::move(elements)); + } + + bool EventHandler(Event event) override { + const int old_selected = *selector_; + if (event == Event::ArrowLeft || event == Event::Character('h')) { + MoveSelector(-1); + } + if (event == Event::ArrowRight || event == Event::Character('l')) { + MoveSelector(+1); + } + if (event == Event::Tab) { + MoveSelectorWrap(+1); + } + if (event == Event::TabReverse) { + MoveSelectorWrap(-1); + } + + *selector_ = std::max(0, std::min(int(children_.size()) - 1, *selector_)); + return old_selected != *selector_; + } +}; + +class TabContainer : public ContainerBase { + public: + using ContainerBase::ContainerBase; + + Element OnRender() override { + const Component active_child = ActiveChild(); + if (active_child) { + return active_child->Render(); + } + return text("Empty container"); + } + + bool Focusable() const override { + if (children_.empty()) { + return false; + } + return children_[size_t(*selector_) % children_.size()]->Focusable(); + } + + bool OnMouseEvent(Event event) override { + return ActiveChild() && ActiveChild()->OnEvent(event); + } +}; + +class StackedContainer : public ContainerBase { + public: + explicit StackedContainer(Components children) + : ContainerBase(std::move(children), nullptr) {} + + private: + Element OnRender() final { + Elements elements; + for (auto& child : children_) { + elements.push_back(child->Render()); + } + // Reverse the order of the elements. + std::reverse(elements.begin(), elements.end()); + return dbox(std::move(elements)); + } + + bool Focusable() const final { + for (const auto& child : children_) { + if (child->Focusable()) { + return true; + } + } + return false; + } + + Component ActiveChild() final { + if (children_.empty()) { + return nullptr; + } + return children_[0]; + } + + void SetActiveChild(ComponentBase* child) final { + if (children_.empty()) { + return; + } + + // Find `child` and put it at the beginning without change the order of the + // other children. + auto it = + std::find_if(children_.begin(), children_.end(), + [child](const Component& c) { return c.get() == child; }); + if (it == children_.end()) { + return; + } + std::rotate(children_.begin(), it, it + 1); + } + + bool OnEvent(Event event) final { + for (auto& child : children_) { + if (child->OnEvent(event)) { + return true; + } + } + return false; + } +}; + +namespace Container { + +/// @brief A list of components, drawn one by one vertically and navigated +/// vertically using up/down arrow key or 'j'/'k' keys. +/// @param children the list of components. +/// @ingroup component +/// @see ContainerBase +/// +/// ### Example +/// +/// ```cpp +/// auto container = Container::Vertical({ +/// children_1, +/// children_2, +/// children_3, +/// children_4, +/// }); +/// ``` +Component Vertical(Components children) { + return Vertical(std::move(children), nullptr); +} + +/// @brief A list of components, drawn one by one vertically and navigated +/// vertically using up/down arrow key or 'j'/'k' keys. +/// This is useful for implementing a Menu for instance. +/// @param children the list of components. +/// @param selector A reference to the index of the selected children. +/// @ingroup component +/// @see ContainerBase +/// +/// ### Example +/// +/// ```cpp +/// auto container = Container::Vertical({ +/// children_1, +/// children_2, +/// children_3, +/// children_4, +/// }, &selected_children); +/// ``` +Component Vertical(Components children, int* selector) { + return std::make_shared<VerticalContainer>(std::move(children), selector); +} + +/// @brief A list of components, drawn one by one horizontally and navigated +/// horizontally using left/right arrow key or 'h'/'l' keys. +/// @param children the list of components. +/// @ingroup component +/// @see ContainerBase +/// +/// ### Example +/// +/// ```cpp +/// int selected_children = 2; +/// auto container = Container::Horizontal({ +/// children_1, +/// children_2, +/// children_3, +/// children_4, +/// }); +/// ``` +Component Horizontal(Components children) { + return Horizontal(std::move(children), nullptr); +} + +/// @brief A list of components, drawn one by one horizontally and navigated +/// horizontally using left/right arrow key or 'h'/'l' keys. +/// @param children the list of components. +/// @param selector A reference to the index of the selected children. +/// @ingroup component +/// @see ContainerBase +/// +/// ### Example +/// +/// ```cpp +/// int selected_children = 2; +/// auto container = Container::Horizontal({ +/// children_1, +/// children_2, +/// children_3, +/// children_4, +/// }, selected_children); +/// ``` +Component Horizontal(Components children, int* selector) { + return std::make_shared<HorizontalContainer>(std::move(children), selector); +} + +/// @brief A list of components, where only one is drawn and interacted with at +/// a time. The |selector| gives the index of the selected component. This is +/// useful to implement tabs. +/// @param children The list of components. +/// @param selector The index of the drawn children. +/// @ingroup component +/// @see ContainerBase +/// +/// ### Example +/// +/// ```cpp +/// int tab_drawn = 0; +/// auto container = Container::Tab({ +/// children_1, +/// children_2, +/// children_3, +/// children_4, +/// }, &tab_drawn); +/// ``` +Component Tab(Components children, int* selector) { + return std::make_shared<TabContainer>(std::move(children), selector); +} + +/// @brief A list of components to be stacked on top of each other. +/// Events are propagated to the first component, then the second if not +/// handled, etc. +/// The components are drawn in the reverse order they are given. +/// When a component take focus, it is put at the front, without changing the +/// relative order of the other elements. +/// +/// This should be used with the `Window` component. +/// +/// @param children The list of components. +/// @ingroup component +/// @see Window +/// +/// ### Example +/// +/// ```cpp +/// auto container = Container::Stacked({ +/// children_1, +/// children_2, +/// children_3, +/// children_4, +/// }); +/// ``` +Component Stacked(Components children) { + return std::make_shared<StackedContainer>(std::move(children)); +} + +} // namespace Container + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/dropdown.cpp b/contrib/libs/ftxui/src/ftxui/component/dropdown.cpp new file mode 100644 index 00000000000..1635f5bee23 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/dropdown.cpp @@ -0,0 +1,141 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <ftxui/component/event.hpp> +#include <functional> // for function +#include <string> // for string + +#include <utility> +#include "ftxui/component/component.hpp" // for Maybe, Checkbox, Make, Radiobox, Vertical, Dropdown +#include "ftxui/component/component_base.hpp" // for Component, ComponentBase +#include "ftxui/component/component_options.hpp" // for CheckboxOption, EntryState +#include "ftxui/dom/elements.hpp" // for operator|, Element, border, filler, operator|=, separator, size, text, vbox, frame, vscroll_indicator, hbox, HEIGHT, LESS_THAN, bold, inverted +#include "ftxui/screen/util.hpp" // for clamp +#include "ftxui/util/ref.hpp" // for ConstStringListRef + +namespace ftxui { + +/// @brief A dropdown menu. +/// @ingroup component +/// @param entries The list of entries to display. +/// @param selected The index of the selected entry. +Component Dropdown(ConstStringListRef entries, int* selected) { + DropdownOption option; + option.radiobox.entries = std::move(entries); + option.radiobox.selected = selected; + return Dropdown(option); +} + +/// @brief A dropdown menu. +/// @ingroup component +/// @param option The options for the dropdown. +// NOLINTNEXTLINE +Component Dropdown(DropdownOption option) { + class Impl : public ComponentBase, public DropdownOption { + public: + explicit Impl(DropdownOption option) : DropdownOption(std::move(option)) { + FillDefault(); + checkbox_ = Checkbox(checkbox); + radiobox_ = Radiobox(radiobox); + + Add(Container::Vertical({ + checkbox_, + Maybe(radiobox_, checkbox.checked), + })); + } + + Element OnRender() override { + selected_ = + util::clamp(radiobox.selected(), 0, int(radiobox.entries.size()) - 1); + selected_ = util::clamp(selected_(), 0, int(radiobox.entries.size()) - 1); + + if (selected_() >= 0 && selected_() < int(radiobox.entries.size())) { + title_ = radiobox.entries[selected_()]; + } + + return transform(*open_, checkbox_->Render(), radiobox_->Render()); + } + + // Switch focus in between the checkbox and the radiobox when selecting it. + bool OnEvent(ftxui::Event event) override { + const bool open_old = open_(); + const int selected_old = selected_(); + bool handled = ComponentBase::OnEvent(event); + + // Transfer focus to the radiobox when the dropdown is opened. + if (!open_old && open_()) { + radiobox_->TakeFocus(); + } + + // Auto-close the dropdown when the user selects an item, even if the item + // it the same as the previous one. + if (open_old && open_()) { + const bool should_close = + (selected_() != selected_old) || // + (event == Event::Return) || // + (event == Event::Character(' ')) || // + (event == Event::Escape) || // + (event.is_mouse() && event.mouse().button == Mouse::Left && + event.mouse().motion == Mouse::Pressed); + + if (should_close) { + checkbox_->TakeFocus(); + open_ = false; + handled = true; + } + } + + return handled; + } + + void FillDefault() { + open_ = checkbox.checked; + selected_ = radiobox.selected; + checkbox.checked = &*open_; + radiobox.selected = &*selected_; + checkbox.label = &title_; + + if (!checkbox.transform) { + checkbox.transform = [](const EntryState& s) { + auto prefix = text(s.state ? "↓ " : "→ "); // NOLINT + auto t = text(s.label); + if (s.active) { + t |= bold; + } + if (s.focused) { + t |= inverted; + } + return hbox({prefix, t}); + }; + } + + if (!transform) { + transform = [](bool is_open, Element checkbox_element, + Element radiobox_element) { + if (is_open) { + const int max_height = 12; + return vbox({ + std::move(checkbox_element), + separator(), + std::move(radiobox_element) | vscroll_indicator | frame | + size(HEIGHT, LESS_THAN, max_height), + }) | + border; + } + return vbox({std::move(checkbox_element), filler()}) | border; + }; + } + } + + private: + Ref<bool> open_; + Ref<int> selected_; + Component checkbox_; + Component radiobox_; + std::string title_; + }; + + return Make<Impl>(option); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/event.cpp b/contrib/libs/ftxui/src/ftxui/component/event.cpp new file mode 100644 index 00000000000..25a50575455 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/event.cpp @@ -0,0 +1,467 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <map> // for map +#include <string> +#include <utility> // for move + +#include "ftxui/component/event.hpp" +#include "ftxui/component/mouse.hpp" // for Mouse +#include "ftxui/screen/string.hpp" // for to_wstring + +// Disable warning for shadowing variable, for every compilers. Indeed, there is +// a static Event for every letter of the alphabet: +#ifdef __clang__ +#pragma clang diagnostic ignored "-Wshadow" +#elif __GNUC__ +#pragma GCC diagnostic ignored "-Wshadow" +#elif defined(_MSC_VER) +#pragma warning(disable : 6244) +#pragma warning(disable : 6246) +#endif + +namespace ftxui { + +/// @brief An event corresponding to a given typed character. +/// @param input The character typed by the user. +/// @ingroup component +// static +Event Event::Character(std::string input) { + Event event; + event.input_ = std::move(input); + event.type_ = Type::Character; + return event; +} + +/// @brief An event corresponding to a given typed character. +/// @param c The character typed by the user. +/// @ingroup component +// static +Event Event::Character(char c) { + return Event::Character(std::string{c}); +} + +/// @brief An event corresponding to a given typed character. +/// @param c The character typed by the user. +/// @ingroup component +// static +Event Event::Character(wchar_t c) { + return Event::Character(to_string(std::wstring{c})); +} + +/// @brief An event corresponding to a given typed character. +/// @param input The sequence of character send by the terminal. +/// @param mouse The mouse state. +/// @ingroup component +// static +Event Event::Mouse(std::string input, struct Mouse mouse) { + Event event; + event.input_ = std::move(input); + event.type_ = Type::Mouse; + event.data_.mouse = mouse; // NOLINT + return event; +} + +/// @brief An event corresponding to a terminal DCS (Device Control String). +// static +Event Event::CursorShape(std::string input, int shape) { + Event event; + event.input_ = std::move(input); + event.type_ = Type::CursorShape; + event.data_.cursor_shape = shape; // NOLINT + return event; +} + +/// @brief An custom event whose meaning is defined by the user of the library. +/// @param input An arbitrary sequence of character defined by the developer. +/// @ingroup component. +// static +Event Event::Special(std::string input) { + Event event; + event.input_ = std::move(input); + return event; +} + +/// @internal +// static +Event Event::CursorPosition(std::string input, int x, int y) { + Event event; + event.input_ = std::move(input); + event.type_ = Type::CursorPosition; + event.data_.cursor = {x, y}; // NOLINT + return event; +} + +/// @brief Return a string representation of the event. +std::string Event::DebugString() const { + static std::map<Event, const char*> event_to_string = { + // --- Arrow --- + {Event::ArrowLeft, "Event::ArrowLeft"}, + {Event::ArrowRight, "Event::ArrowRight"}, + {Event::ArrowUp, "Event::ArrowUp"}, + {Event::ArrowDown, "Event::ArrowDown"}, + + // --- ArrowCtrl --- + {Event::ArrowLeftCtrl, "Event::ArrowLeftCtrl"}, + {Event::ArrowRightCtrl, "Event::ArrowRightCtrl"}, + {Event::ArrowUpCtrl, "Event::ArrowUpCtrl"}, + {Event::ArrowDownCtrl, "Event::ArrowDownCtrl"}, + + // --- Other --- + {Event::Backspace, "Event::Backspace"}, + {Event::Delete, "Event::Delete"}, + {Event::Escape, "Event::Escape"}, + {Event::Return, "Event::Return"}, + {Event::Tab, "Event::Tab"}, + {Event::TabReverse, "Event::TabReverse"}, + + // --- Function keys --- + {Event::F1, "Event::F1"}, + {Event::F2, "Event::F2"}, + {Event::F3, "Event::F3"}, + {Event::F4, "Event::F4"}, + {Event::F5, "Event::F5"}, + {Event::F6, "Event::F6"}, + {Event::F7, "Event::F7"}, + {Event::F8, "Event::F8"}, + {Event::F9, "Event::F9"}, + {Event::F10, "Event::F10"}, + {Event::F11, "Event::F11"}, + {Event::F12, "Event::F12"}, + + // --- Navigation keys --- + {Event::Insert, "Event::Insert"}, + {Event::Home, "Event::Home"}, + {Event::End, "Event::End"}, + {Event::PageUp, "Event::PageUp"}, + {Event::PageDown, "Event::PageDown"}, + + // --- Control keys --- + {Event::CtrlA, "Event::CtrlA"}, + {Event::CtrlB, "Event::CtrlB"}, + {Event::CtrlC, "Event::CtrlC"}, + {Event::CtrlD, "Event::CtrlD"}, + {Event::CtrlE, "Event::CtrlE"}, + {Event::CtrlF, "Event::CtrlF"}, + {Event::CtrlG, "Event::CtrlG"}, + {Event::CtrlH, "Event::CtrlH"}, + {Event::CtrlI, "Event::CtrlI"}, + {Event::CtrlJ, "Event::CtrlJ"}, + {Event::CtrlK, "Event::CtrlK"}, + {Event::CtrlL, "Event::CtrlL"}, + {Event::CtrlM, "Event::CtrlM"}, + {Event::CtrlN, "Event::CtrlN"}, + {Event::CtrlO, "Event::CtrlO"}, + {Event::CtrlP, "Event::CtrlP"}, + {Event::CtrlQ, "Event::CtrlQ"}, + {Event::CtrlR, "Event::CtrlR"}, + {Event::CtrlS, "Event::CtrlS"}, + {Event::CtrlT, "Event::CtrlT"}, + {Event::CtrlU, "Event::CtrlU"}, + {Event::CtrlV, "Event::CtrlV"}, + {Event::CtrlW, "Event::CtrlW"}, + {Event::CtrlX, "Event::CtrlX"}, + {Event::CtrlY, "Event::CtrlY"}, + {Event::CtrlZ, "Event::CtrlZ"}, + + // --- Alt keys --- + {Event::AltA, "Event::AltA"}, + {Event::AltB, "Event::AltB"}, + {Event::AltC, "Event::AltC"}, + {Event::AltD, "Event::AltD"}, + {Event::AltE, "Event::AltE"}, + {Event::AltF, "Event::AltF"}, + {Event::AltG, "Event::AltG"}, + {Event::AltH, "Event::AltH"}, + {Event::AltI, "Event::AltI"}, + {Event::AltJ, "Event::AltJ"}, + {Event::AltK, "Event::AltK"}, + {Event::AltL, "Event::AltL"}, + {Event::AltM, "Event::AltM"}, + {Event::AltN, "Event::AltN"}, + {Event::AltO, "Event::AltO"}, + {Event::AltP, "Event::AltP"}, + {Event::AltQ, "Event::AltQ"}, + {Event::AltR, "Event::AltR"}, + {Event::AltS, "Event::AltS"}, + {Event::AltT, "Event::AltT"}, + {Event::AltU, "Event::AltU"}, + {Event::AltV, "Event::AltV"}, + {Event::AltW, "Event::AltW"}, + {Event::AltX, "Event::AltX"}, + {Event::AltY, "Event::AltY"}, + {Event::AltZ, "Event::AltZ"}, + + // --- CtrlAlt keys --- + {Event::CtrlAltA, "Event::CtrlAltA"}, + {Event::CtrlAltB, "Event::CtrlAltB"}, + {Event::CtrlAltC, "Event::CtrlAltC"}, + {Event::CtrlAltD, "Event::CtrlAltD"}, + {Event::CtrlAltE, "Event::CtrlAltE"}, + {Event::CtrlAltF, "Event::CtrlAltF"}, + {Event::CtrlAltG, "Event::CtrlAltG"}, + {Event::CtrlAltH, "Event::CtrlAltH"}, + {Event::CtrlAltI, "Event::CtrlAltI"}, + {Event::CtrlAltJ, "Event::CtrlAltJ"}, + {Event::CtrlAltK, "Event::CtrlAltK"}, + {Event::CtrlAltL, "Event::CtrlAltL"}, + {Event::CtrlAltM, "Event::CtrlAltM"}, + {Event::CtrlAltN, "Event::CtrlAltN"}, + {Event::CtrlAltO, "Event::CtrlAltO"}, + {Event::CtrlAltP, "Event::CtrlAltP"}, + {Event::CtrlAltQ, "Event::CtrlAltQ"}, + {Event::CtrlAltR, "Event::CtrlAltR"}, + {Event::CtrlAltS, "Event::CtrlAltS"}, + {Event::CtrlAltT, "Event::CtrlAltT"}, + {Event::CtrlAltU, "Event::CtrlAltU"}, + {Event::CtrlAltV, "Event::CtrlAltV"}, + {Event::CtrlAltW, "Event::CtrlAltW"}, + {Event::CtrlAltX, "Event::CtrlAltX"}, + {Event::CtrlAltY, "Event::CtrlAltY"}, + {Event::CtrlAltZ, "Event::CtrlAltZ"}, + + // --- Custom --- + {Event::Custom, "Event::Custom"}, + }; + + static std::map<Mouse::Button, const char*> mouse_button_string = { + {Mouse::Button::Left, ".button = Mouse::Left"}, + {Mouse::Button::Middle, ".button = Mouse::Middle"}, + {Mouse::Button::Right, ".button = Mouse::Right"}, + {Mouse::Button::WheelUp, ".button = Mouse::WheelUp"}, + {Mouse::Button::WheelDown, ".button = Mouse::WheelDown"}, + {Mouse::Button::None, ".button = Mouse::None"}, + {Mouse::Button::WheelLeft, ".button = Mouse::WheelLeft"}, + {Mouse::Button::WheelRight, ".button = Mouse::WheelRight"}, + }; + + static std::map<Mouse::Motion, const char*> mouse_motion_string = { + {Mouse::Motion::Pressed, ".motion = Mouse::Pressed"}, + {Mouse::Motion::Released, ".motion = Mouse::Released"}, + {Mouse::Motion::Moved, ".motion = Mouse::Moved"}, + }; + + switch (type_) { + case Type::Character: { + return "Event::Character(\"" + input_ + "\")"; + } + case Type::Mouse: { + std::string out = "Event::Mouse(\"...\", Mouse{"; + out += std::string(mouse_button_string[data_.mouse.button]); + out += ", "; + out += std::string(mouse_motion_string[data_.mouse.motion]); + out += ", "; + if (data_.mouse.shift) { + out += ".shift = true, "; + } + if (data_.mouse.meta) { + out += ".meta = true, "; + } + if (data_.mouse.control) { + out += ".control = true, "; + } + out += ".x = " + std::to_string(data_.mouse.x); + out += ", "; + out += ".y = " + std::to_string(data_.mouse.y); + out += "})"; + return out; + } + case Type::CursorShape: + return "Event::CursorShape(" + input_ + ", " + + std::to_string(data_.cursor_shape) + ")"; + case Type::CursorPosition: + return "Event::CursorPosition(" + input_ + ", " + + std::to_string(data_.cursor.x) + ", " + + std::to_string(data_.cursor.y) + ")"; + default: { + auto event_it = event_to_string.find(*this); + if (event_it != event_to_string.end()) { + return event_it->second; + } + + return ""; + } + } + return ""; +} + +// clang-format off +// NOLINTBEGIN + +// --- Arrow --- +const Event Event::ArrowLeft = Event::Special("\x1B[D"); +const Event Event::ArrowRight = Event::Special("\x1B[C"); +const Event Event::ArrowUp = Event::Special("\x1B[A"); +const Event Event::ArrowDown = Event::Special("\x1B[B"); +const Event Event::ArrowLeftCtrl = Event::Special("\x1B[1;5D"); +const Event Event::ArrowRightCtrl = Event::Special("\x1B[1;5C"); +const Event Event::ArrowUpCtrl = Event::Special("\x1B[1;5A"); +const Event Event::ArrowDownCtrl = Event::Special("\x1B[1;5B"); +const Event Event::Backspace = Event::Special({127}); +const Event Event::Delete = Event::Special("\x1B[3~"); +const Event Event::Escape = Event::Special("\x1B"); +const Event Event::Return = Event::Special({10}); +const Event Event::Tab = Event::Special({9}); +const Event Event::TabReverse = Event::Special({27, 91, 90}); + +// See https://invisible-island.net/xterm/xterm-function-keys.html +// We follow xterm-new / vterm-xf86-v4 / mgt / screen +const Event Event::F1 = Event::Special("\x1BOP"); +const Event Event::F2 = Event::Special("\x1BOQ"); +const Event Event::F3 = Event::Special("\x1BOR"); +const Event Event::F4 = Event::Special("\x1BOS"); +const Event Event::F5 = Event::Special("\x1B[15~"); +const Event Event::F6 = Event::Special("\x1B[17~"); +const Event Event::F7 = Event::Special("\x1B[18~"); +const Event Event::F8 = Event::Special("\x1B[19~"); +const Event Event::F9 = Event::Special("\x1B[20~"); +const Event Event::F10 = Event::Special("\x1B[21~"); +const Event Event::F11 = Event::Special("\x1B[23~"); +const Event Event::F12 = Event::Special("\x1B[24~"); + +const Event Event::Insert = Event::Special("\x1B[2~"); +const Event Event::Home = Event::Special({27, 91, 72}); +const Event Event::End = Event::Special({27, 91, 70}); +const Event Event::PageUp = Event::Special({27, 91, 53, 126}); +const Event Event::PageDown = Event::Special({27, 91, 54, 126}); +const Event Event::Custom = Event::Special({0}); + +const Event Event::a = Event::Character("a"); +const Event Event::b = Event::Character("b"); +const Event Event::c = Event::Character("c"); +const Event Event::d = Event::Character("d"); +const Event Event::e = Event::Character("e"); +const Event Event::f = Event::Character("f"); +const Event Event::g = Event::Character("g"); +const Event Event::h = Event::Character("h"); +const Event Event::i = Event::Character("i"); +const Event Event::j = Event::Character("j"); +const Event Event::k = Event::Character("k"); +const Event Event::l = Event::Character("l"); +const Event Event::m = Event::Character("m"); +const Event Event::n = Event::Character("n"); +const Event Event::o = Event::Character("o"); +const Event Event::p = Event::Character("p"); +const Event Event::q = Event::Character("q"); +const Event Event::r = Event::Character("r"); +const Event Event::s = Event::Character("s"); +const Event Event::t = Event::Character("t"); +const Event Event::u = Event::Character("u"); +const Event Event::v = Event::Character("v"); +const Event Event::w = Event::Character("w"); +const Event Event::x = Event::Character("x"); +const Event Event::y = Event::Character("y"); +const Event Event::z = Event::Character("z"); + +const Event Event::A = Event::Character("A"); +const Event Event::B = Event::Character("B"); +const Event Event::C = Event::Character("C"); +const Event Event::D = Event::Character("D"); +const Event Event::E = Event::Character("E"); +const Event Event::F = Event::Character("F"); +const Event Event::G = Event::Character("G"); +const Event Event::H = Event::Character("H"); +const Event Event::I = Event::Character("I"); +const Event Event::J = Event::Character("J"); +const Event Event::K = Event::Character("K"); +const Event Event::L = Event::Character("L"); +const Event Event::M = Event::Character("M"); +const Event Event::N = Event::Character("N"); +const Event Event::O = Event::Character("O"); +const Event Event::P = Event::Character("P"); +const Event Event::Q = Event::Character("Q"); +const Event Event::R = Event::Character("R"); +const Event Event::S = Event::Character("S"); +const Event Event::T = Event::Character("T"); +const Event Event::U = Event::Character("U"); +const Event Event::V = Event::Character("V"); +const Event Event::W = Event::Character("W"); +const Event Event::X = Event::Character("X"); +const Event Event::Y = Event::Character("Y"); +const Event Event::Z = Event::Character("Z"); + +const Event Event::CtrlA = Event::Special("\x01"); +const Event Event::CtrlB = Event::Special("\x02"); +const Event Event::CtrlC = Event::Special("\x03"); +const Event Event::CtrlD = Event::Special("\x04"); +const Event Event::CtrlE = Event::Special("\x05"); +const Event Event::CtrlF = Event::Special("\x06"); +const Event Event::CtrlG = Event::Special("\x07"); +const Event Event::CtrlH = Event::Special("\x08"); +const Event Event::CtrlI = Event::Special("\x09"); +const Event Event::CtrlJ = Event::Special("\x0a"); +const Event Event::CtrlK = Event::Special("\x0b"); +const Event Event::CtrlL = Event::Special("\x0c"); +const Event Event::CtrlM = Event::Special("\x0d"); +const Event Event::CtrlN = Event::Special("\x0e"); +const Event Event::CtrlO = Event::Special("\x0f"); +const Event Event::CtrlP = Event::Special("\x10"); +const Event Event::CtrlQ = Event::Special("\x11"); +const Event Event::CtrlR = Event::Special("\x12"); +const Event Event::CtrlS = Event::Special("\x13"); +const Event Event::CtrlT = Event::Special("\x14"); +const Event Event::CtrlU = Event::Special("\x15"); +const Event Event::CtrlV = Event::Special("\x16"); +const Event Event::CtrlW = Event::Special("\x17"); +const Event Event::CtrlX = Event::Special("\x18"); +const Event Event::CtrlY = Event::Special("\x19"); +const Event Event::CtrlZ = Event::Special("\x1a"); + +const Event Event::AltA = Event::Special("\x1b""a"); +const Event Event::AltB = Event::Special("\x1b""b"); +const Event Event::AltC = Event::Special("\x1b""c"); +const Event Event::AltD = Event::Special("\x1b""d"); +const Event Event::AltE = Event::Special("\x1b""e"); +const Event Event::AltF = Event::Special("\x1b""f"); +const Event Event::AltG = Event::Special("\x1b""g"); +const Event Event::AltH = Event::Special("\x1b""h"); +const Event Event::AltI = Event::Special("\x1b""i"); +const Event Event::AltJ = Event::Special("\x1b""j"); +const Event Event::AltK = Event::Special("\x1b""k"); +const Event Event::AltL = Event::Special("\x1b""l"); +const Event Event::AltM = Event::Special("\x1b""m"); +const Event Event::AltN = Event::Special("\x1b""n"); +const Event Event::AltO = Event::Special("\x1b""o"); +const Event Event::AltP = Event::Special("\x1b""p"); +const Event Event::AltQ = Event::Special("\x1b""q"); +const Event Event::AltR = Event::Special("\x1b""r"); +const Event Event::AltS = Event::Special("\x1b""s"); +const Event Event::AltT = Event::Special("\x1b""t"); +const Event Event::AltU = Event::Special("\x1b""u"); +const Event Event::AltV = Event::Special("\x1b""v"); +const Event Event::AltW = Event::Special("\x1b""w"); +const Event Event::AltX = Event::Special("\x1b""x"); +const Event Event::AltY = Event::Special("\x1b""y"); +const Event Event::AltZ = Event::Special("\x1b""z"); + +const Event Event::CtrlAltA = Event::Special("\x1b\x01"); +const Event Event::CtrlAltB = Event::Special("\x1b\x02"); +const Event Event::CtrlAltC = Event::Special("\x1b\x03"); +const Event Event::CtrlAltD = Event::Special("\x1b\x04"); +const Event Event::CtrlAltE = Event::Special("\x1b\x05"); +const Event Event::CtrlAltF = Event::Special("\x1b\x06"); +const Event Event::CtrlAltG = Event::Special("\x1b\x07"); +const Event Event::CtrlAltH = Event::Special("\x1b\x08"); +const Event Event::CtrlAltI = Event::Special("\x1b\x09"); +const Event Event::CtrlAltJ = Event::Special("\x1b\x0a"); +const Event Event::CtrlAltK = Event::Special("\x1b\x0b"); +const Event Event::CtrlAltL = Event::Special("\x1b\x0c"); +const Event Event::CtrlAltM = Event::Special("\x1b\x0d"); +const Event Event::CtrlAltN = Event::Special("\x1b\x0e"); +const Event Event::CtrlAltO = Event::Special("\x1b\x0f"); +const Event Event::CtrlAltP = Event::Special("\x1b\x10"); +const Event Event::CtrlAltQ = Event::Special("\x1b\x11"); +const Event Event::CtrlAltR = Event::Special("\x1b\x12"); +const Event Event::CtrlAltS = Event::Special("\x1b\x13"); +const Event Event::CtrlAltT = Event::Special("\x1b\x14"); +const Event Event::CtrlAltU = Event::Special("\x1b\x15"); +const Event Event::CtrlAltV = Event::Special("\x1b\x16"); +const Event Event::CtrlAltW = Event::Special("\x1b\x17"); +const Event Event::CtrlAltX = Event::Special("\x1b\x18"); +const Event Event::CtrlAltY = Event::Special("\x1b\x19"); +const Event Event::CtrlAltZ = Event::Special("\x1b\x1a"); + +// NOLINTEND +// clang-format on + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/hoverable.cpp b/contrib/libs/ftxui/src/ftxui/component/hoverable.cpp new file mode 100644 index 00000000000..9af6498b613 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/hoverable.cpp @@ -0,0 +1,215 @@ +// Copyright 2022 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <functional> // for function +#include <utility> // for move + +#include "ftxui/component/component.hpp" // for ComponentDecorator, Hoverable, Make +#include "ftxui/component/component_base.hpp" // for ComponentBase +#include "ftxui/component/event.hpp" // for Event +#include "ftxui/component/mouse.hpp" // for Mouse +#include "ftxui/component/screen_interactive.hpp" // for Component, ScreenInteractive +#include "ftxui/dom/elements.hpp" // for operator|, reflect, Element +#include "ftxui/screen/box.hpp" // for Box + +namespace ftxui { + +namespace { + +void Post(std::function<void()> f) { + if (auto* screen = ScreenInteractive::Active()) { + screen->Post(std::move(f)); + return; + } + f(); +} + +} // namespace + +/// @brief Wrap a component. Gives the ability to know if it is hovered by the +/// mouse. +/// @param component The wrapped component. +/// @param hover The value to reflect whether the component is hovered or not. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto button = Button("exit", screen.ExitLoopClosure()); +/// bool hover = false; +/// auto button_hover = Hoverable(button, &hover); +/// ``` +// NOLINTNEXTLINE +Component Hoverable(Component component, bool* hover) { + class Impl : public ComponentBase { + public: + Impl(Component component, bool* hover) + : component_(std::move(component)), hover_(hover) { + Add(component_); + } + + private: + Element OnRender() override { + return ComponentBase::OnRender() | reflect(box_); + } + + bool OnEvent(Event event) override { + if (event.is_mouse()) { + *hover_ = box_.Contain(event.mouse().x, event.mouse().y) && + CaptureMouse(event); + } + + return ComponentBase::OnEvent(event); + } + + Component component_; + bool* hover_; + Box box_; + }; + + return Make<Impl>(component, hover); +} + +/// @brief Wrap a component. Uses callbacks. +/// @param component The wrapped component. +/// @param on_enter Callback OnEnter +/// @param on_leave Callback OnLeave +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto button = Button("exit", screen.ExitLoopClosure()); +/// bool hover = false; +/// auto button_hover = Hoverable(button, &hover); +/// ``` +Component Hoverable(Component component, + std::function<void()> on_enter, + std::function<void()> on_leave) { + class Impl : public ComponentBase { + public: + Impl(Component component, + std::function<void()> on_enter, + std::function<void()> on_leave) + : component_(std::move(component)), + on_enter_(std::move(on_enter)), + on_leave_(std::move(on_leave)) { + Add(component_); + } + + private: + Element OnRender() override { + return ComponentBase::OnRender() | reflect(box_); + } + + bool OnEvent(Event event) override { + if (event.is_mouse()) { + const bool hover = box_.Contain(event.mouse().x, event.mouse().y) && + CaptureMouse(event); + if (hover != hover_) { + Post(hover ? on_enter_ : on_leave_); + } + hover_ = hover; + } + + return ComponentBase::OnEvent(event); + } + + Component component_; + Box box_; + bool hover_ = false; + std::function<void()> on_enter_; + std::function<void()> on_leave_; + }; + + return Make<Impl>(std::move(component), std::move(on_enter), + std::move(on_leave)); +} + +/// @brief Wrap a component. Gives the ability to know if it is hovered by the +/// mouse. +/// @param hover The value to reflect whether the component is hovered or not. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// bool hover = false; +/// auto button = Button("exit", screen.ExitLoopClosure()); +/// button |= Hoverable(&hover); +/// ``` +ComponentDecorator Hoverable(bool* hover) { + return [hover](Component component) { + return Hoverable(std::move(component), hover); + }; +} + +/// @brief Wrap a component. Gives the ability to know if it is hovered by the +/// mouse. +/// @param on_enter is called when the mouse hover the component. +/// @param on_leave is called when the mouse leave the component. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto button = Button("exit", screen.ExitLoopClosure()); +/// int on_enter_cnt = 0; +/// int on_leave_cnt = 0; +/// button |= Hoverable( +/// [&]{ on_enter_cnt++; }, +/// [&]{ on_leave_cnt++; } +// ); +/// ``` +// NOLINTNEXTLINE +ComponentDecorator Hoverable(std::function<void()> on_enter, + // NOLINTNEXTLINE + std::function<void()> on_leave) { + return [on_enter, on_leave](Component component) { + return Hoverable(std::move(component), on_enter, on_leave); + }; +} + +/// @brief Wrap a component. Gives the ability to know if it is hovered by the +/// mouse. +/// @param component the wrapped component. +/// @param on_change is called when the mouse enter or leave the component. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto button = Button("exit", screen.ExitLoopClosure()); +/// bool hovered = false; +/// auto button_hoverable = Hoverable(button, +// [&](bool hover) { hovered = hover;}); +/// ``` +// NOLINTNEXTLINE +Component Hoverable(Component component, std::function<void(bool)> on_change) { + return Hoverable( + std::move(component), // + [on_change] { on_change(true); }, // + [on_change] { on_change(false); } // + ); +} + +/// @brief Wrap a component. Gives the ability to know if it is hovered by the +/// mouse. +/// @param on_change is called when the mouse enter or leave the component. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto button = Button("exit", screen.ExitLoopClosure()); +/// bool hovered = false; +/// button |= Hoverable([&](bool hover) { hovered = hover;}); +/// ``` +// NOLINTNEXTLINE +ComponentDecorator Hoverable(std::function<void(bool)> on_change) { + return [on_change](Component component) { + return Hoverable(std::move(component), on_change); + }; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/input.cpp b/contrib/libs/ftxui/src/ftxui/component/input.cpp new file mode 100644 index 00000000000..a61e62568df --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/input.cpp @@ -0,0 +1,631 @@ +// Copyright 2022 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for max, min +#include <cstddef> // for size_t +#include <cstdint> // for uint32_t +#include <functional> // for function +#include <sstream> // for basic_istream, stringstream +#include <string> // for string, basic_string, operator==, getline +#include <utility> // for move +#include <vector> // for vector + +#include "ftxui/component/component.hpp" // for Make, Input +#include "ftxui/component/component_base.hpp" // for ComponentBase +#include "ftxui/component/component_options.hpp" // for InputOption +#include "ftxui/component/event.hpp" // for Event, Event::ArrowDown, Event::ArrowLeft, Event::ArrowLeftCtrl, Event::ArrowRight, Event::ArrowRightCtrl, Event::ArrowUp, Event::Backspace, Event::Delete, Event::End, Event::Home, Event::Return +#include "ftxui/component/mouse.hpp" // for Mouse, Mouse::Left, Mouse::Pressed +#include "ftxui/component/screen_interactive.hpp" // for Component +#include "ftxui/dom/elements.hpp" // for operator|, reflect, text, Element, xflex, hbox, Elements, frame, operator|=, vbox, focus, focusCursorBarBlinking, select +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/string.hpp" // for string_width +#include "ftxui/screen/string_internal.hpp" // for GlyphNext, GlyphPrevious, WordBreakProperty, EatCodePoint, CodepointToWordBreakProperty, IsFullWidth, WordBreakProperty::ALetter, WordBreakProperty::CR, WordBreakProperty::Double_Quote, WordBreakProperty::Extend, WordBreakProperty::ExtendNumLet, WordBreakProperty::Format, WordBreakProperty::Hebrew_Letter, WordBreakProperty::Katakana, WordBreakProperty::LF, WordBreakProperty::MidLetter, WordBreakProperty::MidNum, WordBreakProperty::MidNumLet, WordBreakProperty::Newline, WordBreakProperty::Numeric, WordBreakProperty::Regional_Indicator, WordBreakProperty::Single_Quote, WordBreakProperty::WSegSpace, WordBreakProperty::ZWJ +#include "ftxui/screen/util.hpp" // for clamp +#include "ftxui/util/ref.hpp" // for StringRef, Ref + +namespace ftxui { + +namespace { + +std::vector<std::string> Split(const std::string& input) { + std::vector<std::string> output; + std::stringstream ss(input); + std::string line; + while (std::getline(ss, line)) { + output.push_back(line); + } + if (input.back() == '\n') { + output.emplace_back(""); + } + return output; +} + +size_t GlyphWidth(const std::string& input, size_t iter) { + uint32_t ucs = 0; + if (!EatCodePoint(input, iter, &iter, &ucs)) { + return 0; + } + if (IsFullWidth(ucs)) { + return 2; + } + return 1; +} + +bool IsWordCodePoint(uint32_t codepoint) { + switch (CodepointToWordBreakProperty(codepoint)) { + case WordBreakProperty::ALetter: + case WordBreakProperty::Hebrew_Letter: + case WordBreakProperty::Katakana: + case WordBreakProperty::Numeric: + return true; + + case WordBreakProperty::CR: + case WordBreakProperty::Double_Quote: + case WordBreakProperty::LF: + case WordBreakProperty::MidLetter: + case WordBreakProperty::MidNum: + case WordBreakProperty::MidNumLet: + case WordBreakProperty::Newline: + case WordBreakProperty::Single_Quote: + case WordBreakProperty::WSegSpace: + // Unexpected/Unsure + case WordBreakProperty::Extend: + case WordBreakProperty::ExtendNumLet: + case WordBreakProperty::Format: + case WordBreakProperty::Regional_Indicator: + case WordBreakProperty::ZWJ: + return false; + } + return false; // NOT_REACHED(); +} + +bool IsWordCharacter(const std::string& input, size_t iter) { + uint32_t ucs = 0; + if (!EatCodePoint(input, iter, &iter, &ucs)) { + return false; + } + + return IsWordCodePoint(ucs); +} + +// An input box. The user can type text into it. +class InputBase : public ComponentBase, public InputOption { + public: + // NOLINTNEXTLINE + InputBase(InputOption option) : InputOption(std::move(option)) {} + + private: + // Component implementation: + Element OnRender() override { + const bool is_focused = Focused(); + const auto focused = (!is_focused && !hovered_) ? nothing + : insert() ? focusCursorBarBlinking + : focusCursorBlockBlinking; + + auto transform_func = + transform ? transform : InputOption::Default().transform; + + // placeholder. + if (content->empty()) { + auto element = text(placeholder()) | xflex | frame; + + return transform_func({ + std::move(element), hovered_, is_focused, + true // placeholder + }) | + focus | reflect(box_); + } + + Elements elements; + const std::vector<std::string> lines = Split(*content); + + cursor_position() = util::clamp(cursor_position(), 0, (int)content->size()); + + // Find the line and index of the cursor. + int cursor_line = 0; + int cursor_char_index = cursor_position(); + for (const auto& line : lines) { + if (cursor_char_index <= (int)line.size()) { + break; + } + + cursor_char_index -= static_cast<int>(line.size() + 1); + cursor_line++; + } + + if (lines.empty()) { + elements.push_back(text("") | focused); + } + + elements.reserve(lines.size()); + for (size_t i = 0; i < lines.size(); ++i) { + const std::string& line = lines[i]; + + // This is not the cursor line. + if (int(i) != cursor_line) { + elements.push_back(Text(line)); + continue; + } + + // The cursor is at the end of the line. + if (cursor_char_index >= (int)line.size()) { + elements.push_back(hbox({ + Text(line), + text(" ") | focused | reflect(cursor_box_), + }) | + xflex); + continue; + } + + // The cursor is on this line. + const int glyph_start = cursor_char_index; + const int glyph_end = static_cast<int>(GlyphNext(line, glyph_start)); + const std::string part_before_cursor = line.substr(0, glyph_start); + const std::string part_at_cursor = + line.substr(glyph_start, glyph_end - glyph_start); + const std::string part_after_cursor = line.substr(glyph_end); + auto element = hbox({ + Text(part_before_cursor), + Text(part_at_cursor) | focused | reflect(cursor_box_), + Text(part_after_cursor), + }) | + xflex; + elements.push_back(element); + } + + auto element = vbox(std::move(elements), cursor_line) | frame; + return transform_func({ + std::move(element), hovered_, is_focused, + false // placeholder + }) | + xflex | reflect(box_); + } + + Element Text(const std::string& input) { + if (!password()) { + return text(input); + } + + std::string out; + out.reserve(10 + input.size() * 3 / 2); + for (size_t i = 0; i < input.size(); ++i) { + out += "•"; + } + return text(out); + } + + bool HandleBackspace() { + if (cursor_position() == 0) { + return false; + } + const size_t start = GlyphPrevious(content(), cursor_position()); + const size_t end = cursor_position(); + content->erase(start, end - start); + cursor_position() = static_cast<int>(start); + on_change(); + return true; + } + + bool DeleteImpl() { + if (cursor_position() == (int)content->size()) { + return false; + } + const size_t start = cursor_position(); + const size_t end = GlyphNext(content(), cursor_position()); + content->erase(start, end - start); + return true; + } + + bool HandleDelete() { + if (DeleteImpl()) { + on_change(); + return true; + } + return false; + } + + bool HandleArrowLeft() { + if (cursor_position() == 0) { + return false; + } + + cursor_position() = + static_cast<int>(GlyphPrevious(content(), cursor_position())); + return true; + } + + bool HandleArrowRight() { + if (cursor_position() == (int)content->size()) { + return false; + } + + cursor_position() = + static_cast<int>(GlyphNext(content(), cursor_position())); + return true; + } + + size_t CursorColumn() { + size_t iter = cursor_position(); + int width = 0; + while (true) { + if (iter == 0) { + break; + } + iter = GlyphPrevious(content(), iter); + if (content()[iter] == '\n') { + break; + } + width += static_cast<int>(GlyphWidth(content(), iter)); + } + return width; + } + + // Move the cursor `columns` on the right, if possible. + void MoveCursorColumn(int columns) { + while (columns > 0) { + if (cursor_position() == (int)content().size() || + content()[cursor_position()] == '\n') { + return; + } + + columns -= static_cast<int>(GlyphWidth(content(), cursor_position())); + cursor_position() = + static_cast<int>(GlyphNext(content(), cursor_position())); + } + } + + bool HandleArrowUp() { + if (cursor_position() == 0) { + return false; + } + + const size_t columns = CursorColumn(); + + // Move cursor at the beginning of 2 lines above. + while (true) { + if (cursor_position() == 0) { + return true; + } + const size_t previous = GlyphPrevious(content(), cursor_position()); + if (content()[previous] == '\n') { + break; + } + cursor_position() = static_cast<int>(previous); + } + cursor_position() = + static_cast<int>(GlyphPrevious(content(), cursor_position())); + while (true) { + if (cursor_position() == 0) { + break; + } + const size_t previous = GlyphPrevious(content(), cursor_position()); + if (content()[previous] == '\n') { + break; + } + cursor_position() = static_cast<int>(previous); + } + + MoveCursorColumn(static_cast<int>(columns)); + return true; + } + + bool HandleArrowDown() { + if (cursor_position() == (int)content->size()) { + return false; + } + + const size_t columns = CursorColumn(); + + // Move cursor at the beginning of the next line + while (true) { + if (content()[cursor_position()] == '\n') { + break; + } + cursor_position() = + static_cast<int>(GlyphNext(content(), cursor_position())); + if (cursor_position() == (int)content().size()) { + return true; + } + } + cursor_position() = + static_cast<int>(GlyphNext(content(), cursor_position())); + + MoveCursorColumn(static_cast<int>(columns)); + return true; + } + + bool HandleHome() { + cursor_position() = 0; + return true; + } + + bool HandleEnd() { + cursor_position() = static_cast<int>(content->size()); + return true; + } + + bool HandleReturn() { + if (multiline()) { + HandleCharacter("\n"); + } + on_enter(); + return true; + } + + bool HandleCharacter(const std::string& character) { + if (!insert() && cursor_position() < (int)content->size() && + content()[cursor_position()] != '\n') { + DeleteImpl(); + } + content->insert(cursor_position(), character); + cursor_position() += static_cast<int>(character.size()); + on_change(); + return true; + } + + bool OnEvent(Event event) override { + cursor_position() = util::clamp(cursor_position(), 0, (int)content->size()); + + if (event == Event::Return) { + return HandleReturn(); + } + if (event.is_character()) { + return HandleCharacter(event.character()); + } + if (event.is_mouse()) { + return HandleMouse(event); + } + if (event == Event::Backspace) { + return HandleBackspace(); + } + if (event == Event::Delete) { + return HandleDelete(); + } + if (event == Event::ArrowLeft) { + return HandleArrowLeft(); + } + if (event == Event::ArrowRight) { + return HandleArrowRight(); + } + if (event == Event::ArrowUp) { + return HandleArrowUp(); + } + if (event == Event::ArrowDown) { + return HandleArrowDown(); + } + if (event == Event::Home) { + return HandleHome(); + } + if (event == Event::End) { + return HandleEnd(); + } + if (event == Event::ArrowLeftCtrl) { + return HandleLeftCtrl(); + } + if (event == Event::ArrowRightCtrl) { + return HandleRightCtrl(); + } + if (event == Event::Insert) { + return HandleInsert(); + } + return false; + } + + bool HandleLeftCtrl() { + if (cursor_position() == 0) { + return false; + } + + // Move left, as long as left it not a word. + while (cursor_position()) { + const size_t previous = GlyphPrevious(content(), cursor_position()); + if (IsWordCharacter(content(), previous)) { + break; + } + cursor_position() = static_cast<int>(previous); + } + // Move left, as long as left is a word character: + while (cursor_position()) { + const size_t previous = GlyphPrevious(content(), cursor_position()); + if (!IsWordCharacter(content(), previous)) { + break; + } + cursor_position() = static_cast<int>(previous); + } + return true; + } + + bool HandleRightCtrl() { + if (cursor_position() == (int)content().size()) { + return false; + } + + // Move right, until entering a word. + while (cursor_position() < (int)content().size()) { + cursor_position() = + static_cast<int>(GlyphNext(content(), cursor_position())); + if (IsWordCharacter(content(), cursor_position())) { + break; + } + } + // Move right, as long as right is a word character: + while (cursor_position() < (int)content().size()) { + const size_t next = GlyphNext(content(), cursor_position()); + if (!IsWordCharacter(content(), cursor_position())) { + break; + } + cursor_position() = static_cast<int>(next); + } + + return true; + } + + bool HandleMouse(Event event) { + hovered_ = box_.Contain(event.mouse().x, // + event.mouse().y) && + CaptureMouse(event); + if (!hovered_) { + return false; + } + + if (event.mouse().button != Mouse::Left) { + return false; + } + if (event.mouse().motion != Mouse::Pressed) { + return false; + } + + TakeFocus(); + + if (content->empty()) { + cursor_position() = 0; + return true; + } + + // Find the line and index of the cursor. + std::vector<std::string> lines = Split(*content); + int cursor_line = 0; + int cursor_char_index = cursor_position(); + for (const auto& line : lines) { + if (cursor_char_index <= (int)line.size()) { + break; + } + + cursor_char_index -= static_cast<int>(line.size() + 1); + cursor_line++; + } + const int cursor_column = + string_width(lines[cursor_line].substr(0, cursor_char_index)); + + int new_cursor_column = cursor_column + event.mouse().x - cursor_box_.x_min; + int new_cursor_line = cursor_line + event.mouse().y - cursor_box_.y_min; + + // Fix the new cursor position: + new_cursor_line = std::max(std::min(new_cursor_line, (int)lines.size()), 0); + + const std::string empty_string; + const std::string& line = new_cursor_line < (int)lines.size() + ? lines[new_cursor_line] + : empty_string; + new_cursor_column = util::clamp(new_cursor_column, 0, string_width(line)); + + if (new_cursor_column == cursor_column && // + new_cursor_line == cursor_line) { + return false; + } + + // Convert back the new_cursor_{line,column} toward cursor_position: + cursor_position() = 0; + for (int i = 0; i < new_cursor_line; ++i) { + cursor_position() += static_cast<int>(lines[i].size() + 1); + } + while (new_cursor_column > 0) { + new_cursor_column -= + static_cast<int>(GlyphWidth(content(), cursor_position())); + cursor_position() = + static_cast<int>(GlyphNext(content(), cursor_position())); + } + + on_change(); + return true; + } + + bool HandleInsert() { + insert() = !insert(); + return true; + } + + bool Focusable() const final { return true; } + + bool hovered_ = false; + + Box box_; + Box cursor_box_; +}; + +} // namespace + +/// @brief An input box for editing text. +/// @param option Additional optional parameters. +/// @ingroup component +/// @see InputBase +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::FitComponent(); +/// std::string content= ""; +/// std::string placeholder = "placeholder"; +/// Component input = Input({ +/// .content = &content, +/// .placeholder = &placeholder, +/// }) +/// screen.Loop(input); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// placeholder +/// ``` +Component Input(InputOption option) { + return Make<InputBase>(std::move(option)); +} + +/// @brief An input box for editing text. +/// @param content The editable content. +/// @param option Additional optional parameters. +/// @ingroup component +/// @see InputBase +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::FitComponent(); +/// std::string content= ""; +/// std::string placeholder = "placeholder"; +/// Component input = Input(content, { +/// .placeholder = &placeholder, +/// .password = true, +/// }) +/// screen.Loop(input); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// placeholder +/// ``` +Component Input(StringRef content, InputOption option) { + option.content = std::move(content); + return Make<InputBase>(std::move(option)); +} + +/// @brief An input box for editing text. +/// @param content The editable content. +/// @param option Additional optional parameters. +/// @ingroup component +/// @see InputBase +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::FitComponent(); +/// std::string content= ""; +/// std::string placeholder = "placeholder"; +/// Component input = Input(content, placeholder); +/// screen.Loop(input); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// placeholder +/// ``` +Component Input(StringRef content, StringRef placeholder, InputOption option) { + option.content = std::move(content); + option.placeholder = std::move(placeholder); + return Make<InputBase>(std::move(option)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/loop.cpp b/contrib/libs/ftxui/src/ftxui/component/loop.cpp new file mode 100644 index 00000000000..d16c67b4d68 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/loop.cpp @@ -0,0 +1,57 @@ +// Copyright 2022 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include "ftxui/component/loop.hpp" + +#include <utility> // for move + +#include "ftxui/component/screen_interactive.hpp" // for ScreenInteractive, Component + +namespace ftxui { + +/// @brief A Loop is a wrapper around a Component and a ScreenInteractive. +/// It is used to run a Component in a terminal. +/// @ingroup component +/// @see Component, ScreenInteractive. +/// @see ScreenInteractive::Loop(). +/// @see ScreenInteractive::ExitLoop(). +/// @param[in] screen The screen to use. +/// @param[in] component The component to run. +// NOLINTNEXTLINE +Loop::Loop(ScreenInteractive* screen, Component component) + : screen_(screen), component_(std::move(component)) { + screen_->PreMain(); +} + +Loop::~Loop() { + screen_->PostMain(); +} + +/// @brief Whether the loop has quitted. +/// @ingroup component +bool Loop::HasQuitted() { + return screen_->HasQuitted(); +} + +/// @brief Execute the loop. Make the `component` to process every pending +/// tasks/events. A new frame might be drawn if the previous was invalidated. +/// Return true until the loop hasn't completed. +void Loop::RunOnce() { + screen_->RunOnce(component_); +} + +/// @brief Wait for at least one event to be handled and execute +/// `Loop::RunOnce()`. +void Loop::RunOnceBlocking() { + screen_->RunOnceBlocking(component_); +} + +/// Execute the loop, blocking the current thread, up until the loop has +/// quitted. +void Loop::Run() { + while (!HasQuitted()) { + RunOnceBlocking(); + } +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/maybe.cpp b/contrib/libs/ftxui/src/ftxui/component/maybe.cpp new file mode 100644 index 00000000000..2dd462e700d --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/maybe.cpp @@ -0,0 +1,92 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <functional> // for function +#include <memory> // for make_unique, __shared_ptr_access, __shared_ptr_access<>::element_type, shared_ptr +#include <utility> // for move + +#include "ftxui/component/component.hpp" // for ComponentDecorator, Maybe, Make +#include "ftxui/component/component_base.hpp" // for Component, ComponentBase +#include "ftxui/component/event.hpp" // for Event +#include "ftxui/dom/elements.hpp" // for Element +#include "ftxui/dom/node.hpp" // for Node + +namespace ftxui { + +/// @brief Decorate a component |child|. It is shown only when |show| returns +/// true. +/// @param child the compoenent to decorate. +/// @param show a function returning whether |child| should shown. +/// @ingroup component +Component Maybe(Component child, std::function<bool()> show) { + class Impl : public ComponentBase { + public: + explicit Impl(std::function<bool()> show) : show_(std::move(show)) {} + + private: + Element OnRender() override { + return show_() ? ComponentBase::OnRender() : std::make_unique<Node>(); + } + bool Focusable() const override { + return show_() && ComponentBase::Focusable(); + } + bool OnEvent(Event event) override { + return show_() && ComponentBase::OnEvent(event); + } + + std::function<bool()> show_; + }; + + auto maybe = Make<Impl>(std::move(show)); + maybe->Add(std::move(child)); + return maybe; +} + +/// @brief Decorate a component. It is shown only when the |show| function +/// returns true. +/// @param show a function returning whether the decorated component should +/// be shown. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto component = Renderer([]{ return text("Hello World!"); }); +/// auto maybe_component = component | Maybe([&]{ return counter == 42; }); +/// ``` +ComponentDecorator Maybe(std::function<bool()> show) { + return [show = std::move(show)](Component child) mutable { + return Maybe(std::move(child), std::move(show)); + }; +} + +/// @brief Decorate a component |child|. It is shown only when |show| is true. +/// @param child the compoennt to decorate. +/// @param show a boolean. |child| is shown when |show| is true. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto component = Renderer([]{ return text("Hello World!"); }); +/// auto maybe_component = Maybe(component, &show); +/// ``` +Component Maybe(Component child, const bool* show) { + return Maybe(std::move(child), [show] { return *show; }); +} + +/// @brief Decorate a component. It is shown only when |show| is true. +/// @param show a boolean. |child| is shown when |show| is true. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto component = Renderer([]{ return text("Hello World!"); }); +/// auto maybe_component = component | Maybe(&show); +/// ``` +ComponentDecorator Maybe(const bool* show) { + return [show](Component child) { return Maybe(std::move(child), show); }; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/menu.cpp b/contrib/libs/ftxui/src/ftxui/component/menu.cpp new file mode 100644 index 00000000000..ae9d2483aa8 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/menu.cpp @@ -0,0 +1,711 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for max, fill_n, reverse +#include <chrono> // for milliseconds +#include <ftxui/dom/direction.hpp> // for Direction, Direction::Down, Direction::Left, Direction::Right, Direction::Up +#include <functional> // for function +#include <string> // for operator+, string +#include <utility> // for move +#include <vector> // for vector, __alloc_traits<>::value_type + +#include "ftxui/component/animation.hpp" // for Animator, Linear +#include "ftxui/component/component.hpp" // for Make, Menu, MenuEntry, Toggle +#include "ftxui/component/component_base.hpp" // for ComponentBase +#include "ftxui/component/component_options.hpp" // for MenuOption, MenuEntryOption, UnderlineOption, AnimatedColorOption, AnimatedColorsOption, EntryState +#include "ftxui/component/event.hpp" // for Event, Event::ArrowDown, Event::ArrowLeft, Event::ArrowRight, Event::ArrowUp, Event::End, Event::Home, Event::PageDown, Event::PageUp, Event::Return, Event::Tab, Event::TabReverse +#include "ftxui/component/mouse.hpp" // for Mouse, Mouse::Left, Mouse::Released, Mouse::WheelDown, Mouse::WheelUp, Mouse::None +#include "ftxui/component/screen_interactive.hpp" // for Component +#include "ftxui/dom/elements.hpp" // for operator|, Element, reflect, Decorator, nothing, Elements, bgcolor, color, hbox, separatorHSelector, separatorVSelector, vbox, xflex, yflex, text, bold, focus, inverted, select +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/color.hpp" // for Color +#include "ftxui/screen/util.hpp" // for clamp +#include "ftxui/util/ref.hpp" // for Ref, ConstStringListRef, ConstStringRef + +namespace ftxui { + +namespace { + +Element DefaultOptionTransform(const EntryState& state) { + std::string label = (state.active ? "> " : " ") + state.label; // NOLINT + Element e = text(std::move(label)); + if (state.focused) { + e = e | inverted; + } + if (state.active) { + e = e | bold; + } + return e; +} + +bool IsInverted(Direction direction) { + switch (direction) { + case Direction::Up: + case Direction::Left: + return true; + case Direction::Down: + case Direction::Right: + return false; + } + return false; // NOT_REACHED() +} + +bool IsHorizontal(Direction direction) { + switch (direction) { + case Direction::Left: + case Direction::Right: + return true; + case Direction::Down: + case Direction::Up: + return false; + } + return false; // NOT_REACHED() +} + +} // namespace + +/// @brief A list of items. The user can navigate through them. +/// @ingroup component +class MenuBase : public ComponentBase, public MenuOption { + public: + explicit MenuBase(const MenuOption& option) : MenuOption(option) {} + + bool IsHorizontal() { return ftxui::IsHorizontal(direction); } + void OnChange() { + if (on_change) { + on_change(); + } + } + + void OnEnter() { + if (on_enter) { + on_enter(); + } + } + + void Clamp() { + if (selected() != selected_previous_) { + SelectedTakeFocus(); + } + boxes_.resize(size()); + selected() = util::clamp(selected(), 0, size() - 1); + selected_previous_ = util::clamp(selected_previous_, 0, size() - 1); + selected_focus_ = util::clamp(selected_focus_, 0, size() - 1); + focused_entry() = util::clamp(focused_entry(), 0, size() - 1); + } + + void OnAnimation(animation::Params& params) override { + animator_first_.OnAnimation(params); + animator_second_.OnAnimation(params); + for (auto& animator : animator_background_) { + animator.OnAnimation(params); + } + for (auto& animator : animator_foreground_) { + animator.OnAnimation(params); + } + } + + Element OnRender() override { + Clamp(); + UpdateAnimationTarget(); + + Elements elements; + const bool is_menu_focused = Focused(); + if (elements_prefix) { + elements.push_back(elements_prefix()); + } + elements.reserve(size()); + for (int i = 0; i < size(); ++i) { + if (i != 0 && elements_infix) { + elements.push_back(elements_infix()); + } + const bool is_focused = (focused_entry() == i) && is_menu_focused; + const bool is_selected = (selected() == i); + + const EntryState state = { + entries[i], false, is_selected, is_focused, i, + }; + + Element element = (entries_option.transform ? entries_option.transform + : DefaultOptionTransform) // + (state); + if (selected_focus_ == i) { + element |= focus; + } + element |= AnimatedColorStyle(i); + element |= reflect(boxes_[i]); + elements.push_back(element); + } + if (elements_postfix) { + elements.push_back(elements_postfix()); + } + + if (IsInverted(direction)) { + std::reverse(elements.begin(), elements.end()); + } + + const Element bar = IsHorizontal() + ? hbox(std::move(elements), selected_focus_) + : vbox(std::move(elements), selected_focus_); + + if (!underline.enabled) { + return bar | reflect(box_); + } + + if (IsHorizontal()) { + return vbox({ + bar | xflex, + separatorHSelector(first_, second_, // + underline.color_active, + underline.color_inactive), + }) | + reflect(box_); + } else { + return hbox({ + separatorVSelector(first_, second_, // + underline.color_active, + underline.color_inactive), + bar | yflex, + }) | + reflect(box_); + } + } + + void SelectedTakeFocus() { + selected_previous_ = selected(); + selected_focus_ = selected(); + } + + void OnUp() { + switch (direction) { + case Direction::Up: + selected()++; + break; + case Direction::Down: + selected()--; + break; + case Direction::Left: + case Direction::Right: + break; + } + } + + void OnDown() { + switch (direction) { + case Direction::Up: + selected()--; + break; + case Direction::Down: + selected()++; + break; + case Direction::Left: + case Direction::Right: + break; + } + } + + void OnLeft() { + switch (direction) { + case Direction::Left: + selected()++; + break; + case Direction::Right: + selected()--; + break; + case Direction::Down: + case Direction::Up: + break; + } + } + + void OnRight() { + switch (direction) { + case Direction::Left: + selected()--; + break; + case Direction::Right: + selected()++; + break; + case Direction::Down: + case Direction::Up: + break; + } + } + + // NOLINTNEXTLINE(readability-function-cognitive-complexity) + bool OnEvent(Event event) override { + Clamp(); + if (!CaptureMouse(event)) { + return false; + } + + if (event.is_mouse()) { + return OnMouseEvent(event); + } + + if (Focused()) { + const int old_selected = selected(); + if (event == Event::ArrowUp || event == Event::Character('k')) { + OnUp(); + } + if (event == Event::ArrowDown || event == Event::Character('j')) { + OnDown(); + } + if (event == Event::ArrowLeft || event == Event::Character('h')) { + OnLeft(); + } + if (event == Event::ArrowRight || event == Event::Character('l')) { + OnRight(); + } + if (event == Event::PageUp) { + selected() -= box_.y_max - box_.y_min; + } + if (event == Event::PageDown) { + selected() += box_.y_max - box_.y_min; + } + if (event == Event::Home) { + selected() = 0; + } + if (event == Event::End) { + selected() = size() - 1; + } + if (event == Event::Tab && size()) { + selected() = (selected() + 1) % size(); + } + if (event == Event::TabReverse && size()) { + selected() = (selected() + size() - 1) % size(); + } + + selected() = util::clamp(selected(), 0, size() - 1); + + if (selected() != old_selected) { + focused_entry() = selected(); + SelectedTakeFocus(); + OnChange(); + return true; + } + } + + if (event == Event::Return) { + OnEnter(); + return true; + } + + return false; + } + + bool OnMouseEvent(Event event) { + if (event.mouse().button == Mouse::WheelDown || + event.mouse().button == Mouse::WheelUp) { + return OnMouseWheel(event); + } + + if (event.mouse().button != Mouse::None && + event.mouse().button != Mouse::Left) { + return false; + } + if (!CaptureMouse(event)) { + return false; + } + for (int i = 0; i < size(); ++i) { + if (!boxes_[i].Contain(event.mouse().x, event.mouse().y)) { + continue; + } + + TakeFocus(); + focused_entry() = i; + + if (event.mouse().button == Mouse::Left && + event.mouse().motion == Mouse::Pressed) { + if (selected() != i) { + selected() = i; + selected_previous_ = selected(); + OnChange(); + } + return true; + } + } + return false; + } + + bool OnMouseWheel(Event event) { + if (!box_.Contain(event.mouse().x, event.mouse().y)) { + return false; + } + const int old_selected = selected(); + + if (event.mouse().button == Mouse::WheelUp) { + selected()--; + } + if (event.mouse().button == Mouse::WheelDown) { + selected()++; + } + + selected() = util::clamp(selected(), 0, size() - 1); + + if (selected() != old_selected) { + SelectedTakeFocus(); + OnChange(); + } + return true; + } + + void UpdateAnimationTarget() { + UpdateColorTarget(); + UpdateUnderlineTarget(); + } + + void UpdateColorTarget() { + if (size() != int(animation_background_.size())) { + animation_background_.resize(size()); + animation_foreground_.resize(size()); + animator_background_.clear(); + animator_foreground_.clear(); + + const int len = size(); + animator_background_.reserve(len); + animator_foreground_.reserve(len); + for (int i = 0; i < len; ++i) { + animation_background_[i] = 0.F; + animation_foreground_[i] = 0.F; + animator_background_.emplace_back(&animation_background_[i], 0.F, + std::chrono::milliseconds(0), + animation::easing::Linear); + animator_foreground_.emplace_back(&animation_foreground_[i], 0.F, + std::chrono::milliseconds(0), + animation::easing::Linear); + } + } + + const bool is_menu_focused = Focused(); + for (int i = 0; i < size(); ++i) { + const bool is_focused = (focused_entry() == i) && is_menu_focused; + const bool is_selected = (selected() == i); + float target = is_selected ? 1.F : is_focused ? 0.5F : 0.F; // NOLINT + if (animator_background_[i].to() != target) { + animator_background_[i] = animation::Animator( + &animation_background_[i], target, + entries_option.animated_colors.background.duration, + entries_option.animated_colors.background.function); + animator_foreground_[i] = animation::Animator( + &animation_foreground_[i], target, + entries_option.animated_colors.foreground.duration, + entries_option.animated_colors.foreground.function); + } + } + } + + Decorator AnimatedColorStyle(int i) { + Decorator style = nothing; + if (entries_option.animated_colors.foreground.enabled) { + style = style | color(Color::Interpolate( + animation_foreground_[i], + entries_option.animated_colors.foreground.inactive, + entries_option.animated_colors.foreground.active)); + } + + if (entries_option.animated_colors.background.enabled) { + style = style | bgcolor(Color::Interpolate( + animation_background_[i], + entries_option.animated_colors.background.inactive, + entries_option.animated_colors.background.active)); + } + return style; + } + + void UpdateUnderlineTarget() { + if (!underline.enabled) { + return; + } + + if (FirstTarget() == animator_first_.to() && + SecondTarget() == animator_second_.to()) { + return; + } + + if (FirstTarget() >= animator_first_.to()) { + animator_first_ = animation::Animator( + &first_, FirstTarget(), underline.follower_duration, + underline.follower_function, underline.follower_delay); + + animator_second_ = animation::Animator( + &second_, SecondTarget(), underline.leader_duration, + underline.leader_function, underline.leader_delay); + } else { + animator_first_ = animation::Animator( + &first_, FirstTarget(), underline.leader_duration, + underline.leader_function, underline.leader_delay); + + animator_second_ = animation::Animator( + &second_, SecondTarget(), underline.follower_duration, + underline.follower_function, underline.follower_delay); + } + } + + bool Focusable() const final { return entries.size(); } + int size() const { return int(entries.size()); } + float FirstTarget() { + if (boxes_.empty()) { + return 0.F; + } + const int value = IsHorizontal() ? boxes_[selected()].x_min - box_.x_min + : boxes_[selected()].y_min - box_.y_min; + return float(value); + } + float SecondTarget() { + if (boxes_.empty()) { + return 0.F; + } + const int value = IsHorizontal() ? boxes_[selected()].x_max - box_.x_min + : boxes_[selected()].y_max - box_.y_min; + return float(value); + } + + protected: + int selected_previous_ = selected(); + int selected_focus_ = selected(); + + // Mouse click support: + std::vector<Box> boxes_; + Box box_; + + // Animation support: + float first_ = 0.F; + float second_ = 0.F; + animation::Animator animator_first_ = animation::Animator(&first_, 0.F); + animation::Animator animator_second_ = animation::Animator(&second_, 0.F); + std::vector<animation::Animator> animator_background_; + std::vector<animation::Animator> animator_foreground_; + std::vector<float> animation_background_; + std::vector<float> animation_foreground_; +}; + +/// @brief A list of text. The focused element is selected. +/// @param option a structure containing all the paramters. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// std::vector<std::string> entries = { +/// "entry 1", +/// "entry 2", +/// "entry 3", +/// }; +/// int selected = 0; +/// auto menu = Menu({ +/// .entries = &entries, +/// .selected = &selected, +/// }); +/// screen.Loop(menu); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// > entry 1 +/// entry 2 +/// entry 3 +/// ``` +// NOLINTNEXTLINE +Component Menu(MenuOption option) { + return Make<MenuBase>(std::move(option)); +} + +/// @brief A list of text. The focused element is selected. +/// @param entries The list of entries in the menu. +/// @param selected The index of the currently selected element. +/// @param option Additional optional parameters. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// std::vector<std::string> entries = { +/// "entry 1", +/// "entry 2", +/// "entry 3", +/// }; +/// int selected = 0; +/// auto menu = Menu(&entries, &selected); +/// screen.Loop(menu); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// > entry 1 +/// entry 2 +/// entry 3 +/// ``` +Component Menu(ConstStringListRef entries, int* selected, MenuOption option) { + option.entries = std::move(entries); + option.selected = selected; + return Menu(option); +} + +/// @brief An horizontal list of elements. The user can navigate through them. +/// @param entries The list of selectable entries to display. +/// @param selected Reference the selected entry. +/// See also |Menu|. +/// @ingroup component +Component Toggle(ConstStringListRef entries, int* selected) { + return Menu(std::move(entries), selected, MenuOption::Toggle()); +} + +/// @brief A specific menu entry. They can be put into a Container::Vertical to +/// form a menu. +/// @param label The text drawn representing this element. +/// @param option Additional optional parameters. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// int selected = 0; +/// auto menu = Container::Vertical({ +/// MenuEntry("entry 1"), +/// MenuEntry("entry 2"), +/// MenuEntry("entry 3"), +/// }, &selected); +/// screen.Loop(menu); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// > entry 1 +/// entry 2 +/// entry 3 +/// ``` +Component MenuEntry(ConstStringRef label, MenuEntryOption option) { + option.label = std::move(label); + return MenuEntry(std::move(option)); +} + +/// @brief A specific menu entry. They can be put into a Container::Vertical to +/// form a menu. +/// @param option The parameters. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// int selected = 0; +/// auto menu = Container::Vertical({ +/// MenuEntry({.label = "entry 1"}), +/// MenuEntry({.label = "entry 2"}), +/// MenuEntry({.label = "entry 3"}), +/// }, &selected); +/// screen.Loop(menu); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// > entry 1 +/// entry 2 +/// entry 3 +/// ``` +Component MenuEntry(MenuEntryOption option) { + class Impl : public ComponentBase, public MenuEntryOption { + public: + explicit Impl(MenuEntryOption option) + : MenuEntryOption(std::move(option)) {} + + private: + Element OnRender() override { + const bool is_focused = Focused(); + UpdateAnimationTarget(); + + const EntryState state{ + label(), false, hovered_, is_focused, Index(), + }; + + Element element = (transform ? transform : DefaultOptionTransform) // + (state); + + if (is_focused) { + element |= focus; + } + + return element | AnimatedColorStyle() | reflect(box_); + } + + void UpdateAnimationTarget() { + const bool focused = Focused(); + float target = focused ? 1.F : hovered_ ? 0.5F : 0.F; // NOLINT + if (target == animator_background_.to()) { + return; + } + animator_background_ = animation::Animator( + &animation_background_, target, animated_colors.background.duration, + animated_colors.background.function); + animator_foreground_ = animation::Animator( + &animation_foreground_, target, animated_colors.foreground.duration, + animated_colors.foreground.function); + } + + Decorator AnimatedColorStyle() { + Decorator style = nothing; + if (animated_colors.foreground.enabled) { + style = style | + color(Color::Interpolate(animation_foreground_, + animated_colors.foreground.inactive, + animated_colors.foreground.active)); + } + + if (animated_colors.background.enabled) { + style = style | + bgcolor(Color::Interpolate(animation_background_, + animated_colors.background.inactive, + animated_colors.background.active)); + } + return style; + } + + bool Focusable() const override { return true; } + bool OnEvent(Event event) override { + if (!event.is_mouse()) { + return false; + } + + hovered_ = box_.Contain(event.mouse().x, event.mouse().y); + + if (!hovered_) { + return false; + } + + if (event.mouse().button == Mouse::Left && + event.mouse().motion == Mouse::Pressed) { + TakeFocus(); + return true; + } + + return false; + } + + void OnAnimation(animation::Params& params) override { + animator_background_.OnAnimation(params); + animator_foreground_.OnAnimation(params); + } + + Box box_; + bool hovered_ = false; + + float animation_background_ = 0.F; + float animation_foreground_ = 0.F; + animation::Animator animator_background_ = + animation::Animator(&animation_background_, 0.F); + animation::Animator animator_foreground_ = + animation::Animator(&animation_foreground_, 0.F); + }; + + return Make<Impl>(std::move(option)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/modal.cpp b/contrib/libs/ftxui/src/ftxui/component/modal.cpp new file mode 100644 index 00000000000..213e9e56ad4 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/modal.cpp @@ -0,0 +1,64 @@ +// Copyright 2022 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <ftxui/component/event.hpp> // for Event +#include <ftxui/dom/elements.hpp> // for operator|, Element, center, clear_under, dbox +#include <memory> // for __shared_ptr_access, shared_ptr +#include <utility> // for move + +#include "ftxui/component/component.hpp" // for Make, Tab, ComponentDecorator, Modal +#include "ftxui/component/component_base.hpp" // for Component, ComponentBase + +namespace ftxui { + +// Add a |modal| window on top of the |main| component. It is shown one on the +// top of the other when |show_modal| is true. +/// @ingroup component +// NOLINTNEXTLINE +Component Modal(Component main, Component modal, const bool* show_modal) { + class Impl : public ComponentBase { + public: + explicit Impl(Component main, Component modal, const bool* show_modal) + : main_(std::move(main)), + modal_(std::move(modal)), + show_modal_(show_modal) { + Add(Container::Tab({main_, modal_}, &selector_)); + } + + private: + Element OnRender() override { + selector_ = *show_modal_; + auto document = main_->Render(); + if (*show_modal_) { + document = dbox({ + document, + modal_->Render() | clear_under | center, + }); + } + return document; + } + + bool OnEvent(Event event) override { + selector_ = *show_modal_; + return ComponentBase::OnEvent(event); + } + + Component main_; + Component modal_; + const bool* show_modal_; + int selector_ = *show_modal_; + }; + return Make<Impl>(main, modal, show_modal); +} + +// Decorate a component. Add a |modal| window on top of it. It is shown one on +// the top of the other when |show_modal| is true. +/// @ingroup component +// NOLINTNEXTLINE +ComponentDecorator Modal(Component modal, const bool* show_modal) { + return [modal, show_modal](Component main) { + return Modal(std::move(main), modal, show_modal); + }; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/radiobox.cpp b/contrib/libs/ftxui/src/ftxui/component/radiobox.cpp new file mode 100644 index 00000000000..f1b1d477b50 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/radiobox.cpp @@ -0,0 +1,244 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <functional> // for function +#include <utility> // for move +#include <vector> // for vector + +#include "ftxui/component/component.hpp" // for Make, Radiobox +#include "ftxui/component/component_base.hpp" // for ComponentBase +#include "ftxui/component/component_options.hpp" // for RadioboxOption, EntryState +#include "ftxui/component/event.hpp" // for Event, Event::ArrowDown, Event::ArrowUp, Event::End, Event::Home, Event::PageDown, Event::PageUp, Event::Return, Event::Tab, Event::TabReverse +#include "ftxui/component/mouse.hpp" // for Mouse, Mouse::WheelDown, Mouse::WheelUp, Mouse::Left, Mouse::Released +#include "ftxui/component/screen_interactive.hpp" // for Component +#include "ftxui/dom/elements.hpp" // for operator|, reflect, Element, vbox, Elements, focus, nothing, select +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/util.hpp" // for clamp +#include "ftxui/util/ref.hpp" // for Ref, ConstStringListRef + +namespace ftxui { + +namespace { +/// @brief A list of selectable element. One and only one can be selected at +/// the same time. +/// @ingroup component +class RadioboxBase : public ComponentBase, public RadioboxOption { + public: + explicit RadioboxBase(const RadioboxOption& option) + : RadioboxOption(option) {} + + private: + Element OnRender() override { + Clamp(); + Elements elements; + const bool is_menu_focused = Focused(); + elements.reserve(size()); + for (int i = 0; i < size(); ++i) { + const bool is_focused = (focused_entry() == i) && is_menu_focused; + const bool is_selected = (hovered_ == i); + auto state = EntryState{ + entries[i], selected() == i, is_selected, is_focused, i, + }; + auto element = + (transform ? transform : RadioboxOption::Simple().transform)(state); + if (is_selected) { + element |= focus; + } + elements.push_back(element | reflect(boxes_[i])); + } + return vbox(std::move(elements), hovered_) | reflect(box_); + } + + // NOLINTNEXTLINE(readability-function-cognitive-complexity) + bool OnEvent(Event event) override { + Clamp(); + if (!CaptureMouse(event)) { + return false; + } + + if (event.is_mouse()) { + return OnMouseEvent(event); + } + + if (Focused()) { + const int old_hovered = hovered_; + if (event == Event::ArrowUp || event == Event::Character('k')) { + (hovered_)--; + } + if (event == Event::ArrowDown || event == Event::Character('j')) { + (hovered_)++; + } + if (event == Event::PageUp) { + (hovered_) -= box_.y_max - box_.y_min; + } + if (event == Event::PageDown) { + (hovered_) += box_.y_max - box_.y_min; + } + if (event == Event::Home) { + (hovered_) = 0; + } + if (event == Event::End) { + (hovered_) = size() - 1; + } + if (event == Event::Tab && size()) { + hovered_ = (hovered_ + 1) % size(); + } + if (event == Event::TabReverse && size()) { + hovered_ = (hovered_ + size() - 1) % size(); + } + + hovered_ = util::clamp(hovered_, 0, size() - 1); + + if (hovered_ != old_hovered) { + focused_entry() = hovered_; + on_change(); + return true; + } + } + + if (event == Event::Character(' ') || event == Event::Return) { + selected() = hovered_; + on_change(); + return true; + } + + return false; + } + + bool OnMouseEvent(Event event) { + if (event.mouse().button == Mouse::WheelDown || + event.mouse().button == Mouse::WheelUp) { + return OnMouseWheel(event); + } + + for (int i = 0; i < size(); ++i) { + if (!boxes_[i].Contain(event.mouse().x, event.mouse().y)) { + continue; + } + + TakeFocus(); + focused_entry() = i; + if (event.mouse().button == Mouse::Left && + event.mouse().motion == Mouse::Pressed) { + if (selected() != i) { + selected() = i; + on_change(); + } + + return true; + } + } + return false; + } + + bool OnMouseWheel(Event event) { + if (!box_.Contain(event.mouse().x, event.mouse().y)) { + return false; + } + + const int old_hovered = hovered_; + + if (event.mouse().button == Mouse::WheelUp) { + (hovered_)--; + } + if (event.mouse().button == Mouse::WheelDown) { + (hovered_)++; + } + + hovered_ = util::clamp(hovered_, 0, size() - 1); + + if (hovered_ != old_hovered) { + on_change(); + } + + return true; + } + + void Clamp() { + boxes_.resize(size()); + selected() = util::clamp(selected(), 0, size() - 1); + focused_entry() = util::clamp(focused_entry(), 0, size() - 1); + hovered_ = util::clamp(hovered_, 0, size() - 1); + } + + bool Focusable() const final { return entries.size(); } + int size() const { return int(entries.size()); } + + int hovered_ = selected(); + std::vector<Box> boxes_; + Box box_; +}; + +} // namespace + +/// @brief A list of element, where only one can be selected. +/// @param option The parameters +/// @ingroup component +/// @see RadioboxBase +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// std::vector<std::string> entries = { +/// "entry 1", +/// "entry 2", +/// "entry 3", +/// }; +/// int selected = 0; +/// auto menu = Radiobox({ +/// .entries = entries, +/// .selected = &selected, +/// }); +/// screen.Loop(menu); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// ◉ entry 1 +/// ○ entry 2 +/// ○ entry 3 +/// ``` +/// NOLINTNEXTLINE +Component Radiobox(RadioboxOption option) { + return Make<RadioboxBase>(std::move(option)); +} + +/// @brief A list of element, where only one can be selected. +/// @param entries The list of entries in the list. +/// @param selected The index of the currently selected element. +/// @param option Additional optional parameters. +/// @ingroup component +/// @see RadioboxBase +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// std::vector<std::string> entries = { +/// "entry 1", +/// "entry 2", +/// "entry 3", +/// }; +/// int selected = 0; +/// auto menu = Radiobox(&entries, &selected); +/// screen.Loop(menu); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// ◉ entry 1 +/// ○ entry 2 +/// ○ entry 3 +/// ``` +Component Radiobox(ConstStringListRef entries, + int* selected, + RadioboxOption option) { + option.entries = std::move(entries); + option.selected = selected; + return Make<RadioboxBase>(std::move(option)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/renderer.cpp b/contrib/libs/ftxui/src/ftxui/component/renderer.cpp new file mode 100644 index 00000000000..8ca47839933 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/renderer.cpp @@ -0,0 +1,133 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <functional> // for function +#include <utility> // for move + +#include "ftxui/component/component.hpp" // for Make, Renderer +#include "ftxui/component/component_base.hpp" // for Component, ComponentBase +#include "ftxui/component/event.hpp" // for Event +#include "ftxui/component/mouse.hpp" // for Mouse +#include "ftxui/dom/elements.hpp" // for Element, operator|, reflect +#include "ftxui/screen/box.hpp" // for Box + +namespace ftxui { + +/// @brief Return a component, using |render| to render its interface. +/// @param render The function drawing the interface. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// auto renderer = Renderer([] { +/// return text("My interface"); +/// }); +/// screen.Loop(renderer); +/// ``` +Component Renderer(std::function<Element()> render) { + class Impl : public ComponentBase { + public: + explicit Impl(std::function<Element()> render) + : render_(std::move(render)) {} + Element OnRender() override { return render_(); } + std::function<Element()> render_; + }; + + return Make<Impl>(std::move(render)); +} + +/// @brief Return a new Component, similar to |child|, but using |render| as the +/// Component::Render() event. +/// @param child The component to forward events to. +/// @param render The function drawing the interface. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// std::string label = "Click to quit"; +/// auto button = Button(&label, screen.ExitLoopClosure()); +/// auto renderer = Renderer(button, [&] { +/// return hbox({ +/// text("A button:"), +/// button->Render(), +/// }); +/// }); +/// screen.Loop(renderer); +/// ``` +Component Renderer(Component child, std::function<Element()> render) { + Component renderer = Renderer(std::move(render)); + renderer->Add(std::move(child)); + return renderer; +} + +/// @brief Return a focusable component, using |render| to render its interface. +/// @param render The function drawing the interface, taking a boolean telling +/// whether the component is focused or not. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// auto renderer = Renderer([] (bool focused) { +/// if (focused) +/// return text("My interface") | inverted; +/// else +/// return text("My interface"); +/// }); +/// screen.Loop(renderer); +/// ``` +Component Renderer(std::function<Element(bool)> render) { + class Impl : public ComponentBase { + public: + explicit Impl(std::function<Element(bool)> render) + : render_(std::move(render)) {} + + private: + Element OnRender() override { return render_(Focused()) | reflect(box_); } + bool Focusable() const override { return true; } + bool OnEvent(Event event) override { + if (event.is_mouse() && box_.Contain(event.mouse().x, event.mouse().y)) { + if (!CaptureMouse(event)) { + return false; + } + + TakeFocus(); + } + + return false; + } + Box box_; + + std::function<Element(bool)> render_; + }; + return Make<Impl>(std::move(render)); +} + +/// @brief Decorate a component, by decorating what it renders. +/// @param decorator the function modifying the element it renders. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// auto renderer = +// Renderer([] { return text("Hello");) +/// | Renderer(bold) +/// | Renderer(inverted); +/// screen.Loop(renderer); +/// ``` +ComponentDecorator Renderer(ElementDecorator decorator) { // NOLINT + return [decorator](Component component) { // NOLINT + return Renderer(component, [component, decorator] { + return component->Render() | decorator; + }); + }; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/resizable_split.cpp b/contrib/libs/ftxui/src/ftxui/component/resizable_split.cpp new file mode 100644 index 00000000000..39848f65b98 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/resizable_split.cpp @@ -0,0 +1,326 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <ftxui/component/component_options.hpp> // for ResizableSplitOption +#include <ftxui/dom/direction.hpp> // for Direction, Direction::Down, Direction::Left, Direction::Right, Direction::Up +#include <ftxui/util/ref.hpp> // for Ref +#include <functional> // for function +#include <utility> // for move + +#include "ftxui/component/captured_mouse.hpp" // for CapturedMouse +#include "ftxui/component/component.hpp" // for Horizontal, Make, ResizableSplit, ResizableSplitBottom, ResizableSplitLeft, ResizableSplitRight, ResizableSplitTop +#include "ftxui/component/component_base.hpp" // for Component, ComponentBase +#include "ftxui/component/event.hpp" // for Event +#include "ftxui/component/mouse.hpp" // for Mouse, Mouse::Left, Mouse::Pressed, Mouse::Released +#include "ftxui/dom/elements.hpp" // for operator|, reflect, Element, size, EQUAL, xflex, yflex, hbox, vbox, HEIGHT, WIDTH, text +#include "ftxui/screen/box.hpp" // for Box + +namespace ftxui { +namespace { + +class ResizableSplitBase : public ComponentBase { + public: + explicit ResizableSplitBase(ResizableSplitOption options) + : options_(std::move(options)) { + switch (options_->direction()) { + case Direction::Left: + Add(Container::Horizontal({ + options_->main, + options_->back, + })); + break; + case Direction::Right: + Add(Container::Horizontal({ + options_->back, + options_->main, + })); + break; + case Direction::Up: + Add(Container::Vertical({ + options_->main, + options_->back, + })); + break; + case Direction::Down: + Add(Container::Vertical({ + options_->back, + options_->main, + })); + break; + } + } + + bool OnEvent(Event event) final { + if (event.is_mouse()) { + return OnMouseEvent(std::move(event)); + } + return ComponentBase::OnEvent(std::move(event)); + } + + bool OnMouseEvent(Event event) { + if (captured_mouse_ && event.mouse().motion == Mouse::Released) { + captured_mouse_.reset(); + return true; + } + + if (event.mouse().button == Mouse::Left && + event.mouse().motion == Mouse::Pressed && + separator_box_.Contain(event.mouse().x, event.mouse().y) && + !captured_mouse_) { + captured_mouse_ = CaptureMouse(event); + return true; + } + + if (!captured_mouse_) { + return ComponentBase::OnEvent(event); + } + + switch (options_->direction()) { + case Direction::Left: + options_->main_size() = event.mouse().x - box_.x_min; + return true; + case Direction::Right: + options_->main_size() = box_.x_max - event.mouse().x; + return true; + case Direction::Up: + options_->main_size() = event.mouse().y - box_.y_min; + return true; + case Direction::Down: + options_->main_size() = box_.y_max - event.mouse().y; + return true; + } + + // NOTREACHED() + return false; + } + + Element OnRender() final { + switch (options_->direction()) { + case Direction::Left: + return RenderLeft(); + case Direction::Right: + return RenderRight(); + case Direction::Up: + return RenderTop(); + case Direction::Down: + return RenderBottom(); + } + // NOTREACHED() + return text("unreacheable"); + } + + Element RenderLeft() { + return hbox({ + options_->main->Render() | + size(WIDTH, EQUAL, options_->main_size()), + options_->separator_func() | reflect(separator_box_), + options_->back->Render() | xflex, + }) | + reflect(box_); + } + + Element RenderRight() { + return hbox({ + options_->back->Render() | xflex, + options_->separator_func() | reflect(separator_box_), + options_->main->Render() | + size(WIDTH, EQUAL, options_->main_size()), + }) | + reflect(box_); + } + + Element RenderTop() { + return vbox({ + options_->main->Render() | + size(HEIGHT, EQUAL, options_->main_size()), + options_->separator_func() | reflect(separator_box_), + options_->back->Render() | yflex, + }) | + reflect(box_); + } + + Element RenderBottom() { + return vbox({ + options_->back->Render() | yflex, + options_->separator_func() | reflect(separator_box_), + options_->main->Render() | + size(HEIGHT, EQUAL, options_->main_size()), + }) | + reflect(box_); + } + + private: + Ref<ResizableSplitOption> options_; + CapturedMouse captured_mouse_; + Box separator_box_; + Box box_; +}; + +} // namespace + +/// @brief A split in between two components. +/// @param options all the parameters. +/// +/// ### Example +/// +/// ```cpp +/// auto left = Renderer([] { return text("Left") | center;}); +/// auto right = Renderer([] { return text("right") | center;}); +/// int left_size = 10; +/// auto component = ResizableSplit({ +/// .main = left, +/// .back = right, +/// .direction = Direction::Left, +/// .main_size = &left_size, +/// .separator_func = [] { return separatorDouble(); }, +/// }); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// ║ +/// left ║ right +/// ║ +/// ``` +Component ResizableSplit(ResizableSplitOption options) { + return Make<ResizableSplitBase>(std::move(options)); +} + +/// @brief An horizontal split in between two components, configurable using the +/// mouse. +/// @param main The main component of size |main_size|, on the left. +/// @param back The back component taking the remaining size, on the right. +/// @param main_size The size of the |main| component. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::Fullscreen(); +/// int left_size = 10; +/// auto left = Renderer([] { return text("Left") | center;}); +/// auto right = Renderer([] { return text("right") | center;}); +/// auto split = ResizableSplitLeft(left, right, &left_size); +/// screen.Loop(split); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// │ +/// left │ right +/// │ +/// ``` +Component ResizableSplitLeft(Component main, Component back, int* main_size) { + return ResizableSplit({ + std::move(main), + std::move(back), + Direction::Left, + main_size, + }); +} + +/// @brief An horizontal split in between two components, configurable using the +/// mouse. +/// @param main The main component of size |main_size|, on the right. +/// @param back The back component taking the remaining size, on the left. +/// @param main_size The size of the |main| component. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::Fullscreen(); +/// int right_size = 10; +/// auto left = Renderer([] { return text("Left") | center;}); +/// auto right = Renderer([] { return text("right") | center;}); +/// auto split = ResizableSplitRight(right, left, &right_size) +/// screen.Loop(split); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// │ +/// left │ right +/// │ +/// ``` +Component ResizableSplitRight(Component main, Component back, int* main_size) { + return ResizableSplit({ + std::move(main), + std::move(back), + Direction::Right, + main_size, + }); +} + +/// @brief An vertical split in between two components, configurable using the +/// mouse. +/// @param main The main component of size |main_size|, on the top. +/// @param back The back component taking the remaining size, on the bottom. +/// @param main_size The size of the |main| component. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::Fullscreen(); +/// int top_size = 1; +/// auto top = Renderer([] { return text("Top") | center;}); +/// auto bottom = Renderer([] { return text("Bottom") | center;}); +/// auto split = ResizableSplitTop(top, bottom, &top_size) +/// screen.Loop(split); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// top +/// ──────────── +/// bottom +/// ``` +Component ResizableSplitTop(Component main, Component back, int* main_size) { + return ResizableSplit({ + std::move(main), + std::move(back), + Direction::Up, + main_size, + }); +} + +/// @brief An vertical split in between two components, configurable using the +/// mouse. +/// @param main The main component of size |main_size|, on the bottom. +/// @param back The back component taking the remaining size, on the top. +/// @param main_size The size of the |main| component. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::Fullscreen(); +/// int bottom_size = 1; +/// auto top = Renderer([] { return text("Top") | center;}); +/// auto bottom = Renderer([] { return text("Bottom") | center;}); +/// auto split = ResizableSplit::Bottom(bottom, top, &bottom_size) +/// screen.Loop(split); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// top +/// ──────────── +/// bottom +/// ``` +Component ResizableSplitBottom(Component main, Component back, int* main_size) { + return ResizableSplit({ + std::move(main), + std::move(back), + Direction::Down, + main_size, + }); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/screen_interactive.cpp b/contrib/libs/ftxui/src/ftxui/component/screen_interactive.cpp new file mode 100644 index 00000000000..309c58c0387 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/screen_interactive.cpp @@ -0,0 +1,1088 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include "ftxui/component/screen_interactive.hpp" +#include <algorithm> // for copy, max, min +#include <array> // for array +#include <atomic> +#include <chrono> // for operator-, milliseconds, operator>=, duration, common_type<>::type, time_point +#include <csignal> // for signal, SIGTSTP, SIGABRT, SIGWINCH, raise, SIGFPE, SIGILL, SIGINT, SIGSEGV, SIGTERM, __sighandler_t, size_t +#include <cstdint> +#include <cstdio> // for fileno, stdin +#include <ftxui/component/task.hpp> // for Task, Closure, AnimationTask +#include <ftxui/screen/screen.hpp> // for Pixel, Screen::Cursor, Screen, Screen::Cursor::Hidden +#include <functional> // for function +#include <initializer_list> // for initializer_list +#include <iostream> // for cout, ostream, operator<<, basic_ostream, endl, flush +#include <memory> +#include <stack> // for stack +#include <string> +#include <thread> // for thread, sleep_for +#include <tuple> // for _Swallow_assign, ignore +#include <type_traits> // for decay_t +#include <utility> // for move, swap +#include <variant> // for visit, variant +#include <vector> // for vector +#include "ftxui/component/animation.hpp" // for TimePoint, Clock, Duration, Params, RequestAnimationFrame +#include "ftxui/component/captured_mouse.hpp" // for CapturedMouse, CapturedMouseInterface +#include "ftxui/component/component_base.hpp" // for ComponentBase +#include "ftxui/component/event.hpp" // for Event +#include "ftxui/component/loop.hpp" // for Loop +#include "ftxui/component/receiver.hpp" // for ReceiverImpl, Sender, MakeReceiver, SenderImpl, Receiver +#include "ftxui/component/terminal_input_parser.hpp" // for TerminalInputParser +#include "ftxui/dom/node.hpp" // for Node, Render +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/pixel.hpp" // for Pixel +#include "ftxui/screen/terminal.hpp" // for Dimensions, Size + +#if defined(_WIN32) +#define DEFINE_CONSOLEV2_PROPERTIES +#define WIN32_LEAN_AND_MEAN +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include <windows.h> +#ifndef UNICODE +#error Must be compiled in UNICODE mode +#endif +#else +#include <sys/select.h> // for select, FD_ISSET, FD_SET, FD_ZERO, fd_set, timeval +#include <termios.h> // for tcsetattr, termios, tcgetattr, TCSANOW, cc_t, ECHO, ICANON, VMIN, VTIME +#include <unistd.h> // for STDIN_FILENO, read +#endif + +// Quick exit is missing in standard CLang headers +#if defined(__clang__) && defined(__APPLE__) +#define quick_exit(a) exit(a) +#endif + +namespace ftxui { + +namespace animation { +void RequestAnimationFrame() { + auto* screen = ScreenInteractive::Active(); + if (screen) { + screen->RequestAnimationFrame(); + } +} +} // namespace animation + +namespace { + +ScreenInteractive* g_active_screen = nullptr; // NOLINT + +void Flush() { + // Emscripten doesn't implement flush. We interpret zero as flush. + std::cout << '\0' << std::flush; +} + +constexpr int timeout_milliseconds = 20; +[[maybe_unused]] constexpr int timeout_microseconds = + timeout_milliseconds * 1000; +#if defined(_WIN32) + +void EventListener(std::atomic<bool>* quit, Sender<Task> out) { + auto console = GetStdHandle(STD_INPUT_HANDLE); + auto parser = TerminalInputParser(out->Clone()); + while (!*quit) { + // Throttle ReadConsoleInput by waiting 250ms, this wait function will + // return if there is input in the console. + auto wait_result = WaitForSingleObject(console, timeout_milliseconds); + if (wait_result == WAIT_TIMEOUT) { + parser.Timeout(timeout_milliseconds); + continue; + } + + DWORD number_of_events = 0; + if (!GetNumberOfConsoleInputEvents(console, &number_of_events)) + continue; + if (number_of_events <= 0) + continue; + + std::vector<INPUT_RECORD> records{number_of_events}; + DWORD number_of_events_read = 0; + ReadConsoleInput(console, records.data(), (DWORD)records.size(), + &number_of_events_read); + records.resize(number_of_events_read); + + for (const auto& r : records) { + switch (r.EventType) { + case KEY_EVENT: { + auto key_event = r.Event.KeyEvent; + // ignore UP key events + if (key_event.bKeyDown == FALSE) + continue; + std::wstring wstring; + wstring += key_event.uChar.UnicodeChar; + for (auto it : to_string(wstring)) { + parser.Add(it); + } + } break; + case WINDOW_BUFFER_SIZE_EVENT: + out->Send(Event::Special({0})); + break; + case MENU_EVENT: + case FOCUS_EVENT: + case MOUSE_EVENT: + // TODO(mauve): Implement later. + break; + } + } + } +} + +#elif defined(__EMSCRIPTEN__) +#error #include <emscripten.h> + +// Read char from the terminal. +void EventListener(std::atomic<bool>* quit, Sender<Task> out) { + auto parser = TerminalInputParser(std::move(out)); + + char c; + while (!*quit) { + while (read(STDIN_FILENO, &c, 1), c) + parser.Add(c); + + emscripten_sleep(1); + parser.Timeout(1); + } +} + +extern "C" { +EMSCRIPTEN_KEEPALIVE +void ftxui_on_resize(int columns, int rows) { + Terminal::SetFallbackSize({ + columns, + rows, + }); + std::raise(SIGWINCH); +} +} + +#else // POSIX (Linux & Mac) + +int CheckStdinReady(int usec_timeout) { + timeval tv = {0, usec_timeout}; // NOLINT + fd_set fds; + FD_ZERO(&fds); // NOLINT + FD_SET(STDIN_FILENO, &fds); // NOLINT + select(STDIN_FILENO + 1, &fds, nullptr, nullptr, &tv); // NOLINT + return FD_ISSET(STDIN_FILENO, &fds); // NOLINT +} + +// Read char from the terminal. +void EventListener(std::atomic<bool>* quit, Sender<Task> out) { + auto parser = TerminalInputParser(std::move(out)); + + while (!*quit) { + if (!CheckStdinReady(timeout_microseconds)) { + parser.Timeout(timeout_milliseconds); + continue; + } + + const size_t buffer_size = 100; + std::array<char, buffer_size> buffer; // NOLINT; + size_t l = read(fileno(stdin), buffer.data(), buffer_size); // NOLINT + for (size_t i = 0; i < l; ++i) { + parser.Add(buffer[i]); // NOLINT + } + } +} +#endif + +std::stack<Closure> on_exit_functions; // NOLINT +void OnExit() { + while (!on_exit_functions.empty()) { + on_exit_functions.top()(); + on_exit_functions.pop(); + } +} + +std::atomic<int> g_signal_exit_count = 0; // NOLINT +#if !defined(_WIN32) +std::atomic<int> g_signal_stop_count = 0; // NOLINT +std::atomic<int> g_signal_resize_count = 0; // NOLINT +#endif + +// Async signal safe function +void RecordSignal(int signal) { + switch (signal) { + case SIGABRT: + case SIGFPE: + case SIGILL: + case SIGINT: + case SIGSEGV: + case SIGTERM: + g_signal_exit_count++; + break; + +#if !defined(_WIN32) + case SIGTSTP: // NOLINT + g_signal_stop_count++; + break; + + case SIGWINCH: // NOLINT + g_signal_resize_count++; + break; +#endif + + default: + break; + } +} + +void ExecuteSignalHandlers() { + int signal_exit_count = g_signal_exit_count.exchange(0); + while (signal_exit_count--) { + ScreenInteractive::Private::Signal(*g_active_screen, SIGABRT); + } + +#if !defined(_WIN32) + int signal_stop_count = g_signal_stop_count.exchange(0); + while (signal_stop_count--) { + ScreenInteractive::Private::Signal(*g_active_screen, SIGTSTP); + } + + int signal_resize_count = g_signal_resize_count.exchange(0); + while (signal_resize_count--) { + ScreenInteractive::Private::Signal(*g_active_screen, SIGWINCH); + } +#endif +} + +void InstallSignalHandler(int sig) { + auto old_signal_handler = std::signal(sig, RecordSignal); + on_exit_functions.emplace( + [=] { std::ignore = std::signal(sig, old_signal_handler); }); +} + +// CSI: Control Sequence Introducer +const std::string CSI = "\x1b["; // NOLINT + // +// DCS: Device Control String +const std::string DCS = "\x1bP"; // NOLINT +// ST: String Terminator +const std::string ST = "\x1b\\"; // NOLINT + +// DECRQSS: Request Status String +// DECSCUSR: Set Cursor Style +const std::string DECRQSS_DECSCUSR = DCS + "$q q" + ST; // NOLINT + +// DEC: Digital Equipment Corporation +enum class DECMode : std::uint16_t { + kLineWrap = 7, + kCursor = 25, + + kMouseX10 = 9, + kMouseVt200 = 1000, + kMouseVt200Highlight = 1001, + + kMouseBtnEventMouse = 1002, + kMouseAnyEvent = 1003, + + kMouseUtf8 = 1005, + kMouseSgrExtMode = 1006, + kMouseUrxvtMode = 1015, + kMouseSgrPixelsMode = 1016, + kAlternateScreen = 1049, +}; + +// Device Status Report (DSR) { +enum class DSRMode : std::uint8_t { + kCursor = 6, +}; + +std::string Serialize(const std::vector<DECMode>& parameters) { + bool first = true; + std::string out; + for (const DECMode parameter : parameters) { + if (!first) { + out += ";"; + } + out += std::to_string(int(parameter)); + first = false; + } + return out; +} + +// DEC Private Mode Set (DECSET) +std::string Set(const std::vector<DECMode>& parameters) { + return CSI + "?" + Serialize(parameters) + "h"; +} + +// DEC Private Mode Reset (DECRST) +std::string Reset(const std::vector<DECMode>& parameters) { + return CSI + "?" + Serialize(parameters) + "l"; +} + +// Device Status Report (DSR) +std::string DeviceStatusReport(DSRMode ps) { + return CSI + std::to_string(int(ps)) + "n"; +} + +class CapturedMouseImpl : public CapturedMouseInterface { + public: + explicit CapturedMouseImpl(std::function<void(void)> callback) + : callback_(std::move(callback)) {} + ~CapturedMouseImpl() override { callback_(); } + CapturedMouseImpl(const CapturedMouseImpl&) = delete; + CapturedMouseImpl(CapturedMouseImpl&&) = delete; + CapturedMouseImpl& operator=(const CapturedMouseImpl&) = delete; + CapturedMouseImpl& operator=(CapturedMouseImpl&&) = delete; + + private: + std::function<void(void)> callback_; +}; + +void AnimationListener(std::atomic<bool>* quit, Sender<Task> out) { + // Animation at around 60fps. + const auto time_delta = std::chrono::milliseconds(15); + while (!*quit) { + out->Send(AnimationTask()); + std::this_thread::sleep_for(time_delta); + } +} + +} // namespace + +ScreenInteractive::ScreenInteractive(int dimx, + int dimy, + Dimension dimension, + bool use_alternative_screen) + : Screen(dimx, dimy), + dimension_(dimension), + use_alternative_screen_(use_alternative_screen) { + task_receiver_ = MakeReceiver<Task>(); +} + +// static +ScreenInteractive ScreenInteractive::FixedSize(int dimx, int dimy) { + return { + dimx, + dimy, + Dimension::Fixed, + false, + }; +} + +/// @ingroup component +/// Create a ScreenInteractive taking the full terminal size. This is using the +/// alternate screen buffer to avoid messing with the terminal content. +/// @note This is the same as `ScreenInteractive::FullscreenAlternateScreen()` +// static +ScreenInteractive ScreenInteractive::Fullscreen() { + return FullscreenAlternateScreen(); +} + +/// @ingroup component +/// Create a ScreenInteractive taking the full terminal size. The primary screen +/// buffer is being used. It means if the terminal is resized, the previous +/// content might mess up with the terminal content. +// static +ScreenInteractive ScreenInteractive::FullscreenPrimaryScreen() { + return { + 0, + 0, + Dimension::Fullscreen, + false, + }; +} + +/// @ingroup component +/// Create a ScreenInteractive taking the full terminal size. This is using the +/// alternate screen buffer to avoid messing with the terminal content. +// static +ScreenInteractive ScreenInteractive::FullscreenAlternateScreen() { + return { + 0, + 0, + Dimension::Fullscreen, + true, + }; +} + +// static +ScreenInteractive ScreenInteractive::TerminalOutput() { + return { + 0, + 0, + Dimension::TerminalOutput, + false, + }; +} + +// static +ScreenInteractive ScreenInteractive::FitComponent() { + return { + 0, + 0, + Dimension::FitComponent, + false, + }; +} + +/// @ingroup component +/// @brief Set whether mouse is tracked and events reported. +/// called outside of the main loop. E.g `ScreenInteractive::Loop(...)`. +/// @param enable Whether to enable mouse event tracking. +/// @note This muse be called outside of the main loop. E.g. before calling +/// `ScreenInteractive::Loop`. +/// @note Mouse tracking is enabled by default. +/// @note Mouse tracking is only supported on terminals that supports it. +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// screen.TrackMouse(false); +/// screen.Loop(component); +/// ``` +void ScreenInteractive::TrackMouse(bool enable) { + track_mouse_ = enable; +} + +/// @brief Add a task to the main loop. +/// It will be executed later, after every other scheduled tasks. +/// @ingroup component +void ScreenInteractive::Post(Task task) { + // Task/Events sent toward inactive screen or screen waiting to become + // inactive are dropped. + if (!task_sender_) { + return; + } + + task_sender_->Send(std::move(task)); +} + +/// @brief Add an event to the main loop. +/// It will be executed later, after every other scheduled events. +/// @ingroup component +void ScreenInteractive::PostEvent(Event event) { + Post(event); +} + +/// @brief Add a task to draw the screen one more time, until all the animations +/// are done. +void ScreenInteractive::RequestAnimationFrame() { + if (animation_requested_) { + return; + } + animation_requested_ = true; + auto now = animation::Clock::now(); + const auto time_histeresis = std::chrono::milliseconds(33); + if (now - previous_animation_time_ >= time_histeresis) { + previous_animation_time_ = now; + } +} + +/// @brief Try to get the unique lock about behing able to capture the mouse. +/// @return A unique lock if the mouse is not already captured, otherwise a +/// null. +/// @ingroup component +CapturedMouse ScreenInteractive::CaptureMouse() { + if (mouse_captured) { + return nullptr; + } + mouse_captured = true; + return std::make_unique<CapturedMouseImpl>( + [this] { mouse_captured = false; }); +} + +/// @brief Execute the main loop. +/// @param component The component to draw. +/// @ingroup component +void ScreenInteractive::Loop(Component component) { // NOLINT + class Loop loop(this, std::move(component)); + loop.Run(); +} + +/// @brief Return whether the main loop has been quit. +/// @ingroup component +bool ScreenInteractive::HasQuitted() { + return task_receiver_->HasQuitted(); +} + +// private +void ScreenInteractive::PreMain() { + // Suspend previously active screen: + if (g_active_screen) { + std::swap(suspended_screen_, g_active_screen); + // Reset cursor position to the top of the screen and clear the screen. + suspended_screen_->ResetCursorPosition(); + std::cout << suspended_screen_->ResetPosition(/*clear=*/true); + suspended_screen_->dimx_ = 0; + suspended_screen_->dimy_ = 0; + + // Reset dimensions to force drawing the screen again next time: + suspended_screen_->Uninstall(); + } + + // This screen is now active: + g_active_screen = this; + g_active_screen->Install(); + + previous_animation_time_ = animation::Clock::now(); +} + +// private +void ScreenInteractive::PostMain() { + // Put cursor position at the end of the drawing. + ResetCursorPosition(); + + g_active_screen = nullptr; + + // Restore suspended screen. + if (suspended_screen_) { + // Clear screen, and put the cursor at the beginning of the drawing. + std::cout << ResetPosition(/*clear=*/true); + dimx_ = 0; + dimy_ = 0; + Uninstall(); + std::swap(g_active_screen, suspended_screen_); + g_active_screen->Install(); + } else { + Uninstall(); + + std::cout << '\r'; + // On final exit, keep the current drawing and reset cursor position one + // line after it. + if (!use_alternative_screen_) { + std::cout << '\n'; + std::cout << std::flush; + } + } +} + +/// @brief Decorate a function. It executes the same way, but with the currently +/// active screen terminal hooks temporarilly uninstalled during its execution. +/// @param fn The function to decorate. +Closure ScreenInteractive::WithRestoredIO(Closure fn) { // NOLINT + return [this, fn] { + Uninstall(); + fn(); + Install(); + }; +} + +/// @brief Force FTXUI to handle or not handle Ctrl-C, even if the component +/// catches the Event::CtrlC. +void ScreenInteractive::ForceHandleCtrlC(bool force) { + force_handle_ctrl_c_ = force; +} + +/// @brief Force FTXUI to handle or not handle Ctrl-Z, even if the component +/// catches the Event::CtrlZ. +void ScreenInteractive::ForceHandleCtrlZ(bool force) { + force_handle_ctrl_z_ = force; +} + +/// @brief Returns the content of the current selection +std::string ScreenInteractive::GetSelection() { + if (!selection_) { + return ""; + } + return selection_->GetParts(); +} + +void ScreenInteractive::SelectionChange(std::function<void()> callback) { + selection_on_change_ = std::move(callback); +} + +/// @brief Return the currently active screen, or null if none. +// static +ScreenInteractive* ScreenInteractive::Active() { + return g_active_screen; +} + +// private +void ScreenInteractive::Install() { + frame_valid_ = false; + + // Flush the buffer for stdout to ensure whatever the user has printed before + // is fully applied before we start modifying the terminal configuration. This + // is important, because we are using two different channels (stdout vs + // termios/WinAPI) to communicate with the terminal emulator below. See + // https://github.com/ArthurSonzogni/FTXUI/issues/846 + Flush(); + + // After uninstalling the new configuration, flush it to the terminal to + // ensure it is fully applied: + on_exit_functions.emplace([] { Flush(); }); + + on_exit_functions.emplace([this] { ExitLoopClosure()(); }); + + // Request the terminal to report the current cursor shape. We will restore it + // on exit. + std::cout << DECRQSS_DECSCUSR; + on_exit_functions.emplace([this] { + std::cout << "\033[?25h"; // Enable cursor. + std::cout << "\033[" + std::to_string(cursor_reset_shape_) + " q"; + }); + + // Install signal handlers to restore the terminal state on exit. The default + // signal handlers are restored on exit. + for (const int signal : {SIGTERM, SIGSEGV, SIGINT, SIGILL, SIGABRT, SIGFPE}) { + InstallSignalHandler(signal); + } + +// Save the old terminal configuration and restore it on exit. +#if defined(_WIN32) + // Enable VT processing on stdout and stdin + auto stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE); + auto stdin_handle = GetStdHandle(STD_INPUT_HANDLE); + + DWORD out_mode = 0; + DWORD in_mode = 0; + GetConsoleMode(stdout_handle, &out_mode); + GetConsoleMode(stdin_handle, &in_mode); + on_exit_functions.push([=] { SetConsoleMode(stdout_handle, out_mode); }); + on_exit_functions.push([=] { SetConsoleMode(stdin_handle, in_mode); }); + + // https://docs.microsoft.com/en-us/windows/console/setconsolemode + const int enable_virtual_terminal_processing = 0x0004; + const int disable_newline_auto_return = 0x0008; + out_mode |= enable_virtual_terminal_processing; + out_mode |= disable_newline_auto_return; + + // https://docs.microsoft.com/en-us/windows/console/setconsolemode + const int enable_line_input = 0x0002; + const int enable_echo_input = 0x0004; + const int enable_virtual_terminal_input = 0x0200; + const int enable_window_input = 0x0008; + in_mode &= ~enable_echo_input; + in_mode &= ~enable_line_input; + in_mode |= enable_virtual_terminal_input; + in_mode |= enable_window_input; + + SetConsoleMode(stdin_handle, in_mode); + SetConsoleMode(stdout_handle, out_mode); +#else + for (const int signal : {SIGWINCH, SIGTSTP}) { + InstallSignalHandler(signal); + } + + struct termios terminal; // NOLINT + tcgetattr(STDIN_FILENO, &terminal); + on_exit_functions.emplace( + [=] { tcsetattr(STDIN_FILENO, TCSANOW, &terminal); }); + + // Enabling raw terminal input mode + terminal.c_iflag &= ~IGNBRK; // Disable ignoring break condition + terminal.c_iflag &= ~BRKINT; // Disable break causing input and output to be + // flushed + terminal.c_iflag &= ~PARMRK; // Disable marking parity errors. + terminal.c_iflag &= ~ISTRIP; // Disable striping 8th bit off characters. + terminal.c_iflag &= ~INLCR; // Disable mapping NL to CR. + terminal.c_iflag &= ~IGNCR; // Disable ignoring CR. + terminal.c_iflag &= ~ICRNL; // Disable mapping CR to NL. + terminal.c_iflag &= ~IXON; // Disable XON/XOFF flow control on output + + terminal.c_lflag &= ~ECHO; // Disable echoing input characters. + terminal.c_lflag &= ~ECHONL; // Disable echoing new line characters. + terminal.c_lflag &= ~ICANON; // Disable Canonical mode. + terminal.c_lflag &= ~ISIG; // Disable sending signal when hitting: + // - => DSUSP + // - C-Z => SUSP + // - C-C => INTR + // - C-d => QUIT + terminal.c_lflag &= ~IEXTEN; // Disable extended input processing + terminal.c_cflag |= CS8; // 8 bits per byte + + terminal.c_cc[VMIN] = 0; // Minimum number of characters for non-canonical + // read. + terminal.c_cc[VTIME] = 0; // Timeout in deciseconds for non-canonical read. + + tcsetattr(STDIN_FILENO, TCSANOW, &terminal); + +#endif + + auto enable = [&](const std::vector<DECMode>& parameters) { + std::cout << Set(parameters); + on_exit_functions.emplace([=] { std::cout << Reset(parameters); }); + }; + + auto disable = [&](const std::vector<DECMode>& parameters) { + std::cout << Reset(parameters); + on_exit_functions.emplace([=] { std::cout << Set(parameters); }); + }; + + if (use_alternative_screen_) { + enable({ + DECMode::kAlternateScreen, + }); + } + + disable({ + // DECMode::kCursor, + DECMode::kLineWrap, + }); + + if (track_mouse_) { + enable({DECMode::kMouseVt200}); + enable({DECMode::kMouseAnyEvent}); + enable({DECMode::kMouseUrxvtMode}); + enable({DECMode::kMouseSgrExtMode}); + } + + // After installing the new configuration, flush it to the terminal to + // ensure it is fully applied: + Flush(); + + quit_ = false; + task_sender_ = task_receiver_->MakeSender(); + event_listener_ = + std::thread(&EventListener, &quit_, task_receiver_->MakeSender()); + animation_listener_ = + std::thread(&AnimationListener, &quit_, task_receiver_->MakeSender()); +} + +// private +void ScreenInteractive::Uninstall() { + ExitNow(); + event_listener_.join(); + animation_listener_.join(); + OnExit(); +} + +// private +// NOLINTNEXTLINE +void ScreenInteractive::RunOnceBlocking(Component component) { + ExecuteSignalHandlers(); + Task task; + if (task_receiver_->Receive(&task)) { + HandleTask(component, task); + } + RunOnce(component); +} + +// private +void ScreenInteractive::RunOnce(Component component) { + Task task; + while (task_receiver_->ReceiveNonBlocking(&task)) { + HandleTask(component, task); + ExecuteSignalHandlers(); + } + Draw(std::move(component)); + + if (selection_data_previous_ != selection_data_) { + selection_data_previous_ = selection_data_; + if (selection_on_change_) { + selection_on_change_(); + Post(Event::Custom); + } + } +} + +// private +// NOLINTNEXTLINE +void ScreenInteractive::HandleTask(Component component, Task& task) { + std::visit( + [&](auto&& arg) { + using T = std::decay_t<decltype(arg)>; + + // clang-format off + // Handle Event. + if constexpr (std::is_same_v<T, Event>) { + if (arg.is_cursor_position()) { + cursor_x_ = arg.cursor_x(); + cursor_y_ = arg.cursor_y(); + return; + } + + if (arg.is_cursor_shape()) { + cursor_reset_shape_= arg.cursor_shape(); + return; + } + + if (arg.is_mouse()) { + arg.mouse().x -= cursor_x_; + arg.mouse().y -= cursor_y_; + } + + arg.screen_ = this; + + bool handled = component->OnEvent(arg); + + handled = HandleSelection(handled, arg); + + if (arg == Event::CtrlC && (!handled || force_handle_ctrl_c_)) { + RecordSignal(SIGABRT); + } + +#if !defined(_WIN32) + if (arg == Event::CtrlZ && (!handled || force_handle_ctrl_z_)) { + RecordSignal(SIGTSTP); + } +#endif + + frame_valid_ = false; + return; + } + + // Handle callback + if constexpr (std::is_same_v<T, Closure>) { + arg(); + return; + } + + // Handle Animation + if constexpr (std::is_same_v<T, AnimationTask>) { + if (!animation_requested_) { + return; + } + + animation_requested_ = false; + const animation::TimePoint now = animation::Clock::now(); + const animation::Duration delta = now - previous_animation_time_; + previous_animation_time_ = now; + + animation::Params params(delta); + component->OnAnimation(params); + frame_valid_ = false; + return; + } + }, + task); + // clang-format on +} + +// private +bool ScreenInteractive::HandleSelection(bool handled, Event event) { + if (handled) { + selection_pending_ = nullptr; + selection_data_.empty = true; + selection_ = nullptr; + return true; + } + + if (!event.is_mouse()) { + return false; + } + + auto& mouse = event.mouse(); + if (mouse.button != Mouse::Left) { + return false; + } + + if (mouse.motion == Mouse::Pressed) { + selection_pending_ = CaptureMouse(); + selection_data_.start_x = mouse.x; + selection_data_.start_y = mouse.y; + selection_data_.end_x = mouse.x; + selection_data_.end_y = mouse.y; + return false; + } + + if (!selection_pending_) { + return false; + } + + if (mouse.motion == Mouse::Moved) { + if ((mouse.x != selection_data_.end_x) || + (mouse.y != selection_data_.end_y)) { + selection_data_.end_x = mouse.x; + selection_data_.end_y = mouse.y; + selection_data_.empty = false; + } + + return true; + } + + if (mouse.motion == Mouse::Released) { + selection_pending_ = nullptr; + selection_data_.end_x = mouse.x; + selection_data_.end_y = mouse.y; + selection_data_.empty = false; + return true; + } + + return false; +} + +// private +// NOLINTNEXTLINE +void ScreenInteractive::Draw(Component component) { + if (frame_valid_) { + return; + } + auto document = component->Render(); + int dimx = 0; + int dimy = 0; + auto terminal = Terminal::Size(); + document->ComputeRequirement(); + switch (dimension_) { + case Dimension::Fixed: + dimx = dimx_; + dimy = dimy_; + break; + case Dimension::TerminalOutput: + dimx = terminal.dimx; + dimy = document->requirement().min_y; + break; + case Dimension::Fullscreen: + dimx = terminal.dimx; + dimy = terminal.dimy; + break; + case Dimension::FitComponent: + dimx = std::min(document->requirement().min_x, terminal.dimx); + dimy = std::min(document->requirement().min_y, terminal.dimy); + break; + } + + const bool resized = (dimx != dimx_) || (dimy != dimy_); + ResetCursorPosition(); + std::cout << ResetPosition(/*clear=*/resized); + + // If the terminal width decrease, the terminal emulator will start wrapping + // lines and make the display dirty. We should clear it completely. + if ((dimx < dimx_) && !use_alternative_screen_) { + std::cout << "\033[J"; // clear terminal output + std::cout << "\033[H"; // move cursor to home position + } + + // Resize the screen if needed. + if (resized) { + dimx_ = dimx; + dimy_ = dimy; + pixels_ = std::vector<std::vector<Pixel>>(dimy, std::vector<Pixel>(dimx)); + cursor_.x = dimx_ - 1; + cursor_.y = dimy_ - 1; + } + + // Periodically request the terminal emulator the frame position relative to + // the screen. This is useful for converting mouse position reported in + // screen's coordinates to frame's coordinates. +#if defined(FTXUI_MICROSOFT_TERMINAL_FALLBACK) + // Microsoft's terminal suffers from a [bug]. When reporting the cursor + // position, several output sequences are mixed together into garbage. + // This causes FTXUI user to see some "1;1;R" sequences into the Input + // component. See [issue]. Solution is to request cursor position less + // often. [bug]: https://github.com/microsoft/terminal/pull/7583 [issue]: + // https://github.com/ArthurSonzogni/FTXUI/issues/136 + static int i = -3; + ++i; + if (!use_alternative_screen_ && (i % 150 == 0)) { // NOLINT + std::cout << DeviceStatusReport(DSRMode::kCursor); + } +#else + static int i = -3; + ++i; + if (!use_alternative_screen_ && + (previous_frame_resized_ || i % 40 == 0)) { // NOLINT + std::cout << DeviceStatusReport(DSRMode::kCursor); + } +#endif + previous_frame_resized_ = resized; + + selection_ = selection_data_.empty + ? std::make_unique<Selection>() + : std::make_unique<Selection>( + selection_data_.start_x, selection_data_.start_y, // + selection_data_.end_x, selection_data_.end_y); + Render(*this, document.get(), *selection_); + + // Set cursor position for user using tools to insert CJK characters. + { + const int dx = dimx_ - 1 - cursor_.x + int(dimx_ != terminal.dimx); + const int dy = dimy_ - 1 - cursor_.y; + + set_cursor_position.clear(); + reset_cursor_position.clear(); + + if (dy != 0) { + set_cursor_position += "\x1B[" + std::to_string(dy) + "A"; + reset_cursor_position += "\x1B[" + std::to_string(dy) + "B"; + } + + if (dx != 0) { + set_cursor_position += "\x1B[" + std::to_string(dx) + "D"; + reset_cursor_position += "\x1B[" + std::to_string(dx) + "C"; + } + + if (cursor_.shape == Cursor::Hidden) { + set_cursor_position += "\033[?25l"; + } else { + set_cursor_position += "\033[?25h"; + set_cursor_position += + "\033[" + std::to_string(int(cursor_.shape)) + " q"; + } + } + + std::cout << ToString() << set_cursor_position; + Flush(); + Clear(); + frame_valid_ = true; +} + +// private +void ScreenInteractive::ResetCursorPosition() { + std::cout << reset_cursor_position; + reset_cursor_position = ""; +} + +/// @brief Return a function to exit the main loop. +/// @ingroup component +Closure ScreenInteractive::ExitLoopClosure() { + return [this] { Exit(); }; +} + +/// @brief Exit the main loop. +/// @ingroup component +void ScreenInteractive::Exit() { + Post([this] { ExitNow(); }); +} + +// private: +void ScreenInteractive::ExitNow() { + quit_ = true; + task_sender_.reset(); +} + +// private: +void ScreenInteractive::Signal(int signal) { + if (signal == SIGABRT) { + Exit(); + return; + } + +// Windows do no support SIGTSTP / SIGWINCH +#if !defined(_WIN32) + if (signal == SIGTSTP) { + Post([&] { + ResetCursorPosition(); + std::cout << ResetPosition(/*clear*/ true); // Cursor to the beginning + Uninstall(); + dimx_ = 0; + dimy_ = 0; + Flush(); + std::ignore = std::raise(SIGTSTP); + Install(); + }); + return; + } + + if (signal == SIGWINCH) { + Post(Event::Special({0})); + return; + } +#endif +} + +bool ScreenInteractive::SelectionData::operator==( + const ScreenInteractive::SelectionData& other) const { + if (empty && other.empty) { + return true; + } + if (empty || other.empty) { + return false; + } + return start_x == other.start_x && start_y == other.start_y && + end_x == other.end_x && end_y == other.end_y; +} + +bool ScreenInteractive::SelectionData::operator!=( + const ScreenInteractive::SelectionData& other) const { + return !(*this == other); +} + +} // namespace ftxui. diff --git a/contrib/libs/ftxui/src/ftxui/component/slider.cpp b/contrib/libs/ftxui/src/ftxui/component/slider.cpp new file mode 100644 index 00000000000..adbb72031f3 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/slider.cpp @@ -0,0 +1,363 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for max, min +#include <ftxui/component/component_options.hpp> // for SliderOption +#include <ftxui/dom/direction.hpp> // for Direction, Direction::Down, Direction::Left, Direction::Right, Direction::Up +#include <string> // for allocator +#include <utility> // for move + +#include "ftxui/component/captured_mouse.hpp" // for CapturedMouse +#include "ftxui/component/component.hpp" // for Make, Slider +#include "ftxui/component/component_base.hpp" // for ComponentBase +#include "ftxui/component/event.hpp" // for Event, Event::ArrowDown, Event::ArrowLeft, Event::ArrowRight, Event::ArrowUp +#include "ftxui/component/mouse.hpp" // for Mouse, Mouse::Left, Mouse::Pressed, Mouse::Released +#include "ftxui/component/screen_interactive.hpp" // for Component +#include "ftxui/dom/elements.hpp" // for operator|, text, Element, xflex, hbox, color, underlined, reflect, Decorator, dim, vcenter, focus, nothing, select, yflex, gaugeDirection +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/color.hpp" // for Color, Color::GrayDark, Color::White +#include "ftxui/util/ref.hpp" // for ConstRef, Ref, ConstStringRef + +namespace ftxui { + +namespace { +Decorator flexDirection(Direction direction) { + switch (direction) { + case Direction::Up: + case Direction::Down: + return yflex; + case Direction::Left: + case Direction::Right: + return xflex; + } + return xflex; // NOT_REACHED() +} + +template <class T> +class SliderBase : public SliderOption<T>, public ComponentBase { + public: + explicit SliderBase(SliderOption<T> options) : SliderOption<T>(options) {} + + Element OnRender() override { + auto gauge_color = + Focused() ? color(this->color_active) : color(this->color_inactive); + const float percent = + float(this->value() - this->min()) / float(this->max() - this->min()); + return gaugeDirection(percent, this->direction) | + flexDirection(this->direction) | reflect(gauge_box_) | gauge_color; + } + + void OnLeft() { + switch (this->direction) { + case Direction::Right: + this->value() -= this->increment(); + break; + case Direction::Left: + this->value() += this->increment(); + break; + case Direction::Up: + case Direction::Down: + break; + } + } + + void OnRight() { + switch (this->direction) { + case Direction::Right: + this->value() += this->increment(); + break; + case Direction::Left: + this->value() -= this->increment(); + break; + case Direction::Up: + case Direction::Down: + break; + } + } + + void OnUp() { + switch (this->direction) { + case Direction::Up: + this->value() -= this->increment(); + break; + case Direction::Down: + this->value() += this->increment(); + break; + case Direction::Left: + case Direction::Right: + break; + } + } + + void OnDown() { + switch (this->direction) { + case Direction::Down: + this->value() += this->increment(); + break; + case Direction::Up: + this->value() -= this->increment(); + break; + case Direction::Left: + case Direction::Right: + break; + } + } + + bool OnEvent(Event event) final { + if (event.is_mouse()) { + return OnMouseEvent(event); + } + + T old_value = this->value(); + if (event == Event::ArrowLeft || event == Event::Character('h')) { + OnLeft(); + } + if (event == Event::ArrowRight || event == Event::Character('l')) { + OnRight(); + } + if (event == Event::ArrowUp || event == Event::Character('k')) { + OnDown(); + } + if (event == Event::ArrowDown || event == Event::Character('j')) { + OnUp(); + } + + this->value() = std::max(this->min(), std::min(this->max(), this->value())); + if (old_value != this->value()) { + if (this->on_change) { + this->on_change(); + } + return true; + } + + return ComponentBase::OnEvent(event); + } + + bool OnCapturedMouseEvent(Event event) { + if (event.mouse().motion == Mouse::Released) { + captured_mouse_ = nullptr; + return true; + } + + T old_value = this->value(); + switch (this->direction) { + case Direction::Right: { + this->value() = this->min() + (event.mouse().x - gauge_box_.x_min) * + (this->max() - this->min()) / + (gauge_box_.x_max - gauge_box_.x_min); + + break; + } + case Direction::Left: { + this->value() = this->max() - (event.mouse().x - gauge_box_.x_min) * + (this->max() - this->min()) / + (gauge_box_.x_max - gauge_box_.x_min); + break; + } + case Direction::Down: { + this->value() = this->min() + (event.mouse().y - gauge_box_.y_min) * + (this->max() - this->min()) / + (gauge_box_.y_max - gauge_box_.y_min); + break; + } + case Direction::Up: { + this->value() = this->max() - (event.mouse().y - gauge_box_.y_min) * + (this->max() - this->min()) / + (gauge_box_.y_max - gauge_box_.y_min); + break; + } + } + + this->value() = std::max(this->min(), std::min(this->max(), this->value())); + + if (old_value != this->value() && this->on_change) { + this->on_change(); + } + return true; + } + + bool OnMouseEvent(Event event) { + if (captured_mouse_) { + return OnCapturedMouseEvent(event); + } + + if (event.mouse().button != Mouse::Left) { + return false; + } + if (event.mouse().motion != Mouse::Pressed) { + return false; + } + + if (!gauge_box_.Contain(event.mouse().x, event.mouse().y)) { + return false; + } + + captured_mouse_ = CaptureMouse(event); + + if (captured_mouse_) { + TakeFocus(); + return OnCapturedMouseEvent(event); + } + + return false; + } + + bool Focusable() const final { return true; } + + private: + Box gauge_box_; + CapturedMouse captured_mouse_; +}; + +class SliderWithLabel : public ComponentBase { + public: + SliderWithLabel(ConstStringRef label, Component inner) + : label_(std::move(label)) { + Add(std::move(inner)); + SetActiveChild(ChildAt(0)); + } + + private: + bool OnEvent(Event event) final { + if (ComponentBase::OnEvent(event)) { + return true; + } + + if (!event.is_mouse()) { + return false; + } + + mouse_hover_ = box_.Contain(event.mouse().x, event.mouse().y); + + if (!mouse_hover_) { + return false; + } + + if (!CaptureMouse(event)) { + return false; + } + + return true; + } + + Element OnRender() override { + auto gauge_color = (Focused() || mouse_hover_) ? color(Color::White) + : color(Color::GrayDark); + auto element = hbox({ + text(label_()) | dim | vcenter, + hbox({ + text("["), + ComponentBase::Render() | underlined, + text("]"), + }) | xflex, + }) | + gauge_color | xflex | reflect(box_); + + element |= focus; + return element; + } + + ConstStringRef label_; + Box box_; + bool mouse_hover_ = false; +}; + +} // namespace + +/// @brief An horizontal slider. +/// @param label The name of the slider. +/// @param value The current value of the slider. +/// @param min The minimum value. +/// @param max The maximum value. +/// @param increment The increment when used by the cursor. +/// @ingroup component +/// +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// int value = 50; +/// auto slider = Slider("Value:", &value, 0, 100, 1); +/// screen.Loop(slider); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// Value:[██████████████████████████ ] +/// ``` +Component Slider(ConstStringRef label, + Ref<int> value, + ConstRef<int> min, + ConstRef<int> max, + ConstRef<int> increment) { + SliderOption<int> option; + option.value = value; + option.min = min; + option.max = max; + option.increment = increment; + auto slider = Make<SliderBase<int>>(option); + return Make<SliderWithLabel>(std::move(label), slider); +} + +Component Slider(ConstStringRef label, + Ref<float> value, + ConstRef<float> min, + ConstRef<float> max, + ConstRef<float> increment) { + SliderOption<float> option; + option.value = value; + option.min = min; + option.max = max; + option.increment = increment; + auto slider = Make<SliderBase<float>>(option); + return Make<SliderWithLabel>(std::move(label), slider); +} +Component Slider(ConstStringRef label, + Ref<long> value, + ConstRef<long> min, + ConstRef<long> max, + ConstRef<long> increment) { + SliderOption<long> option; + option.value = value; + option.min = min; + option.max = max; + option.increment = increment; + auto slider = Make<SliderBase<long>>(option); + return Make<SliderWithLabel>(std::move(label), slider); +} + +/// @brief A slider in any direction. +/// @param options The options +/// ### Example +/// +/// ```cpp +/// auto screen = ScreenInteractive::TerminalOutput(); +/// int value = 50; +/// auto slider = Slider({ +/// .value = &value, +/// .min = 0, +/// .max = 100, +/// .increment= 20, +/// }); +/// screen.Loop(slider); +/// ``` +template <typename T> +Component Slider(SliderOption<T> options) { + return Make<SliderBase<T>>(options); +} + +template Component Slider(SliderOption<int8_t>); +template Component Slider(SliderOption<int16_t>); +template Component Slider(SliderOption<int32_t>); +template Component Slider(SliderOption<int64_t>); + +template Component Slider(SliderOption<uint8_t>); +template Component Slider(SliderOption<uint16_t>); +template Component Slider(SliderOption<uint32_t>); +template Component Slider(SliderOption<uint64_t>); + +template Component Slider(SliderOption<float>); +template Component Slider(SliderOption<double>); + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/terminal_input_parser.cpp b/contrib/libs/ftxui/src/ftxui/component/terminal_input_parser.cpp new file mode 100644 index 00000000000..2100d9edd3b --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/terminal_input_parser.cpp @@ -0,0 +1,464 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include "ftxui/component/terminal_input_parser.hpp" + +#include <cstdint> // for uint32_t +#include <ftxui/component/mouse.hpp> // for Mouse, Mouse::Button, Mouse::Motion +#include <ftxui/component/receiver.hpp> // for SenderImpl, Sender +#include <map> +#include <memory> // for unique_ptr, allocator +#include <utility> // for move +#include <vector> +#include "ftxui/component/event.hpp" // for Event +#include "ftxui/component/task.hpp" // for Task + +namespace ftxui { + +// NOLINTNEXTLINE +const std::map<std::string, std::string> g_uniformize = { + // Microsoft's terminal uses a different new line character for the return + // key. This also happens with linux with the `bind` command: + // See https://github.com/ArthurSonzogni/FTXUI/issues/337 + // Here, we uniformize the new line character to `\n`. + {"\r", "\n"}, + + // See: https://github.com/ArthurSonzogni/FTXUI/issues/508 + {std::string({8}), std::string({127})}, + + // See: https://github.com/ArthurSonzogni/FTXUI/issues/626 + // + // Depending on the Cursor Key Mode (DECCKM), the terminal sends different + // escape sequences: + // + // Key Normal Application + // ----- -------- ----------- + // Up ESC [ A ESC O A + // Down ESC [ B ESC O B + // Right ESC [ C ESC O C + // Left ESC [ D ESC O D + // Home ESC [ H ESC O H + // End ESC [ F ESC O F + // + {"\x1BOA", "\x1B[A"}, // UP + {"\x1BOB", "\x1B[B"}, // DOWN + {"\x1BOC", "\x1B[C"}, // RIGHT + {"\x1BOD", "\x1B[D"}, // LEFT + {"\x1BOH", "\x1B[H"}, // HOME + {"\x1BOF", "\x1B[F"}, // END + + // Variations around the FN keys. + // Internally, we are using: + // vt220, xterm-vt200, xterm-xf86-v44, xterm-new, mgt, screen + // See: https://invisible-island.net/xterm/xterm-function-keys.html + + // For linux OS console (CTRL+ALT+FN), who do not belong to any + // real standard. + // See: https://github.com/ArthurSonzogni/FTXUI/issues/685 + {"\x1B[[A", "\x1BOP"}, // F1 + {"\x1B[[B", "\x1BOQ"}, // F2 + {"\x1B[[C", "\x1BOR"}, // F3 + {"\x1B[[D", "\x1BOS"}, // F4 + {"\x1B[[E", "\x1B[15~"}, // F5 + + // xterm-r5, xterm-r6, rxvt + {"\x1B[11~", "\x1BOP"}, // F1 + {"\x1B[12~", "\x1BOQ"}, // F2 + {"\x1B[13~", "\x1BOR"}, // F3 + {"\x1B[14~", "\x1BOS"}, // F4 + + // vt100 + {"\x1BOt", "\x1B[15~"}, // F5 + {"\x1BOu", "\x1B[17~"}, // F6 + {"\x1BOv", "\x1B[18~"}, // F7 + {"\x1BOl", "\x1B[19~"}, // F8 + {"\x1BOw", "\x1B[20~"}, // F9 + {"\x1BOx", "\x1B[21~"}, // F10 + + // scoansi + {"\x1B[M", "\x1BOP"}, // F1 + {"\x1B[N", "\x1BOQ"}, // F2 + {"\x1B[O", "\x1BOR"}, // F3 + {"\x1B[P", "\x1BOS"}, // F4 + {"\x1B[Q", "\x1B[15~"}, // F5 + {"\x1B[R", "\x1B[17~"}, // F6 + {"\x1B[S", "\x1B[18~"}, // F7 + {"\x1B[T", "\x1B[19~"}, // F8 + {"\x1B[U", "\x1B[20~"}, // F9 + {"\x1B[V", "\x1B[21~"}, // F10 + {"\x1B[W", "\x1B[23~"}, // F11 + {"\x1B[X", "\x1B[24~"}, // F12 +}; + +TerminalInputParser::TerminalInputParser(Sender<Task> out) + : out_(std::move(out)) {} + +void TerminalInputParser::Timeout(int time) { + timeout_ += time; + const int timeout_threshold = 50; + if (timeout_ < timeout_threshold) { + return; + } + timeout_ = 0; + if (!pending_.empty()) { + Send(SPECIAL); + } +} + +void TerminalInputParser::Add(char c) { + pending_ += c; + timeout_ = 0; + position_ = -1; + Send(Parse()); +} + +unsigned char TerminalInputParser::Current() { + return pending_[position_]; +} + +bool TerminalInputParser::Eat() { + position_++; + return position_ < static_cast<int>(pending_.size()); +} + +void TerminalInputParser::Send(TerminalInputParser::Output output) { + switch (output.type) { + case UNCOMPLETED: + return; + + case DROP: + pending_.clear(); + return; + + case CHARACTER: + out_->Send(Event::Character(std::move(pending_))); + pending_.clear(); + return; + + case SPECIAL: { + auto it = g_uniformize.find(pending_); + if (it != g_uniformize.end()) { + pending_ = it->second; + } + out_->Send(Event::Special(std::move(pending_))); + pending_.clear(); + } + return; + + case MOUSE: + out_->Send(Event::Mouse(std::move(pending_), output.mouse)); // NOLINT + pending_.clear(); + return; + + case CURSOR_POSITION: + out_->Send(Event::CursorPosition(std::move(pending_), // NOLINT + output.cursor.x, // NOLINT + output.cursor.y)); // NOLINT + pending_.clear(); + return; + + case CURSOR_SHAPE: + out_->Send(Event::CursorShape(std::move(pending_), output.cursor_shape)); + pending_.clear(); + return; + } + // NOT_REACHED(). +} + +TerminalInputParser::Output TerminalInputParser::Parse() { + if (!Eat()) { + return UNCOMPLETED; + } + + if (Current() == '\x1B') { + return ParseESC(); + } + + if (Current() < 32) { // C0 NOLINT + return SPECIAL; + } + + if (Current() == 127) { // Delete // NOLINT + return SPECIAL; + } + + return ParseUTF8(); +} + +// Code point <-> UTF-8 conversion +// +// ┏━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┓ +// ┃Byte 1 ┃Byte 2 ┃Byte 3 ┃Byte 4 ┃ +// ┡━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━┩ +// │0xxxxxxx│ │ │ │ +// ├────────┼────────┼────────┼────────┤ +// │110xxxxx│10xxxxxx│ │ │ +// ├────────┼────────┼────────┼────────┤ +// │1110xxxx│10xxxxxx│10xxxxxx│ │ +// ├────────┼────────┼────────┼────────┤ +// │11110xxx│10xxxxxx│10xxxxxx│10xxxxxx│ +// └────────┴────────┴────────┴────────┘ +// +// Then some sequences are illegal if it exist a shorter representation of the +// same codepoint. +TerminalInputParser::Output TerminalInputParser::ParseUTF8() { + auto head = Current(); + unsigned char selector = 0b1000'0000; // NOLINT + + // The non code-point part of the first byte. + unsigned char mask = selector; + + // Find the first zero in the first byte. + unsigned int first_zero = 8; // NOLINT + for (unsigned int i = 0; i < 8; ++i) { // NOLINT + mask |= selector; + if (!(head & selector)) { + first_zero = i; + break; + } + selector >>= 1U; + } + + // Accumulate the value of the first byte. + auto value = uint32_t(head & ~mask); // NOLINT + + // Invalid UTF8, with more than 5 bytes. + const unsigned int max_utf8_bytes = 5; + if (first_zero == 1 || first_zero >= max_utf8_bytes) { + return DROP; + } + + // Multi byte UTF-8. + for (unsigned int i = 2; i <= first_zero; ++i) { + if (!Eat()) { + return UNCOMPLETED; + } + + // Invalid continuation byte. + head = Current(); + if ((head & 0b1100'0000) != 0b1000'0000) { // NOLINT + return DROP; + } + value <<= 6; // NOLINT + value += head & 0b0011'1111; // NOLINT + } + + // Check for overlong UTF8 encoding. + int extra_byte = 0; + if (value <= 0b000'0000'0111'1111) { // NOLINT + extra_byte = 0; // NOLINT + } else if (value <= 0b000'0111'1111'1111) { // NOLINT + extra_byte = 1; // NOLINT + } else if (value <= 0b1111'1111'1111'1111) { // NOLINT + extra_byte = 2; // NOLINT + } else if (value <= 0b1'0000'1111'1111'1111'1111) { // NOLINT + extra_byte = 3; // NOLINT + } else { // NOLINT + return DROP; + } + + if (extra_byte != position_) { + return DROP; + } + + return CHARACTER; +} + +TerminalInputParser::Output TerminalInputParser::ParseESC() { + if (!Eat()) { + return UNCOMPLETED; + } + switch (Current()) { + case 'P': + return ParseDCS(); + case '[': + return ParseCSI(); + case ']': + return ParseOSC(); + + // Expecting 2 characters. + case ' ': + case '#': + case '%': + case '(': + case ')': + case '*': + case '+': + case 'O': + case 'N': { + if (!Eat()) { + return UNCOMPLETED; + } + return SPECIAL; + } + // Expecting 1 character: + default: + return SPECIAL; + } +} + +// ESC P ... ESC BACKSLASH +TerminalInputParser::Output TerminalInputParser::ParseDCS() { + // Parse until the string terminator ST. + while (true) { + if (!Eat()) { + return UNCOMPLETED; + } + + if (Current() != '\x1B') { + continue; + } + + if (!Eat()) { + return UNCOMPLETED; + } + + if (Current() != '\\') { + continue; + } + + if (pending_.size() == 10 && // + pending_[2] == '1' && // + pending_[3] == '$' && // + pending_[4] == 'r' && // + true) { + Output output(CURSOR_SHAPE); + output.cursor_shape = pending_[5] - '0'; + return output; + } + + return SPECIAL; + } +} + +TerminalInputParser::Output TerminalInputParser::ParseCSI() { + bool altered = false; + int argument = 0; + std::vector<int> arguments; + while (true) { + if (!Eat()) { + return UNCOMPLETED; + } + + if (Current() == '<') { + altered = true; + continue; + } + + if (Current() >= '0' && Current() <= '9') { + argument *= 10; // NOLINT + argument += Current() - '0'; + continue; + } + + if (Current() == ';') { + arguments.push_back(argument); + argument = 0; + continue; + } + + // CSI is terminated by a character in the range 0x40–0x7E + // (ASCII @A–Z[\]^_`a–z{|}~), + if (Current() >= '@' && Current() <= '~' && + // Note: I don't remember why we exclude '<' + Current() != '<' && + // To handle F1-F4, we exclude '['. + Current() != '[') { + arguments.push_back(argument); + argument = 0; // NOLINT + + switch (Current()) { + case 'M': + return ParseMouse(altered, true, std::move(arguments)); + case 'm': + return ParseMouse(altered, false, std::move(arguments)); + case 'R': + return ParseCursorPosition(std::move(arguments)); + default: + return SPECIAL; + } + } + + // Invalid ESC in CSI. + if (Current() == '\x1B') { + return SPECIAL; + } + } +} + +TerminalInputParser::Output TerminalInputParser::ParseOSC() { + // Parse until the string terminator ST. + while (true) { + if (!Eat()) { + return UNCOMPLETED; + } + if (Current() != '\x1B') { + continue; + } + if (!Eat()) { + return UNCOMPLETED; + } + if (Current() != '\\') { + continue; + } + return SPECIAL; + } +} + +TerminalInputParser::Output TerminalInputParser::ParseMouse( // NOLINT + bool altered, + bool pressed, + std::vector<int> arguments) { + if (arguments.size() != 3) { + return SPECIAL; + } + + (void)altered; + + Output output(MOUSE); + output.mouse.motion = Mouse::Motion(pressed); // NOLINT + + // Bits value Modifer Comment + // ---- ----- ------- --------- + // 0 1 1 2 button 0 = Left, 1 = Middle, 2 = Right, 3 = Release + // 2 4 Shift + // 3 8 Meta + // 4 16 Control + // 5 32 Move + // 6 64 Wheel + + // clang-format off + const int button = arguments[0] & (1 + 2); // NOLINT + const bool is_shift = arguments[0] & 4; // NOLINT + const bool is_meta = arguments[0] & 8; // NOLINT + const bool is_control = arguments[0] & 16; // NOLINT + const bool is_move = arguments[0] & 32; // NOLINT + const bool is_wheel = arguments[0] & 64; // NOLINT + // clang-format on + + output.mouse.motion = is_move ? Mouse::Moved : Mouse::Motion(pressed); + output.mouse.button = is_wheel ? Mouse::Button(Mouse::WheelUp + button) // + : Mouse::Button(button); + output.mouse.shift = is_shift; + output.mouse.meta = is_meta; + output.mouse.control = is_control; + output.mouse.x = arguments[1]; // NOLINT + output.mouse.y = arguments[2]; // NOLINT + + // Motion event. + return output; +} + +// NOLINTNEXTLINE +TerminalInputParser::Output TerminalInputParser::ParseCursorPosition( + std::vector<int> arguments) { + if (arguments.size() != 2) { + return SPECIAL; + } + Output output(CURSOR_POSITION); + output.cursor.y = arguments[0]; // NOLINT + output.cursor.x = arguments[1]; // NOLINT + return output; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/terminal_input_parser.hpp b/contrib/libs/ftxui/src/ftxui/component/terminal_input_parser.hpp new file mode 100644 index 00000000000..524994a2f0e --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/terminal_input_parser.hpp @@ -0,0 +1,73 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_COMPONENT_TERMINAL_INPUT_PARSER +#define FTXUI_COMPONENT_TERMINAL_INPUT_PARSER + +#include <string> // for string +#include <vector> // for vector + +#include "ftxui/component/mouse.hpp" // for Mouse +#include "ftxui/component/receiver.hpp" // for Sender +#include "ftxui/component/task.hpp" // for Task + +namespace ftxui { +struct Event; + +// Parse a sequence of |char| accross |time|. Produces |Event|. +class TerminalInputParser { + public: + explicit TerminalInputParser(Sender<Task> out); + void Timeout(int time); + void Add(char c); + + private: + unsigned char Current(); + bool Eat(); + + enum Type { + UNCOMPLETED, + DROP, + CHARACTER, + MOUSE, + CURSOR_POSITION, + CURSOR_SHAPE, + SPECIAL, + }; + + struct CursorPosition { + int x; + int y; + }; + + struct Output { + Type type; + union { + Mouse mouse; + CursorPosition cursor{}; + int cursor_shape; + }; + + Output(Type t) // NOLINT + : type(t) {} + }; + + void Send(Output output); + Output Parse(); + Output ParseUTF8(); + Output ParseESC(); + Output ParseDCS(); + Output ParseCSI(); + Output ParseOSC(); + Output ParseMouse(bool altered, bool pressed, std::vector<int> arguments); + Output ParseCursorPosition(std::vector<int> arguments); + + Sender<Task> out_; + int position_ = -1; + int timeout_ = 0; + std::string pending_; +}; + +} // namespace ftxui + +#endif /* end of include guard: FTXUI_COMPONENT_TERMINAL_INPUT_PARSER */ diff --git a/contrib/libs/ftxui/src/ftxui/component/util.cpp b/contrib/libs/ftxui/src/ftxui/component/util.cpp new file mode 100644 index 00000000000..fa100696d03 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/util.cpp @@ -0,0 +1,33 @@ +// Copyright 2022 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <functional> // for function + +#include "ftxui/component/component.hpp" // for Renderer, ComponentDecorator, ElementDecorator, operator|, operator|= +#include "ftxui/component/component_base.hpp" // for Component + +namespace ftxui { + +// NOLINTNEXTLINE +Component operator|(Component component, ComponentDecorator decorator) { + return decorator(component); // NOLINT +} + +// NOLINTNEXTLINE +Component operator|(Component component, ElementDecorator decorator) { + return component | Renderer(decorator); // NOLINT +} + +// NOLINTNEXTLINE +Component& operator|=(Component& component, ComponentDecorator decorator) { + component = component | decorator; // NOLINT + return component; +} + +// NOLINTNEXTLINE +Component& operator|=(Component& component, ElementDecorator decorator) { + component = component | decorator; // NOLINT + return component; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/component/window.cpp b/contrib/libs/ftxui/src/ftxui/component/window.cpp new file mode 100644 index 00000000000..76907817305 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/component/window.cpp @@ -0,0 +1,316 @@ +// Copyright 2023 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#define NOMINMAX +#include <algorithm> +#include <ftxui/component/component.hpp> +#include <ftxui/component/component_base.hpp> +#include <ftxui/component/component_options.hpp> +#include <ftxui/component/screen_interactive.hpp> // for ScreenInteractive +#include <memory> +#include <utility> +#include "ftxui/dom/elements.hpp" // for text, window, hbox, vbox, size, clear_under, reflect, emptyElement +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/color.hpp" // for Color +#include "ftxui/screen/screen.hpp" // for Screen + +namespace ftxui { + +namespace { + +Decorator PositionAndSize(int left, int top, int width, int height) { + return [=](Element element) { + element |= size(WIDTH, EQUAL, width); + element |= size(HEIGHT, EQUAL, height); + + auto padding_left = emptyElement() | size(WIDTH, EQUAL, left); + auto padding_top = emptyElement() | size(HEIGHT, EQUAL, top); + + return vbox({ + padding_top, + hbox({ + padding_left, + element, + }), + }); + }; +} + +class ResizeDecorator : public NodeDecorator { + public: + ResizeDecorator(Element child, + bool resize_left, + bool resize_right, + bool resize_top, + bool resize_down, + Color color) + : NodeDecorator(std::move(child)), + color_(color), + resize_left_(resize_left), + resize_right_(resize_right), + resize_top_(resize_top), + resize_down_(resize_down) {} + + void Render(Screen& screen) override { + NodeDecorator::Render(screen); + + if (resize_left_) { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + auto& cell = screen.PixelAt(box_.x_min, y); + cell.foreground_color = color_; + cell.automerge = false; + } + } + if (resize_right_) { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + auto& cell = screen.PixelAt(box_.x_max, y); + cell.foreground_color = color_; + cell.automerge = false; + } + } + if (resize_top_) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + auto& cell = screen.PixelAt(x, box_.y_min); + cell.foreground_color = color_; + cell.automerge = false; + } + } + if (resize_down_) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + auto& cell = screen.PixelAt(x, box_.y_max); + cell.foreground_color = color_; + cell.automerge = false; + } + } + } + + Color color_; + const bool resize_left_; + const bool resize_right_; + const bool resize_top_; + const bool resize_down_; +}; + +Element DefaultRenderState(const WindowRenderState& state) { + Element element = state.inner; + if (!state.active) { + element |= dim; + } + + element = window(text(state.title), element); + element |= clear_under; + + const Color color = Color::Red; + + element = std::make_shared<ResizeDecorator>( // + element, // + state.hover_left, // + state.hover_right, // + state.hover_top, // + state.hover_down, // + color // + ); + + return element; +} + +class WindowImpl : public ComponentBase, public WindowOptions { + public: + explicit WindowImpl(WindowOptions option) : WindowOptions(std::move(option)) { + if (!inner) { + inner = Make<ComponentBase>(); + } + Add(inner); + } + + private: + Element OnRender() final { + auto element = ComponentBase::Render(); + + const bool captureable = + captured_mouse_ || ScreenInteractive::Active()->CaptureMouse(); + + const WindowRenderState state = { + element, + title(), + Active(), + drag_, + resize_left_ || resize_right_ || resize_down_ || resize_top_, + (resize_left_hover_ || resize_left_) && captureable, + (resize_right_hover_ || resize_right_) && captureable, + (resize_top_hover_ || resize_top_) && captureable, + (resize_down_hover_ || resize_down_) && captureable, + }; + + element = render ? render(state) : DefaultRenderState(state); + + // Position and record the drawn area of the window. + element |= reflect(box_window_); + element |= PositionAndSize(left(), top(), width(), height()); + element |= reflect(box_); + + return element; + } + + bool OnEvent(Event event) final { + if (ComponentBase::OnEvent(event)) { + return true; + } + + if (!event.is_mouse()) { + return false; + } + + mouse_hover_ = box_window_.Contain(event.mouse().x, event.mouse().y); + + resize_down_hover_ = false; + resize_top_hover_ = false; + resize_left_hover_ = false; + resize_right_hover_ = false; + + if (mouse_hover_) { + resize_left_hover_ = event.mouse().x == left() + box_.x_min; + resize_right_hover_ = + event.mouse().x == left() + width() - 1 + box_.x_min; + resize_top_hover_ = event.mouse().y == top() + box_.y_min; + resize_down_hover_ = event.mouse().y == top() + height() - 1 + box_.y_min; + + // Apply the component options: + resize_top_hover_ &= resize_top(); + resize_left_hover_ &= resize_left(); + resize_down_hover_ &= resize_down(); + resize_right_hover_ &= resize_right(); + } + + if (captured_mouse_) { + if (event.mouse().motion == Mouse::Released) { + captured_mouse_ = nullptr; + return true; + } + + if (resize_left_) { + width() = left() + width() - event.mouse().x + box_.x_min; + left() = event.mouse().x - box_.x_min; + } + + if (resize_right_) { + width() = event.mouse().x - resize_start_x - box_.x_min; + } + + if (resize_top_) { + height() = top() + height() - event.mouse().y + box_.y_min; + top() = event.mouse().y - box_.y_min; + } + + if (resize_down_) { + height() = event.mouse().y - resize_start_y - box_.y_min; + } + + if (drag_) { + left() = event.mouse().x - drag_start_x - box_.x_min; + top() = event.mouse().y - drag_start_y - box_.y_min; + } + + // Clamp the window size. + width() = std::max<int>(width(), static_cast<int>(title().size() + 2)); + height() = std::max<int>(height(), 2); + + return true; + } + + resize_left_ = false; + resize_right_ = false; + resize_top_ = false; + resize_down_ = false; + + if (!mouse_hover_) { + return false; + } + + if (!CaptureMouse(event)) { + return true; + } + + if (event.mouse().button != Mouse::Left) { + return true; + } + if (event.mouse().motion != Mouse::Pressed) { + return true; + } + + TakeFocus(); + + captured_mouse_ = CaptureMouse(event); + if (!captured_mouse_) { + return true; + } + + resize_left_ = resize_left_hover_; + resize_right_ = resize_right_hover_; + resize_top_ = resize_top_hover_; + resize_down_ = resize_down_hover_; + + resize_start_x = event.mouse().x - width() - box_.x_min; + resize_start_y = event.mouse().y - height() - box_.y_min; + drag_start_x = event.mouse().x - left() - box_.x_min; + drag_start_y = event.mouse().y - top() - box_.y_min; + + // Drag only if we are not resizeing a border yet: + drag_ = !resize_right_ && !resize_down_ && !resize_top_ && !resize_left_; + return true; + } + + Box box_; + Box box_window_; + + CapturedMouse captured_mouse_; + int drag_start_x = 0; + int drag_start_y = 0; + int resize_start_x = 0; + int resize_start_y = 0; + + bool mouse_hover_ = false; + bool drag_ = false; + bool resize_top_ = false; + bool resize_left_ = false; + bool resize_down_ = false; + bool resize_right_ = false; + + bool resize_top_hover_ = false; + bool resize_left_hover_ = false; + bool resize_down_hover_ = false; + bool resize_right_hover_ = false; +}; + +} // namespace + +/// @brief A draggeable / resizeable window. To use multiple of them, they must +/// be stacked using `Container::Stacked({...})` component; +/// +/// @param option A struct holding every parameters. +/// @ingroup component +/// @see Window +/// +/// ### Example +/// +/// ```cpp +/// auto window_1= Window({ +/// .inner = DummyWindowContent(), +/// .title = "First window", +/// }); +/// +/// auto window_2= Window({ +/// .inner = DummyWindowContent(), +/// .title = "Second window", +/// }); +/// +/// auto container = Container::Stacked({ +/// window_1, +/// window_2, +/// }); +/// ``` +Component Window(WindowOptions option) { + return Make<WindowImpl>(std::move(option)); +} + +}; // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/automerge.cpp b/contrib/libs/ftxui/src/ftxui/dom/automerge.cpp new file mode 100644 index 00000000000..d8456281f93 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/automerge.cpp @@ -0,0 +1,35 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <memory> // for make_shared +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, automerge +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Pixel, Screen + +namespace ftxui { + +/// @brief Enable character to be automatically merged with others nearby. +/// @ingroup dom +Element automerge(Element child) { + class Impl : public NodeDecorator { + public: + using NodeDecorator::NodeDecorator; + + void Render(Screen& screen) override { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, y).automerge = true; + } + } + Node::Render(screen); + } + }; + + return std::make_shared<Impl>(std::move(child)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/blink.cpp b/contrib/libs/ftxui/src/ftxui/dom/blink.cpp new file mode 100644 index 00000000000..9a594a4732c --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/blink.cpp @@ -0,0 +1,37 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <memory> // for make_shared +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, blink +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Pixel, Screen + +namespace ftxui { + +namespace { +class Blink : public NodeDecorator { + public: + using NodeDecorator::NodeDecorator; + + void Render(Screen& screen) override { + Node::Render(screen); + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, y).blink = true; + } + } + } +}; +} // namespace + +/// @brief The text drawn alternates in between visible and hidden. +/// @ingroup dom +Element blink(Element child) { + return std::make_shared<Blink>(std::move(child)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/bold.cpp b/contrib/libs/ftxui/src/ftxui/dom/bold.cpp new file mode 100644 index 00000000000..eee109245dc --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/bold.cpp @@ -0,0 +1,37 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <memory> // for make_shared +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, bold +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Pixel, Screen + +namespace ftxui { + +namespace { +class Bold : public NodeDecorator { + public: + using NodeDecorator::NodeDecorator; + + void Render(Screen& screen) override { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, y).bold = true; + } + } + Node::Render(screen); + } +}; +} // namespace + +/// @brief Use a bold font, for elements with more emphasis. +/// @ingroup dom +Element bold(Element child) { + return std::make_shared<Bold>(std::move(child)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/border.cpp b/contrib/libs/ftxui/src/ftxui/dom/border.cpp new file mode 100644 index 00000000000..eb793b21fa5 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/border.cpp @@ -0,0 +1,511 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for max +#include <array> // for array +#include <ftxui/screen/color.hpp> // for Color +#include <memory> // for allocator, make_shared, __shared_ptr_access +#include <optional> // for optional, nullopt +#include <string> // for basic_string, string +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for unpack, Element, Decorator, BorderStyle, ROUNDED, borderStyled, Elements, DASHED, DOUBLE, EMPTY, HEAVY, LIGHT, border, borderDashed, borderDouble, borderEmpty, borderHeavy, borderLight, borderRounded, borderWith, window +#include "ftxui/dom/node.hpp" // for Node, Elements +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/pixel.hpp" // for Pixel +#include "ftxui/screen/screen.hpp" // for Pixel, Screen + +namespace ftxui { + +namespace { +using Charset = std::array<std::string, 6>; // NOLINT +using Charsets = std::array<Charset, 6>; // NOLINT +// NOLINTNEXTLINE +static Charsets simple_border_charset = { + Charset{"┌", "┐", "└", "┘", "─", "│"}, // LIGHT + Charset{"┏", "┓", "┗", "┛", "╍", "╏"}, // DASHED + Charset{"┏", "┓", "┗", "┛", "━", "┃"}, // HEAVY + Charset{"╔", "╗", "╚", "╝", "═", "║"}, // DOUBLE + Charset{"╭", "╮", "╰", "╯", "─", "│"}, // ROUNDED + Charset{" ", " ", " ", " ", " ", " "}, // EMPTY +}; + +// For reference, here is the charset for normal border: +class Border : public Node { + public: + Border(Elements children, + BorderStyle style, + std::optional<Color> foreground_color = std::nullopt) + : Node(std::move(children)), + charset_(simple_border_charset[style]) // NOLINT + , + foreground_color_(foreground_color) {} // NOLINT + + const Charset& charset_; // NOLINT + std::optional<Color> foreground_color_; + + void ComputeRequirement() override { + Node::ComputeRequirement(); + requirement_ = children_[0]->requirement(); + requirement_.min_x += 2; + requirement_.min_y += 2; + if (children_.size() == 2) { + requirement_.min_x = + std::max(requirement_.min_x, children_[1]->requirement().min_x + 2); + } + requirement_.focused.box.x_min++; + requirement_.focused.box.x_max++; + requirement_.focused.box.y_min++; + requirement_.focused.box.y_max++; + } + + void SetBox(Box box) override { + Node::SetBox(box); + if (children_.size() == 2) { + Box title_box; + title_box.x_min = box.x_min + 1; + title_box.x_max = std::min(box.x_max - 1, + box.x_min + children_[1]->requirement().min_x); + title_box.y_min = box.y_min; + title_box.y_max = box.y_min; + children_[1]->SetBox(title_box); + } + box.x_min++; + box.x_max--; + box.y_min++; + box.y_max--; + children_[0]->SetBox(box); + } + + void Render(Screen& screen) override { + // Draw content. + children_[0]->Render(screen); + + // Draw the border. + if (box_.x_min >= box_.x_max || box_.y_min >= box_.y_max) { + return; + } + + screen.at(box_.x_min, box_.y_min) = charset_[0]; // NOLINT + screen.at(box_.x_max, box_.y_min) = charset_[1]; // NOLINT + screen.at(box_.x_min, box_.y_max) = charset_[2]; // NOLINT + screen.at(box_.x_max, box_.y_max) = charset_[3]; // NOLINT + + for (int x = box_.x_min + 1; x < box_.x_max; ++x) { + Pixel& p1 = screen.PixelAt(x, box_.y_min); + Pixel& p2 = screen.PixelAt(x, box_.y_max); + p1.character = charset_[4]; // NOLINT + p2.character = charset_[4]; // NOLINT + p1.automerge = true; + p2.automerge = true; + } + for (int y = box_.y_min + 1; y < box_.y_max; ++y) { + Pixel& p3 = screen.PixelAt(box_.x_min, y); + Pixel& p4 = screen.PixelAt(box_.x_max, y); + p3.character = charset_[5]; // NOLINT + p4.character = charset_[5]; // NOLINT + p3.automerge = true; + p4.automerge = true; + } + + // Draw title. + if (children_.size() == 2) { + children_[1]->Render(screen); + } + + // Draw the border color. + if (foreground_color_) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, box_.y_min).foreground_color = *foreground_color_; + screen.PixelAt(x, box_.y_max).foreground_color = *foreground_color_; + } + for (int y = box_.y_min; y <= box_.y_max; ++y) { + screen.PixelAt(box_.x_min, y).foreground_color = *foreground_color_; + screen.PixelAt(box_.x_max, y).foreground_color = *foreground_color_; + } + } + } +}; + +// For reference, here is the charset for normal border: +class BorderPixel : public Node { + public: + BorderPixel(Elements children, Pixel pixel) + : Node(std::move(children)), pixel_(std::move(pixel)) {} + + private: + Pixel pixel_; + + void ComputeRequirement() override { + Node::ComputeRequirement(); + requirement_ = children_[0]->requirement(); + requirement_.min_x += 2; + requirement_.min_y += 2; + if (children_.size() == 2) { + requirement_.min_x = + std::max(requirement_.min_x, children_[1]->requirement().min_x + 2); + } + + requirement_.focused.box.Shift(1, 1); + } + + void SetBox(Box box) override { + Node::SetBox(box); + if (children_.size() == 2) { + Box title_box; + title_box.x_min = box.x_min + 1; + title_box.x_max = box.x_max - 1; + title_box.y_min = box.y_min; + title_box.y_max = box.y_min; + children_[1]->SetBox(title_box); + } + box.x_min++; + box.x_max--; + box.y_min++; + box.y_max--; + children_[0]->SetBox(box); + } + + void Render(Screen& screen) override { + // Draw content. + children_[0]->Render(screen); + + // Draw the border. + if (box_.x_min >= box_.x_max || box_.y_min >= box_.y_max) { + return; + } + + screen.PixelAt(box_.x_min, box_.y_min) = pixel_; + screen.PixelAt(box_.x_max, box_.y_min) = pixel_; + screen.PixelAt(box_.x_min, box_.y_max) = pixel_; + screen.PixelAt(box_.x_max, box_.y_max) = pixel_; + + for (int x = box_.x_min + 1; x < box_.x_max; ++x) { + screen.PixelAt(x, box_.y_min) = pixel_; + screen.PixelAt(x, box_.y_max) = pixel_; + } + for (int y = box_.y_min + 1; y < box_.y_max; ++y) { + screen.PixelAt(box_.x_min, y) = pixel_; + screen.PixelAt(box_.x_max, y) = pixel_; + } + } +}; +} // namespace + +/// @brief Draw a border around the element. +/// @ingroup dom +/// @see border +/// @see borderLight +/// @see borderDashed +/// @see borderDouble +/// @see borderHeavy +/// @see borderEmpty +/// @see borderRounded +/// @see borderStyled +/// @see borderWith +/// +/// Add a border around an element +/// +/// ### Example +/// +/// ```cpp +/// // Use 'border' as a function... +/// Element document = border(text("The element")); +/// +/// // ...Or as a 'pipe'. +/// Element document = text("The element") | border; +/// ``` +/// +/// ### Output +/// +/// ```bash +/// ┌───────────┐ +/// │The element│ +/// └───────────┘ +/// ``` +Element border(Element child) { + return std::make_shared<Border>(unpack(std::move(child)), ROUNDED); +} + +/// @brief Same as border but with a constant Pixel around the element. +/// @ingroup dom +/// @see border +Decorator borderWith(const Pixel& pixel) { + return [pixel](Element child) { + return std::make_shared<BorderPixel>(unpack(std::move(child)), pixel); + }; +} + +/// @brief Same as border but with different styles. +/// @ingroup dom +/// @see border +Decorator borderStyled(BorderStyle style) { + return [style](Element child) { + return std::make_shared<Border>(unpack(std::move(child)), style); + }; +} + +/// @brief Same as border but with a foreground color. +/// @ingroup dom +/// @see border +Decorator borderStyled(Color foreground_color) { + return [foreground_color](Element child) { + return std::make_shared<Border>(unpack(std::move(child)), ROUNDED, + foreground_color); + }; +} + +/// @brief Same as border but with a foreground color and a different style +/// @ingroup dom +/// @see border +Decorator borderStyled(BorderStyle style, Color foreground_color) { + return [style, foreground_color](Element child) { + return std::make_shared<Border>(unpack(std::move(child)), style, + foreground_color); + }; +} + +/// @brief Draw a dashed border around the element. +/// @ingroup dom +/// @see border +/// @see borderLight +/// @see borderDashed +/// @see borderDouble +/// @see borderHeavy +/// @see borderRounded +/// @see borderEmpty +/// @see borderStyled +/// @see borderWith +/// +/// Add a border around an element +/// +/// ### Example +/// +/// ```cpp +/// // Use 'borderDash' as a function... +/// Element document = borderDash(text("The element")); +/// +/// // ...Or as a 'pipe'. +/// Element document = text("The element") | borderDAsh; +/// ``` +/// +/// ### Output +/// +/// ```bash +/// ┏╍╍╍╍╍╍╍╍╍╍╍╍╍╍┓ +/// ╏The element ╏ +/// ┗╍╍╍╍╍╍╍╍╍╍╍╍╍╍┛ +/// ``` +Element borderDashed(Element child) { + return std::make_shared<Border>(unpack(std::move(child)), DASHED); +} + +/// @brief Draw a light border around the element. +/// @ingroup dom +/// @see border +/// @see borderLight +/// @see borderDashed +/// @see borderDouble +/// @see borderHeavy +/// @see borderRounded +/// @see borderEmpty +/// @see borderStyled +/// @see borderWith +/// +/// Add a border around an element +/// +/// ### Example +/// +/// ```cpp +/// // Use 'borderLight' as a function... +/// Element document = borderLight(text("The element")); +/// +/// // ...Or as a 'pipe'. +/// Element document = text("The element") | borderLight; +/// ``` +/// +/// ### Output +/// +/// ```bash +/// ┌──────────────┐ +/// │The element │ +/// └──────────────┘ +/// ``` +Element borderLight(Element child) { + return std::make_shared<Border>(unpack(std::move(child)), LIGHT); +} + +/// @brief Draw a heavy border around the element. +/// @ingroup dom +/// @see border +/// @see borderLight +/// @see borderDashed +/// @see borderDouble +/// @see borderHeavy +/// @see borderRounded +/// @see borderEmpty +/// @see borderStyled +/// @see borderWith +/// +/// Add a border around an element +/// +/// ### Example +/// +/// ```cpp +/// // Use 'borderHeavy' as a function... +/// Element document = borderHeavy(text("The element")); +/// +/// // ...Or as a 'pipe'. +/// Element document = text("The element") | borderHeavy; +/// ``` +/// +/// ### Output +/// +/// ```bash +/// ┏━━━━━━━━━━━━━━┓ +/// ┃The element ┃ +/// ┗━━━━━━━━━━━━━━┛ +/// ``` +Element borderHeavy(Element child) { + return std::make_shared<Border>(unpack(std::move(child)), HEAVY); +} + +/// @brief Draw a double border around the element. +/// @ingroup dom +/// @see border +/// @see borderLight +/// @see borderDashed +/// @see borderDouble +/// @see borderHeavy +/// @see borderRounded +/// @see borderEmpty +/// @see borderStyled +/// @see borderWith +/// +/// Add a border around an element +/// +/// ### Example +/// +/// ```cpp +/// // Use 'borderDouble' as a function... +/// Element document = borderDouble(text("The element")); +/// +/// // ...Or as a 'pipe'. +/// Element document = text("The element") | borderDouble; +/// ``` +/// +/// ### Output +/// +/// ```bash +/// ╔══════════════╗ +/// ║The element ║ +/// ╚══════════════╝ +/// ``` +Element borderDouble(Element child) { + return std::make_shared<Border>(unpack(std::move(child)), DOUBLE); +} + +/// @brief Draw a rounded border around the element. +/// @ingroup dom +/// @see border +/// @see borderLight +/// @see borderDashed +/// @see borderDouble +/// @see borderHeavy +/// @see borderRounded +/// @see borderEmpty +/// @see borderStyled +/// @see borderWith +/// +/// Add a border around an element +/// +/// ### Example +/// +/// ```cpp +/// // Use 'borderRounded' as a function... +/// Element document = borderRounded(text("The element")); +/// +/// // ...Or as a 'pipe'. +/// Element document = text("The element") | borderRounded; +/// ``` +/// +/// ### Output +/// +/// ```bash +/// ╭──────────────╮ +/// │The element │ +/// ╰──────────────╯ +/// ``` +Element borderRounded(Element child) { + return std::make_shared<Border>(unpack(std::move(child)), ROUNDED); +} + +/// @brief Draw an empty border around the element. +/// @ingroup dom +/// @see border +/// @see borderLight +/// @see borderDashed +/// @see borderDouble +/// @see borderHeavy +/// @see borderRounded +/// @see borderEmpty +/// @see borderStyled +/// @see borderWith +/// +/// Add a border around an element +/// +/// ### Example +/// +/// ```cpp +/// // Use 'borderRounded' as a function... +/// Element document = borderRounded(text("The element")); +/// +/// // ...Or as a 'pipe'. +/// Element document = text("The element") | borderRounded; +/// ``` +/// +/// ### Output +/// +/// ```bash +/// +/// The element +/// +/// ``` +Element borderEmpty(Element child) { + return std::make_shared<Border>(unpack(std::move(child)), EMPTY); +} + +/// @brief Draw window with a title and a border around the element. +/// @param title The title of the window. +/// @param content The element to be wrapped. +/// @param border The style of the border. Default is ROUNDED. +/// @ingroup dom +/// @see border +/// +/// ### Example +/// +/// ```cpp +/// Element document = window(text("Title"), +/// text("content") +/// ); +/// +/// // With specifying border +/// Element document = window(text("Title"), +/// text("content"), +/// ROUNDED +/// ); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// ┌Title──┐ +/// │content│ +/// └───────┘ +/// ``` +Element window(Element title, Element content, BorderStyle border) { + return std::make_shared<Border>(unpack(std::move(content), std::move(title)), + border); +} +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/box_helper.cpp b/contrib/libs/ftxui/src/ftxui/dom/box_helper.cpp new file mode 100644 index 00000000000..8284cf30941 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/box_helper.cpp @@ -0,0 +1,92 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include "ftxui/dom/box_helper.hpp" + +#include <algorithm> // for max +#include <vector> // for vector + +namespace ftxui::box_helper { + +namespace { +// Called when the size allowed is greater than the requested size. This +// distributes the extra spaces toward the flexible elements, in relative +// proportions. +void ComputeGrow(std::vector<Element>* elements, + int extra_space, + int flex_grow_sum) { + for (Element& element : *elements) { + const int added_space = + extra_space * element.flex_grow / std::max(flex_grow_sum, 1); + extra_space -= added_space; + flex_grow_sum -= element.flex_grow; + element.size = element.min_size + added_space; + } +} + +// Called when the size allowed is lower than the requested size, and the +// shrinkable element can absorbe the (negative) extra_space. This distribute +// the extra_space toward those. +void ComputeShrinkEasy(std::vector<Element>* elements, + int extra_space, + int flex_shrink_sum) { + for (Element& element : *elements) { + const int added_space = extra_space * element.min_size * + element.flex_shrink / std::max(flex_shrink_sum, 1); + extra_space -= added_space; + flex_shrink_sum -= element.flex_shrink * element.min_size; + element.size = element.min_size + added_space; + } +} + +// Called when the size allowed is lower than the requested size, and the +// shrinkable element can not absorbe the (negative) extra_space. This assign +// zero to shrinkable elements and distribute the remaining (negative) +// extra_space toward the other non shrinkable elements. +void ComputeShrinkHard(std::vector<Element>* elements, + int extra_space, + int size) { + for (Element& element : *elements) { + if (element.flex_shrink != 0) { + element.size = 0; + continue; + } + + const int added_space = extra_space * element.min_size / std::max(1, size); + extra_space -= added_space; + size -= element.min_size; + + element.size = element.min_size + added_space; + } +} + +} // namespace + +void Compute(std::vector<Element>* elements, int target_size) { + int size = 0; + int flex_grow_sum = 0; + int flex_shrink_sum = 0; + int flex_shrink_size = 0; + + for (auto& element : *elements) { + flex_grow_sum += element.flex_grow; + flex_shrink_sum += element.min_size * element.flex_shrink; + if (element.flex_shrink != 0) { + flex_shrink_size += element.min_size; + } + size += element.min_size; + } + + const int extra_space = target_size - size; + if (extra_space >= 0) { + ComputeGrow(elements, extra_space, flex_grow_sum); + } else if (flex_shrink_size + extra_space >= 0) { + ComputeShrinkEasy(elements, extra_space, flex_shrink_sum); + + } else { + ComputeShrinkHard(elements, extra_space + flex_shrink_size, + size - flex_shrink_size); + } +} + +} // namespace ftxui::box_helper diff --git a/contrib/libs/ftxui/src/ftxui/dom/box_helper.hpp b/contrib/libs/ftxui/src/ftxui/dom/box_helper.hpp new file mode 100644 index 00000000000..e7782dfa19c --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/box_helper.hpp @@ -0,0 +1,25 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file.line. +#ifndef FTXUI_DOM_BOX_HELPER_HPP +#define FTXUI_DOM_BOX_HELPER_HPP + +#include <vector> +#include "ftxui/dom/requirement.hpp" + +namespace ftxui::box_helper { + +struct Element { + // Input: + int min_size = 0; + int flex_grow = 0; + int flex_shrink = 0; + + // Output; + int size = 0; +}; + +void Compute(std::vector<Element>* elements, int target_size); +} // namespace ftxui::box_helper + +#endif /* end of include guard: FTXUI_DOM_BOX_HELPER_HPP */ diff --git a/contrib/libs/ftxui/src/ftxui/dom/canvas.cpp b/contrib/libs/ftxui/src/ftxui/dom/canvas.cpp new file mode 100644 index 00000000000..ef726721b61 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/canvas.cpp @@ -0,0 +1,943 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include "ftxui/dom/canvas.hpp" + +#include <algorithm> // for max, min +#include <cmath> // for abs +#include <cstdint> // for uint8_t +#include <cstdlib> // for abs +#include <ftxui/screen/color.hpp> // for Color +#include <functional> // for function +#include <map> // for map +#include <memory> // for make_shared +#include <utility> // for move, pair +#include <vector> // for vector + +#include "ftxui/dom/elements.hpp" // for Element, canvas +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/image.hpp" // for Image +#include "ftxui/screen/pixel.hpp" // for Pixel +#include "ftxui/screen/screen.hpp" // for Pixel, Screen +#include "ftxui/screen/string.hpp" // for Utf8ToGlyphs +#include "ftxui/util/ref.hpp" // for ConstRef + +namespace ftxui { + +namespace { + +// Base UTF8 pattern: +// 11100010 10100000 10000000 // empty + +// Pattern for the individual dots: +// ┌──────┬───────┐ +// │dot1 │ dot4 │ +// ├──────┼───────┤ +// │dot2 │ dot5 │ +// ├──────┼───────┤ +// │dot3 │ dot6 │ +// ├──────┼───────┤ +// │dot0-1│ dot0-2│ +// └──────┴───────┘ +// 11100010 10100000 10000001 // dot1 +// 11100010 10100000 10000010 // dot2 +// 11100010 10100000 10000100 // dot3 +// 11100010 10100001 10000000 // dot0-1 +// 11100010 10100000 10001000 // dot4 +// 11100010 10100000 10010000 // dot5 +// 11100010 10100000 10100000 // dot6 +// 11100010 10100010 10000000 // dot0-2 + +// NOLINTNEXTLINE +uint8_t g_map_braille[2][4][2] = { + { + {0b00000000, 0b00000001}, // NOLINT | dot1 + {0b00000000, 0b00000010}, // NOLINT | dot2 + {0b00000000, 0b00000100}, // NOLINT | dot3 + {0b00000001, 0b00000000}, // NOLINT | dot0-1 + }, + { + {0b00000000, 0b00001000}, // NOLINT | dot4 + {0b00000000, 0b00010000}, // NOLINT | dot5 + {0b00000000, 0b00100000}, // NOLINT | dot6 + {0b00000010, 0b00000000}, // NOLINT | dot0-2 + }, +}; + +// NOLINTNEXTLINE +std::vector<std::string> g_map_block = { + " ", "▘", "▖", "▌", "▝", "▀", "▞", "▛", + "▗", "▚", "▄", "▙", "▐", "▜", "▟", "█", +}; + +// NOLINTNEXTLINE +const std::map<std::string, uint8_t> g_map_block_inversed = { + {" ", 0b0000}, {"▘", 0b0001}, {"▖", 0b0010}, {"▌", 0b0011}, + {"▝", 0b0100}, {"▀", 0b0101}, {"▞", 0b0110}, {"▛", 0b0111}, + {"▗", 0b1000}, {"▚", 0b1001}, {"▄", 0b1010}, {"▙", 0b1011}, + {"▐", 0b1100}, {"▜", 0b1101}, {"▟", 0b1110}, {"█", 0b1111}, +}; + +constexpr auto nostyle = [](Pixel& /*pixel*/) {}; + +} // namespace + +/// @brief Constructor. +/// @param width the width of the canvas. A cell is a 2x4 braille dot. +/// @param height the height of the canvas. A cell is a 2x4 braille dot. +Canvas::Canvas(int width, int height) + : width_(width), + height_(height), + storage_(width_ * height_ / 8 /* NOLINT */) {} + +/// @brief Get the content of a cell. +/// @param x the x coordinate of the cell. +/// @param y the y coordinate of the cell. +Pixel Canvas::GetPixel(int x, int y) const { + auto it = storage_.find(XY{x, y}); + return (it == storage_.end()) ? Pixel() : it->second.content; +} + +/// @brief Draw a braille dot. +/// @param x the x coordinate of the dot. +/// @param y the y coordinate of the dot. +/// @param value whether the dot is filled or not. +void Canvas::DrawPoint(int x, int y, bool value) { + DrawPoint(x, y, value, [](Pixel& /*pixel*/) {}); +} + +/// @brief Draw a braille dot. +/// @param x the x coordinate of the dot. +/// @param y the y coordinate of the dot. +/// @param value whether the dot is filled or not. +/// @param color the color of the dot. +void Canvas::DrawPoint(int x, int y, bool value, const Color& color) { + DrawPoint(x, y, value, [color](Pixel& p) { p.foreground_color = color; }); +} + +/// @brief Draw a braille dot. +/// @param x the x coordinate of the dot. +/// @param y the y coordinate of the dot. +/// @param value whether the dot is filled or not. +/// @param style the style of the cell. +void Canvas::DrawPoint(int x, int y, bool value, const Stylizer& style) { + Style(x, y, style); + if (value) { + DrawPointOn(x, y); + } else { + DrawPointOff(x, y); + } +} + +/// @brief Draw a braille dot. +/// @param x the x coordinate of the dot. +/// @param y the y coordinate of the dot. +void Canvas::DrawPointOn(int x, int y) { + if (!IsIn(x, y)) { + return; + } + Cell& cell = storage_[XY{x / 2, y / 4}]; + if (cell.type != CellType::kBraille) { + cell.content.character = "⠀"; // 3 bytes. + cell.type = CellType::kBraille; + } + + cell.content.character[1] |= g_map_braille[x % 2][y % 4][0]; // NOLINT + cell.content.character[2] |= g_map_braille[x % 2][y % 4][1]; // NOLINT +} + +/// @brief Erase a braille dot. +/// @param x the x coordinate of the dot. +/// @param y the y coordinate of the dot. +void Canvas::DrawPointOff(int x, int y) { + if (!IsIn(x, y)) { + return; + } + Cell& cell = storage_[XY{x / 2, y / 4}]; + if (cell.type != CellType::kBraille) { + cell.content.character = "⠀"; // 3 byt + cell.type = CellType::kBraille; + } + + cell.content.character[1] &= ~(g_map_braille[x % 2][y % 4][0]); // NOLINT + cell.content.character[2] &= ~(g_map_braille[x % 2][y % 4][1]); // NOLINT +} + +/// @brief Toggle a braille dot. A filled one will be erased, and the other will +/// be drawn. +/// @param x the x coordinate of the dot. +/// @param y the y coordinate of the dot. +void Canvas::DrawPointToggle(int x, int y) { + if (!IsIn(x, y)) { + return; + } + Cell& cell = storage_[XY{x / 2, y / 4}]; + if (cell.type != CellType::kBraille) { + cell.content.character = "⠀"; // 3 byt + cell.type = CellType::kBraille; + } + + cell.content.character[1] ^= g_map_braille[x % 2][y % 4][0]; // NOLINT + cell.content.character[2] ^= g_map_braille[x % 2][y % 4][1]; // NOLINT +} + +/// @brief Draw a line made of braille dots. +/// @param x1 the x coordinate of the first dot. +/// @param y1 the y coordinate of the first dot. +/// @param x2 the x coordinate of the second dot. +/// @param y2 the y coordinate of the second dot. +void Canvas::DrawPointLine(int x1, int y1, int x2, int y2) { + DrawPointLine(x1, y1, x2, y2, [](Pixel& /*pixel*/) {}); +} + +/// @brief Draw a line made of braille dots. +/// @param x1 the x coordinate of the first dot. +/// @param y1 the y coordinate of the first dot. +/// @param x2 the x coordinate of the second dot. +/// @param y2 the y coordinate of the second dot. +/// @param color the color of the line. +void Canvas::DrawPointLine(int x1, int y1, int x2, int y2, const Color& color) { + DrawPointLine(x1, y1, x2, y2, + [color](Pixel& p) { p.foreground_color = color; }); +} + +/// @brief Draw a line made of braille dots. +/// @param x1 the x coordinate of the first dot. +/// @param y1 the y coordinate of the first dot.o +/// @param x2 the x coordinate of the second dot. +/// @param y2 the y coordinate of the second dot. +/// @param style the style of the line. +void Canvas::DrawPointLine(int x1, + int y1, + int x2, + int y2, + const Stylizer& style) { + const int dx = std::abs(x2 - x1); + const int dy = std::abs(y2 - y1); + const int sx = x1 < x2 ? 1 : -1; + const int sy = y1 < y2 ? 1 : -1; + const int length = std::max(dx, dy); + + if (!IsIn(x1, y1) && !IsIn(x2, y2)) { + return; + } + if (dx + dx > width_ * height_) { + return; + } + + int error = dx - dy; + for (int i = 0; i < length; ++i) { + DrawPoint(x1, y1, true, style); + if (2 * error >= -dy) { + error -= dy; + x1 += sx; + } + if (2 * error <= dx) { + error += dx; + y1 += sy; + } + } + DrawPoint(x2, y2, true, style); +} + +/// @brief Draw a circle made of braille dots. +/// @param x the x coordinate of the center of the circle. +/// @param y the y coordinate of the center of the circle. +/// @param radius the radius of the circle. +void Canvas::DrawPointCircle(int x, int y, int radius) { + DrawPointCircle(x, y, radius, [](Pixel& /*pixel*/) {}); +} + +/// @brief Draw a circle made of braille dots. +/// @param x the x coordinate of the center of the circle. +/// @param y the y coordinate of the center of the circle. +/// @param radius the radius of the circle. +/// @param color the color of the circle. +void Canvas::DrawPointCircle(int x, int y, int radius, const Color& color) { + DrawPointCircle(x, y, radius, + [color](Pixel& p) { p.foreground_color = color; }); +} + +/// @brief Draw a circle made of braille dots. +/// @param x the x coordinate of the center of the circle. +/// @param y the y coordinate of the center of the circle. +/// @param radius the radius of the circle. +/// @param style the style of the circle. +void Canvas::DrawPointCircle(int x, int y, int radius, const Stylizer& style) { + DrawPointEllipse(x, y, radius, radius, style); +} + +/// @brief Draw a filled circle made of braille dots. +/// @param x the x coordinate of the center of the circle. +/// @param y the y coordinate of the center of the circle. +/// @param radius the radius of the circle. +void Canvas::DrawPointCircleFilled(int x, int y, int radius) { + DrawPointCircleFilled(x, y, radius, [](Pixel& /*pixel*/) {}); +} + +/// @brief Draw a filled circle made of braille dots. +/// @param x the x coordinate of the center of the circle. +/// @param y the y coordinate of the center of the circle. +/// @param radius the radius of the circle. +/// @param color the color of the circle. +void Canvas::DrawPointCircleFilled(int x, + int y, + int radius, + const Color& color) { + DrawPointCircleFilled(x, y, radius, + [color](Pixel& p) { p.foreground_color = color; }); +} + +/// @brief Draw a filled circle made of braille dots. +/// @param x the x coordinate of the center of the circle. +/// @param y the y coordinate of the center of the circle. +/// @param radius the radius of the circle. +/// @param style the style of the circle. +void Canvas::DrawPointCircleFilled(int x, + int y, + int radius, + const Stylizer& style) { + DrawPointEllipseFilled(x, y, radius, radius, style); +} + +/// @brief Draw an ellipse made of braille dots. +/// @param x the x coordinate of the center of the ellipse. +/// @param y the y coordinate of the center of the ellipse. +/// @param r1 the radius of the ellipse along the x axis. +/// @param r2 the radius of the ellipse along the y axis. +void Canvas::DrawPointEllipse(int x, int y, int r1, int r2) { + DrawPointEllipse(x, y, r1, r2, [](Pixel& /*pixel*/) {}); +} + +/// @brief Draw an ellipse made of braille dots. +/// @param x the x coordinate of the center of the ellipse. +/// @param y the y coordinate of the center of the ellipse. +/// @param r1 the radius of the ellipse along the x axis. +/// @param r2 the radius of the ellipse along the y axis. +/// @param color the color of the ellipse. +void Canvas::DrawPointEllipse(int x, + int y, + int r1, + int r2, + const Color& color) { + DrawPointEllipse(x, y, r1, r2, + [color](Pixel& p) { p.foreground_color = color; }); +} + +/// @brief Draw an ellipse made of braille dots. +/// @param x1 the x coordinate of the center of the ellipse. +/// @param y1 the y coordinate of the center of the ellipse. +/// @param r1 the radius of the ellipse along the x axis. +/// @param r2 the radius of the ellipse along the y axis. +/// @param s the style of the ellipse. +void Canvas::DrawPointEllipse(int x1, + int y1, + int r1, + int r2, + const Stylizer& s) { + int x = -r1; + int y = 0; + int e2 = r2; + int dx = (1 + 2 * x) * e2 * e2; + int dy = x * x; + int err = dx + dy; + + do { // NOLINT + DrawPoint(x1 - x, y1 + y, true, s); + DrawPoint(x1 + x, y1 + y, true, s); + DrawPoint(x1 + x, y1 - y, true, s); + DrawPoint(x1 - x, y1 - y, true, s); + e2 = 2 * err; + if (e2 >= dx) { + x++; + err += dx += 2 * r2 * r2; + } + if (e2 <= dy) { + y++; + err += dy += 2 * r1 * r1; + } + } while (x <= 0); + + while (y++ < r2) { + DrawPoint(x1, y1 + y, true, s); + DrawPoint(x1, y1 - y, true, s); + } +} + +/// @brief Draw a filled ellipse made of braille dots. +/// @param x1 the x coordinate of the center of the ellipse. +/// @param y1 the y coordinate of the center of the ellipse. +/// @param r1 the radius of the ellipse along the x axis. +/// @param r2 the radius of the ellipse along the y axis. +void Canvas::DrawPointEllipseFilled(int x1, int y1, int r1, int r2) { + DrawPointEllipseFilled(x1, y1, r1, r2, [](Pixel& /*pixel*/) {}); +} + +/// @brief Draw a filled ellipse made of braille dots. +/// @param x1 the x coordinate of the center of the ellipse. +/// @param y1 the y coordinate of the center of the ellipse. +/// @param r1 the radius of the ellipse along the x axis. +/// @param r2 the radius of the ellipse along the y axis. +/// @param color the color of the ellipse. +void Canvas::DrawPointEllipseFilled(int x1, + int y1, + int r1, + int r2, + const Color& color) { + DrawPointEllipseFilled(x1, y1, r1, r2, + [color](Pixel& p) { p.foreground_color = color; }); +} + +/// @brief Draw a filled ellipse made of braille dots. +/// @param x1 the x coordinate of the center of the ellipse. +/// @param y1 the y coordinate of the center of the ellipse. +/// @param r1 the radius of the ellipse along the x axis. +/// @param r2 the radius of the ellipse along the y axis. +/// @param s the style of the ellipse. +void Canvas::DrawPointEllipseFilled(int x1, + int y1, + int r1, + int r2, + const Stylizer& s) { + int x = -r1; + int y = 0; + int e2 = r2; + int dx = (1 + 2 * x) * e2 * e2; + int dy = x * x; + int err = dx + dy; + + do { // NOLINT + for (int xx = x1 + x; xx <= x1 - x; ++xx) { + DrawPoint(xx, y1 + y, true, s); + DrawPoint(xx, y1 - y, true, s); + } + e2 = 2 * err; + if (e2 >= dx) { + x++; + err += dx += 2 * r2 * r2; + } + if (e2 <= dy) { + y++; + err += dy += 2 * r1 * r1; + } + } while (x <= 0); + + while (y++ < r2) { + for (int yy = y1 - y; yy <= y1 + y; ++yy) { + DrawPoint(x1, yy, true, s); + } + } +} + +/// @brief Draw a block. +/// @param x the x coordinate of the block. +/// @param y the y coordinate of the block. +/// @param value whether the block is filled or not. +void Canvas::DrawBlock(int x, int y, bool value) { + DrawBlock(x, y, value, [](Pixel& /*pixel*/) {}); +} + +/// @brief Draw a block. +/// @param x the x coordinate of the block. +/// @param y the y coordinate of the block. +/// @param value whether the block is filled or not. +/// @param color the color of the block. +void Canvas::DrawBlock(int x, int y, bool value, const Color& color) { + DrawBlock(x, y, value, [color](Pixel& p) { p.foreground_color = color; }); +} + +/// @brief Draw a block. +/// @param x the x coordinate of the block. +/// @param y the y coordinate of the block. +/// @param value whether the block is filled or not. +/// @param style the style of the block. +void Canvas::DrawBlock(int x, int y, bool value, const Stylizer& style) { + Style(x, y, style); + if (value) { + DrawBlockOn(x, y); + } else { + DrawBlockOff(x, y); + } +} + +/// @brief Draw a block. +/// @param x the x coordinate of the block. +/// @param y the y coordinate of the block. +void Canvas::DrawBlockOn(int x, int y) { + if (!IsIn(x, y)) { + return; + } + y /= 2; + Cell& cell = storage_[XY{x / 2, y / 2}]; + if (cell.type != CellType::kBlock) { + cell.content.character = " "; + cell.type = CellType::kBlock; + } + + const uint8_t bit = (x % 2) * 2 + y % 2; + uint8_t value = g_map_block_inversed.at(cell.content.character); + value |= 1U << bit; + cell.content.character = g_map_block[value]; +} + +/// @brief Erase a block. +/// @param x the x coordinate of the block. +/// @param y the y coordinate of the block. +void Canvas::DrawBlockOff(int x, int y) { + if (!IsIn(x, y)) { + return; + } + Cell& cell = storage_[XY{x / 2, y / 4}]; + if (cell.type != CellType::kBlock) { + cell.content.character = " "; + cell.type = CellType::kBlock; + } + y /= 2; + + const uint8_t bit = (y % 2) * 2 + x % 2; + uint8_t value = g_map_block_inversed.at(cell.content.character); + value &= ~(1U << bit); + cell.content.character = g_map_block[value]; +} + +/// @brief Toggle a block. If it is filled, it will be erased. If it is empty, +/// it will be filled. +/// @param x the x coordinate of the block. +/// @param y the y coordinate of the block. +void Canvas::DrawBlockToggle(int x, int y) { + if (!IsIn(x, y)) { + return; + } + Cell& cell = storage_[XY{x / 2, y / 4}]; + if (cell.type != CellType::kBlock) { + cell.content.character = " "; + cell.type = CellType::kBlock; + } + y /= 2; + + const uint8_t bit = (y % 2) * 2 + x % 2; + uint8_t value = g_map_block_inversed.at(cell.content.character); + value ^= 1U << bit; + cell.content.character = g_map_block[value]; +} + +/// @brief Draw a line made of block characters. +/// @param x1 the x coordinate of the first point of the line. +/// @param y1 the y coordinate of the first point of the line. +/// @param x2 the x coordinate of the second point of the line. +/// @param y2 the y coordinate of the second point of the line. +void Canvas::DrawBlockLine(int x1, int y1, int x2, int y2) { + DrawBlockLine(x1, y1, x2, y2, [](Pixel& /*pixel*/) {}); +} + +/// @brief Draw a line made of block characters. +/// @param x1 the x coordinate of the first point of the line. +/// @param y1 the y coordinate of the first point of the line. +/// @param x2 the x coordinate of the second point of the line. +/// @param y2 the y coordinate of the second point of the line. +/// @param color the color of the line. +void Canvas::DrawBlockLine(int x1, int y1, int x2, int y2, const Color& color) { + DrawBlockLine(x1, y1, x2, y2, + [color](Pixel& p) { p.foreground_color = color; }); +} + +/// @brief Draw a line made of block characters. +/// @param x1 the x coordinate of the first point of the line. +/// @param y1 the y coordinate of the first point of the line. +/// @param x2 the x coordinate of the second point of the line. +/// @param y2 the y coordinate of the second point of the line. +/// @param style the style of the line. +void Canvas::DrawBlockLine(int x1, + int y1, + int x2, + int y2, + const Stylizer& style) { + y1 /= 2; + y2 /= 2; + + const int dx = std::abs(x2 - x1); + const int dy = std::abs(y2 - y1); + const int sx = x1 < x2 ? 1 : -1; + const int sy = y1 < y2 ? 1 : -1; + const int length = std::max(dx, dy); + + if (!IsIn(x1, y1) && !IsIn(x2, y2)) { + return; + } + if (dx + dx > width_ * height_) { + return; + } + + int error = dx - dy; + for (int i = 0; i < length; ++i) { + DrawBlock(x1, y1 * 2, true, style); + if (2 * error >= -dy) { + error -= dy; + x1 += sx; + } + if (2 * error <= dx) { + error += dx; + y1 += sy; + } + } + DrawBlock(x2, y2 * 2, true, style); +} + +/// @brief Draw a circle made of block characters. +/// @param x the x coordinate of the center of the circle. +/// @param y the y coordinate of the center of the circle. +/// @param radius the radius of the circle. +void Canvas::DrawBlockCircle(int x, int y, int radius) { + DrawBlockCircle(x, y, radius, nostyle); +} + +/// @brief Draw a circle made of block characters. +/// @param x the x coordinate of the center of the circle. +/// @param y the y coordinate of the center of the circle. +/// @param radius the radius of the circle. +/// @param color the color of the circle. +void Canvas::DrawBlockCircle(int x, int y, int radius, const Color& color) { + DrawBlockCircle(x, y, radius, + [color](Pixel& p) { p.foreground_color = color; }); +} + +/// @brief Draw a circle made of block characters. +/// @param x the x coordinate of the center of the circle. +/// @param y the y coordinate of the center of the circle. +/// @param radius the radius of the circle. +/// @param style the style of the circle. +void Canvas::DrawBlockCircle(int x, int y, int radius, const Stylizer& style) { + DrawBlockEllipse(x, y, radius, radius, style); +} + +/// @brief Draw a filled circle made of block characters. +/// @param x the x coordinate of the center of the circle. +/// @param y the y coordinate of the center of the circle. +/// @param radius the radius of the circle. +void Canvas::DrawBlockCircleFilled(int x, int y, int radius) { + DrawBlockCircleFilled(x, y, radius, nostyle); +} + +/// @brief Draw a filled circle made of block characters. +/// @param x the x coordinate of the center of the circle. +/// @param y the y coordinate of the center of the circle. +/// @param radius the radius of the circle. +/// @param color the color of the circle. +void Canvas::DrawBlockCircleFilled(int x, + int y, + int radius, + const Color& color) { + DrawBlockCircleFilled(x, y, radius, + [color](Pixel& p) { p.foreground_color = color; }); +} + +/// @brief Draw a filled circle made of block characters. +/// @param x the x coordinate of the center of the circle. +/// @param y the y coordinate of the center of the circle. +/// @param radius the radius of the circle. +/// @param s the style of the circle. +void Canvas::DrawBlockCircleFilled(int x, + int y, + int radius, + const Stylizer& s) { + DrawBlockEllipseFilled(x, y, radius, radius, s); +} + +/// @brief Draw an ellipse made of block characters. +/// @param x the x coordinate of the center of the ellipse. +/// @param y the y coordinate of the center of the ellipse. +/// @param r1 the radius of the ellipse along the x axis. +/// @param r2 the radius of the ellipse along the y axis. +void Canvas::DrawBlockEllipse(int x, int y, int r1, int r2) { + DrawBlockEllipse(x, y, r1, r2, nostyle); +} + +/// @brief Draw an ellipse made of block characters. +/// @param x the x coordinate of the center of the ellipse. +/// @param y the y coordinate of the center of the ellipse. +/// @param r1 the radius of the ellipse along the x axis. +/// @param r2 the radius of the ellipse along the y axis. +/// @param color the color of the ellipse. +void Canvas::DrawBlockEllipse(int x, + int y, + int r1, + int r2, + const Color& color) { + DrawBlockEllipse(x, y, r1, r2, + [color](Pixel& p) { p.foreground_color = color; }); +} + +/// @brief Draw an ellipse made of block characters. +/// @param x1 the x coordinate of the center of the ellipse. +/// @param y1 the y coordinate of the center of the ellipse. +/// @param r1 the radius of the ellipse along the x axis. +/// @param r2 the radius of the ellipse along the y axis. +/// @param s the style of the ellipse. +void Canvas::DrawBlockEllipse(int x1, + int y1, + int r1, + int r2, + const Stylizer& s) { + y1 /= 2; + r2 /= 2; + int x = -r1; + int y = 0; + int e2 = r2; + int dx = (1 + 2 * x) * e2 * e2; + int dy = x * x; + int err = dx + dy; + + do { // NOLINT + DrawBlock(x1 - x, 2 * (y1 + y), true, s); + DrawBlock(x1 + x, 2 * (y1 + y), true, s); + DrawBlock(x1 + x, 2 * (y1 - y), true, s); + DrawBlock(x1 - x, 2 * (y1 - y), true, s); + e2 = 2 * err; + if (e2 >= dx) { + x++; + err += dx += 2 * r2 * r2; + } + if (e2 <= dy) { + y++; + err += dy += 2 * r1 * r1; + } + } while (x <= 0); + + while (y++ < r2) { + DrawBlock(x1, 2 * (y1 + y), true, s); + DrawBlock(x1, 2 * (y1 - y), true, s); + } +} + +/// @brief Draw a filled ellipse made of block characters. +/// @param x the x coordinate of the center of the ellipse. +/// @param y the y coordinate of the center of the ellipse. +/// @param r1 the radius of the ellipse along the x axis. +/// @param r2 the radius of the ellipse along the y axis. +void Canvas::DrawBlockEllipseFilled(int x, int y, int r1, int r2) { + DrawBlockEllipseFilled(x, y, r1, r2, nostyle); +} + +/// @brief Draw a filled ellipse made of block characters. +/// @param x the x coordinate of the center of the ellipse. +/// @param y the y coordinate of the center of the ellipse. +/// @param r1 the radius of the ellipse along the x axis. +/// @param r2 the radius of the ellipse along the y axis. +/// @param color the color of the ellipse. +void Canvas::DrawBlockEllipseFilled(int x, + int y, + int r1, + int r2, + const Color& color) { + DrawBlockEllipseFilled(x, y, r1, r2, + [color](Pixel& p) { p.foreground_color = color; }); +} + +/// @brief Draw a filled ellipse made of block characters. +/// @param x1 the x coordinate of the center of the ellipse. +/// @param y1 the y coordinate of the center of the ellipse. +/// @param r1 the radius of the ellipse along the x axis. +/// @param r2 the radius of the ellipse along the y axis. +/// @param s the style of the ellipse. +void Canvas::DrawBlockEllipseFilled(int x1, + int y1, + int r1, + int r2, + const Stylizer& s) { + y1 /= 2; + r2 /= 2; + int x = -r1; + int y = 0; + int e2 = r2; + int dx = (1 + 2 * x) * e2 * e2; + int dy = x * x; + int err = dx + dy; + + do { // NOLINT + for (int xx = x1 + x; xx <= x1 - x; ++xx) { + DrawBlock(xx, 2 * (y1 + y), true, s); + DrawBlock(xx, 2 * (y1 - y), true, s); + } + e2 = 2 * err; + if (e2 >= dx) { + x++; + err += dx += 2 * r2 * r2; + } + if (e2 <= dy) { + y++; + err += dy += 2 * r1 * r1; + } + } while (x <= 0); + + while (y++ < r2) { + for (int yy = y1 + y; yy <= y1 - y; ++yy) { + DrawBlock(x1, 2 * yy, true, s); + } + } +} + +/// @brief Draw a piece of text. +/// @param x the x coordinate of the text. +/// @param y the y coordinate of the text. +/// @param value the text to draw. +void Canvas::DrawText(int x, int y, const std::string& value) { + DrawText(x, y, value, nostyle); +} + +/// @brief Draw a piece of text. +/// @param x the x coordinate of the text. +/// @param y the y coordinate of the text. +/// @param value the text to draw. +/// @param color the color of the text. +void Canvas::DrawText(int x, + int y, + const std::string& value, + const Color& color) { + DrawText(x, y, value, [color](Pixel& p) { p.foreground_color = color; }); +} + +/// @brief Draw a piece of text. +/// @param x the x coordinate of the text. +/// @param y the y coordinate of the text. +/// @param value the text to draw. +/// @param style the style of the text. +void Canvas::DrawText(int x, + int y, + const std::string& value, + const Stylizer& style) { + for (const auto& it : Utf8ToGlyphs(value)) { + if (!IsIn(x, y)) { + x += 2; + continue; + } + Cell& cell = storage_[XY{x / 2, y / 4}]; + cell.type = CellType::kCell; + cell.content.character = it; + style(cell.content); + x += 2; + } +} + +/// @brief Directly draw a predefined pixel at the given coordinate +/// @param x the x coordinate of the pixel. +/// @param y the y coordinate of the pixel. +/// @param p the pixel to draw. +void Canvas::DrawPixel(int x, int y, const Pixel& p) { + Cell& cell = storage_[XY{x / 2, y / 4}]; + cell.type = CellType::kCell; + cell.content = p; +} + +/// @brief Draw a predefined image, with top-left corner at the given coordinate +/// You can supply negative coordinates to align the image however you like - +/// only the 'visible' portion will be drawn +/// @param x the x coordinate corresponding to the top-left corner of the image. +/// @param y the y coordinate corresponding to the top-left corner of the image. +/// @param image the image to draw. +void Canvas::DrawImage(int x, int y, const Image& image) { + x /= 2; + y /= 4; + const int dx_begin = std::max(0, -x); + const int dy_begin = std::max(0, -y); + const int dx_end = std::min(image.dimx(), width_ - x); + const int dy_end = std::min(image.dimy(), height_ - y); + + for (int dy = dy_begin; dy < dy_end; ++dy) { + for (int dx = dx_begin; dx < dx_end; ++dx) { + Cell& cell = storage_[XY{ + x + dx, + y + dy, + }]; + cell.type = CellType::kCell; + cell.content = image.PixelAt(dx, dy); + } + } +} + +/// @brief Modify a pixel at a given location. +/// @param style a function that modifies the pixel. +void Canvas::Style(int x, int y, const Stylizer& style) { + if (IsIn(x, y)) { + style(storage_[XY{x / 2, y / 4}].content); + } +} + +namespace { + +class CanvasNodeBase : public Node { + public: + CanvasNodeBase() = default; + + void Render(Screen& screen) override { + const Canvas& c = canvas(); + const int y_max = std::min(c.height() / 4, box_.y_max - box_.y_min + 1); + const int x_max = std::min(c.width() / 2, box_.x_max - box_.x_min + 1); + for (int y = 0; y < y_max; ++y) { + for (int x = 0; x < x_max; ++x) { + screen.PixelAt(box_.x_min + x, box_.y_min + y) = c.GetPixel(x, y); + } + } + } + + virtual const Canvas& canvas() = 0; +}; + +} // namespace + +/// @brief Produce an element from a Canvas, or a reference to a Canvas. +// NOLINTNEXTLINE +Element canvas(ConstRef<Canvas> canvas) { + class Impl : public CanvasNodeBase { + public: + explicit Impl(ConstRef<Canvas> canvas) : canvas_(std::move(canvas)) { + requirement_.min_x = (canvas_->width() + 1) / 2; + requirement_.min_y = (canvas_->height() + 3) / 4; + } + const Canvas& canvas() final { return *canvas_; } + ConstRef<Canvas> canvas_; + }; + return std::make_shared<Impl>(canvas); +} + +/// @brief Produce an element drawing a canvas of requested size. +/// @param width the width of the canvas. +/// @param height the height of the canvas. +/// @param fn a function drawing the canvas. +Element canvas(int width, int height, std::function<void(Canvas&)> fn) { + class Impl : public CanvasNodeBase { + public: + Impl(int width, int height, std::function<void(Canvas&)> fn) + : width_(width), height_(height), fn_(std::move(fn)) {} + + void ComputeRequirement() final { + requirement_.min_x = (width_ + 1) / 2; + requirement_.min_y = (height_ + 3) / 4; + } + + void Render(Screen& screen) final { + const int width = (box_.x_max - box_.x_min + 1) * 2; + const int height = (box_.y_max - box_.y_min + 1) * 4; + canvas_ = Canvas(width, height); + fn_(canvas_); + CanvasNodeBase::Render(screen); + } + + const Canvas& canvas() final { return canvas_; } + Canvas canvas_; + int width_; + int height_; + std::function<void(Canvas&)> fn_; + }; + return std::make_shared<Impl>(width, height, std::move(fn)); +} + +/// @brief Produce an element drawing a canvas. +/// @param fn a function drawing the canvas. +Element canvas(std::function<void(Canvas&)> fn) { + const int default_dim = 12; + return canvas(default_dim, default_dim, std::move(fn)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/clear_under.cpp b/contrib/libs/ftxui/src/ftxui/dom/clear_under.cpp new file mode 100644 index 00000000000..81dc2bfce14 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/clear_under.cpp @@ -0,0 +1,42 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <memory> // for make_shared +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, clear_under +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Pixel, Screen + +namespace ftxui { + +namespace { +using ftxui::Screen; + +class ClearUnder : public NodeDecorator { + public: + using NodeDecorator::NodeDecorator; + + void Render(Screen& screen) override { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, y) = Pixel(); + screen.PixelAt(x, y).character = " "; // Consider the pixel written. + } + } + Node::Render(screen); + } +}; +} // namespace + +/// @brief Before drawing |child|, clear the pixels below. This is useful in +// combinaison with dbox. +/// @see ftxui::dbox +/// @ingroup dom +Element clear_under(Element element) { + return std::make_shared<ClearUnder>(std::move(element)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/color.cpp b/contrib/libs/ftxui/src/ftxui/dom/color.cpp new file mode 100644 index 00000000000..431d522eb6a --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/color.cpp @@ -0,0 +1,128 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <memory> // for make_shared +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, Decorator, bgcolor, color +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/color.hpp" // for Color +#include "ftxui/screen/screen.hpp" // for Pixel, Screen + +namespace ftxui { + +namespace { +class BgColor : public NodeDecorator { + public: + BgColor(Element child, Color color) + : NodeDecorator(std::move(child)), color_(color) {} + + void Render(Screen& screen) override { + if (color_.IsOpaque()) { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, y).background_color = color_; + } + } + } else { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + Color& color = screen.PixelAt(x, y).background_color; + color = Color::Blend(color, color_); + } + } + } + NodeDecorator::Render(screen); + } + + Color color_; +}; + +class FgColor : public NodeDecorator { + public: + FgColor(Element child, Color color) + : NodeDecorator(std::move(child)), color_(color) {} + + void Render(Screen& screen) override { + if (color_.IsOpaque()) { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, y).foreground_color = color_; + } + } + } else { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + Color& color = screen.PixelAt(x, y).foreground_color; + color = Color::Blend(color, color_); + } + } + } + NodeDecorator::Render(screen); + } + + Color color_; +}; + +} // namespace + +/// @brief Set the foreground color of an element. +/// @param color The color of the output element. +/// @param child The input element. +/// @return The output element colored. +/// @ingroup dom +/// +/// ### Example +/// +/// ```cpp +/// Element document = color(Color::Green, text("Success")), +/// ``` +Element color(Color color, Element child) { + return std::make_shared<FgColor>(std::move(child), color); +} + +/// @brief Set the background color of an element. +/// @param color The color of the output element. +/// @param child The input element. +/// @return The output element colored. +/// @ingroup dom +/// +/// ### Example +/// +/// ```cpp +/// Element document = bgcolor(Color::Green, text("Success")), +/// ``` +Element bgcolor(Color color, Element child) { + return std::make_shared<BgColor>(std::move(child), color); +} + +/// @brief Decorate using a foreground color. +/// @param c The foreground color to be applied. +/// @return The Decorator applying the color. +/// @ingroup dom +/// +/// ### Example +/// +/// ```cpp +/// Element document = text("red") | color(Color::Red); +/// ``` +Decorator color(Color c) { + return [c](Element child) { return color(c, std::move(child)); }; +} + +/// @brief Decorate using a background color. +/// @param color The background color to be applied. +/// @return The Decorator applying the color. +/// @ingroup dom +/// +/// ### Example +/// +/// ```cpp +/// Element document = text("red") | bgcolor(Color::Red); +/// ``` +Decorator bgcolor(Color color) { + return [color](Element child) { return bgcolor(color, std::move(child)); }; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/composite_decorator.cpp b/contrib/libs/ftxui/src/ftxui/dom/composite_decorator.cpp new file mode 100644 index 00000000000..f7ce45d4050 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/composite_decorator.cpp @@ -0,0 +1,43 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, filler, operator|, hbox, flex_grow, vbox, xflex_grow, yflex_grow, align_right, center, hcenter, vcenter + +namespace ftxui { + +/// @brief Center an element horizontally. +/// @param child The decorated element. +/// @return The centered element. +/// @ingroup dom +Element hcenter(Element child) { + return hbox(filler(), std::move(child), filler()); +} + +/// @brief Center an element vertically. +/// @param child The decorated element. +/// @return The centered element. +/// @ingroup dom +Element vcenter(Element child) { + return vbox(filler(), std::move(child), filler()); +} + +/// @brief Center an element horizontally and vertically. +/// @param child The decorated element. +/// @return The centered element. +/// @ingroup dom +Element center(Element child) { + return hcenter(vcenter(std::move(child))); +} + +/// @brief Align an element on the right side. +/// @param child The decorated element. +/// @return The right aligned element. +/// @ingroup dom +Element align_right(Element child) { + return hbox(filler(), std::move(child)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/dbox.cpp b/contrib/libs/ftxui/src/ftxui/dom/dbox.cpp new file mode 100644 index 00000000000..2fd6fe116fa --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/dbox.cpp @@ -0,0 +1,112 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for max +#include <cstddef> // for size_t +#include <memory> // for __shared_ptr_access, shared_ptr, make_shared +#include <utility> // for move +#include <vector> + +#include "ftxui/dom/elements.hpp" // for Element, Elements, dbox +#include "ftxui/dom/node.hpp" // for Node, Elements +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/pixel.hpp" // for Pixel + +namespace ftxui { + +namespace { +class DBox : public Node { + public: + explicit DBox(Elements children) : Node(std::move(children)) {} + + void ComputeRequirement() override { + requirement_ = Requirement{}; + for (auto& child : children_) { + child->ComputeRequirement(); + + // Propagate the focused requirement. + if (requirement_.focused.Prefer(child->requirement().focused)) { + requirement_.focused = child->requirement().focused; + } + + // Extend the min_x and min_y to contain all the children + requirement_.min_x = + std::max(requirement_.min_x, child->requirement().min_x); + requirement_.min_y = + std::max(requirement_.min_y, child->requirement().min_y); + } + } + + void SetBox(Box box) override { + Node::SetBox(box); + + for (auto& child : children_) { + child->SetBox(box); + } + } + + void Render(Screen& screen) override { + if (children_.size() <= 1) { + Node::Render(screen); + return; + } + + const int width = box_.x_max - box_.x_min + 1; + const int height = box_.y_max - box_.y_min + 1; + std::vector<Pixel> pixels(std::size_t(width * height)); + + for (auto& child : children_) { + child->Render(screen); + + // Accumulate the pixels + Pixel* acc = pixels.data(); + for (int x = 0; x < width; ++x) { + for (int y = 0; y < height; ++y) { + auto& pixel = screen.PixelAt(x + box_.x_min, y + box_.y_min); + acc->background_color = + Color::Blend(acc->background_color, pixel.background_color); + acc->automerge = pixel.automerge || acc->automerge; + if (pixel.character.empty()) { + acc->foreground_color = + Color::Blend(acc->foreground_color, pixel.background_color); + } else { + acc->blink = pixel.blink; + acc->bold = pixel.bold; + acc->dim = pixel.dim; + acc->inverted = pixel.inverted; + acc->italic = pixel.italic; + acc->underlined = pixel.underlined; + acc->underlined_double = pixel.underlined_double; + acc->strikethrough = pixel.strikethrough; + acc->hyperlink = pixel.hyperlink; + acc->character = pixel.character; + acc->foreground_color = pixel.foreground_color; + } + ++acc; // NOLINT + + pixel = Pixel(); + } + } + } + + // Render the accumulated pixels: + Pixel* acc = pixels.data(); + for (int x = 0; x < width; ++x) { + for (int y = 0; y < height; ++y) { + screen.PixelAt(x + box_.x_min, y + box_.y_min) = *acc++; // NOLINT + } + } + } +}; +} // namespace + +/// @brief Stack several element on top of each other. +/// @param children_ The input element. +/// @return The right aligned element. +/// @ingroup dom +Element dbox(Elements children_) { + return std::make_shared<DBox>(std::move(children_)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/dim.cpp b/contrib/libs/ftxui/src/ftxui/dom/dim.cpp new file mode 100644 index 00000000000..7628b073b3d --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/dim.cpp @@ -0,0 +1,37 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <memory> // for make_shared +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, dim +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Pixel, Screen + +namespace ftxui { + +namespace { +class Dim : public NodeDecorator { + public: + using NodeDecorator::NodeDecorator; + + void Render(Screen& screen) override { + Node::Render(screen); + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, y).dim = true; + } + } + } +}; +} // namespace + +/// @brief Use a light font, for elements with less emphasis. +/// @ingroup dom +Element dim(Element child) { + return std::make_shared<Dim>(std::move(child)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/flex.cpp b/contrib/libs/ftxui/src/ftxui/dom/flex.cpp new file mode 100644 index 00000000000..b6f95c6e6c0 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/flex.cpp @@ -0,0 +1,181 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <memory> // for make_shared, __shared_ptr_access +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, unpack, filler, flex, flex_grow, flex_shrink, notflex, xflex, xflex_grow, xflex_shrink, yflex, yflex_grow, yflex_shrink +#include "ftxui/dom/node.hpp" // for Elements, Node +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/box.hpp" // for Box + +namespace ftxui { + +namespace { + +using FlexFunction = void (*)(Requirement&); + +void function_flex_grow(Requirement& r) { + r.flex_grow_x = 1; + r.flex_grow_y = 1; +} + +void function_xflex_grow(Requirement& r) { + r.flex_grow_x = 1; +} + +void function_yflex_grow(Requirement& r) { + r.flex_grow_y = 1; +} + +void function_flex_shrink(Requirement& r) { + r.flex_shrink_x = 1; + r.flex_shrink_y = 1; +} + +void function_xflex_shrink(Requirement& r) { + r.flex_shrink_x = 1; +} + +void function_yflex_shrink(Requirement& r) { + r.flex_shrink_y = 1; +} + +void function_flex(Requirement& r) { + r.flex_grow_x = 1; + r.flex_grow_y = 1; + r.flex_shrink_x = 1; + r.flex_shrink_y = 1; +} + +void function_xflex(Requirement& r) { + r.flex_grow_x = 1; + r.flex_shrink_x = 1; +} + +void function_yflex(Requirement& r) { + r.flex_grow_y = 1; + r.flex_shrink_y = 1; +} + +void function_not_flex(Requirement& r) { + r.flex_grow_x = 0; + r.flex_grow_y = 0; + r.flex_shrink_x = 0; + r.flex_shrink_y = 0; +} + +class Flex : public Node { + public: + explicit Flex(FlexFunction f) : f_(f) {} + Flex(FlexFunction f, Element child) : Node(unpack(std::move(child))), f_(f) {} + void ComputeRequirement() override { + requirement_.min_x = 0; + requirement_.min_y = 0; + if (!children_.empty()) { + children_[0]->ComputeRequirement(); + requirement_ = children_[0]->requirement(); + } + f_(requirement_); + } + + void SetBox(Box box) override { + Node::SetBox(box); + if (children_.empty()) { + return; + } + children_[0]->SetBox(box); + } + + FlexFunction f_; +}; + +} // namespace + +/// @brief An element that will take expand proportionally to the space left in +/// a container. +/// @ingroup dom +Element filler() { + return std::make_shared<Flex>(function_flex); +} + +/// @brief Make a child element to expand proportionally to the space left in a +/// container. +/// @ingroup dom +/// +/// #### Examples: +/// +/// ~~~cpp +/// hbox({ +/// text("left") | border , +/// text("middle") | border | flex, +/// text("right") | border, +/// }); +/// ~~~ +/// +/// #### Output: +/// +/// ~~~bash +/// ┌────┐┌─────────────────────────────────────────────────────────┐┌─────┐ +/// │left││middle ││right│ +/// └────┘└─────────────────────────────────────────────────────────┘└─────┘ +/// ~~~ +Element flex(Element child) { + return std::make_shared<Flex>(function_flex, std::move(child)); +} + +/// @brief Expand/Minimize if possible/needed on the X axis. +/// @ingroup dom +Element xflex(Element child) { + return std::make_shared<Flex>(function_xflex, std::move(child)); +} + +/// @brief Expand/Minimize if possible/needed on the Y axis. +/// @ingroup dom +Element yflex(Element child) { + return std::make_shared<Flex>(function_yflex, std::move(child)); +} + +/// @brief Expand if possible. +/// @ingroup dom +Element flex_grow(Element child) { + return std::make_shared<Flex>(function_flex_grow, std::move(child)); +} + +/// @brief Expand if possible on the X axis. +/// @ingroup dom +Element xflex_grow(Element child) { + return std::make_shared<Flex>(function_xflex_grow, std::move(child)); +} + +/// @brief Expand if possible on the Y axis. +/// @ingroup dom +Element yflex_grow(Element child) { + return std::make_shared<Flex>(function_yflex_grow, std::move(child)); +} + +/// @brief Minimize if needed. +/// @ingroup dom +Element flex_shrink(Element child) { + return std::make_shared<Flex>(function_flex_shrink, std::move(child)); +} + +/// @brief Minimize if needed on the X axis. +/// @ingroup dom +Element xflex_shrink(Element child) { + return std::make_shared<Flex>(function_xflex_shrink, std::move(child)); +} + +/// @brief Minimize if needed on the Y axis. +/// @ingroup dom +Element yflex_shrink(Element child) { + return std::make_shared<Flex>(function_yflex_shrink, std::move(child)); +} + +/// @brief Make the element not flexible. +/// @ingroup dom +Element notflex(Element child) { + return std::make_shared<Flex>(function_not_flex, std::move(child)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/flexbox.cpp b/contrib/libs/ftxui/src/ftxui/dom/flexbox.cpp new file mode 100644 index 00000000000..00780d2e197 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/flexbox.cpp @@ -0,0 +1,294 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for min, max +#include <cstddef> // for size_t +#include <memory> // for __shared_ptr_access, shared_ptr, allocator_traits<>::value_type, make_shared +#include <tuple> // for ignore +#include <utility> // for move, swap +#include <vector> // for vector + +#include "ftxui/dom/elements.hpp" // for Element, Elements, flexbox, hflow, vflow +#include "ftxui/dom/flexbox_config.hpp" // for FlexboxConfig, FlexboxConfig::Direction, FlexboxConfig::Direction::Column, FlexboxConfig::AlignContent, FlexboxConfig::Direction::ColumnInversed, FlexboxConfig::Direction::Row, FlexboxConfig::JustifyContent, FlexboxConfig::Wrap, FlexboxConfig::AlignContent::FlexStart, FlexboxConfig::Direction::RowInversed, FlexboxConfig::JustifyContent::FlexStart, FlexboxConfig::Wrap::Wrap +#include "ftxui/dom/flexbox_helper.hpp" // for Block, Global, Compute +#include "ftxui/dom/node.hpp" // for Node, Elements, Node::Status +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/dom/selection.hpp" // for Selection +#include "ftxui/screen/box.hpp" // for Box + +namespace ftxui { + +namespace { +void Normalize(FlexboxConfig::Direction& direction) { + switch (direction) { + case FlexboxConfig::Direction::Row: + case FlexboxConfig::Direction::RowInversed: { + direction = FlexboxConfig::Direction::Row; + } break; + case FlexboxConfig::Direction::Column: + case FlexboxConfig::Direction::ColumnInversed: { + direction = FlexboxConfig::Direction::Column; + } break; + } +} + +void Normalize(FlexboxConfig::AlignContent& align_content) { + align_content = FlexboxConfig::AlignContent::FlexStart; +} + +void Normalize(FlexboxConfig::JustifyContent& justify_content) { + justify_content = FlexboxConfig::JustifyContent::FlexStart; +} + +void Normalize(FlexboxConfig::Wrap& wrap) { + wrap = FlexboxConfig::Wrap::Wrap; +} + +FlexboxConfig Normalize(FlexboxConfig config) { + Normalize(config.direction); + Normalize(config.wrap); + Normalize(config.justify_content); + Normalize(config.align_content); + return config; +} + +class Flexbox : public Node { + public: + Flexbox(Elements children, FlexboxConfig config) + : Node(std::move(children)), + config_(config), + config_normalized_(Normalize(config)) { + requirement_.flex_grow_x = 1; + requirement_.flex_grow_y = 0; + + if (IsColumnOriented()) { + std::swap(requirement_.flex_grow_x, requirement_.flex_grow_y); + } + } + + bool IsColumnOriented() const { + return config_.direction == FlexboxConfig::Direction::Column || + config_.direction == FlexboxConfig::Direction::ColumnInversed; + } + + void Layout(flexbox_helper::Global& global, + bool compute_requirement = false) { + global.blocks.reserve(children_.size()); + for (auto& child : children_) { + flexbox_helper::Block block; + block.min_size_x = child->requirement().min_x; + block.min_size_y = child->requirement().min_y; + if (!compute_requirement) { + block.flex_grow_x = child->requirement().flex_grow_x; + block.flex_grow_y = child->requirement().flex_grow_y; + block.flex_shrink_x = child->requirement().flex_shrink_x; + block.flex_shrink_y = child->requirement().flex_shrink_y; + } + global.blocks.push_back(block); + } + + flexbox_helper::Compute(global); + } + + void ComputeRequirement() override { + requirement_ = Requirement{}; + for (auto& child : children_) { + child->ComputeRequirement(); + } + global_ = flexbox_helper::Global(); + global_.config = config_normalized_; + if (IsColumnOriented()) { + global_.size_x = 100000; // NOLINT + global_.size_y = asked_; + } else { + global_.size_x = asked_; + global_.size_y = 100000; // NOLINT + } + Layout(global_, true); + + if (global_.blocks.empty()) { + return; + } + + // Compute the union of all the blocks: + Box box; + box.x_min = global_.blocks[0].x; + box.y_min = global_.blocks[0].y; + box.x_max = global_.blocks[0].x + global_.blocks[0].dim_x; + box.y_max = global_.blocks[0].y + global_.blocks[0].dim_y; + for (auto& b : global_.blocks) { + box.x_min = std::min(box.x_min, b.x); + box.y_min = std::min(box.y_min, b.y); + box.x_max = std::max(box.x_max, b.x + b.dim_x); + box.y_max = std::max(box.y_max, b.y + b.dim_y); + } + requirement_.min_x = box.x_max - box.x_min; + requirement_.min_y = box.y_max - box.y_min; + + // Find the selection: + for (size_t i = 0; i < children_.size(); ++i) { + if (requirement_.focused.Prefer(children_[i]->requirement().focused)) { + requirement_.focused = children_[i]->requirement().focused; + // Shift |focused.box| according to its position inside this component: + auto& b = global_.blocks[i]; + requirement_.focused.box.Shift(b.x, b.y); + requirement_.focused.box = + Box::Intersection(requirement_.focused.box, box); + } + } + } + + void SetBox(Box box) override { + Node::SetBox(box); + + const int asked_previous = asked_; + asked_ = std::min(asked_, IsColumnOriented() ? box.y_max - box.y_min + 1 + : box.x_max - box.x_min + 1); + need_iteration_ = (asked_ != asked_previous); + + flexbox_helper::Global global; + global.config = config_; + global.size_x = box.x_max - box.x_min + 1; + global.size_y = box.y_max - box.y_min + 1; + Layout(global); + + for (size_t i = 0; i < children_.size(); ++i) { + auto& child = children_[i]; + auto& b = global.blocks[i]; + + Box children_box; + children_box.x_min = box.x_min + b.x; + children_box.y_min = box.y_min + b.y; + children_box.x_max = box.x_min + b.x + b.dim_x - 1; + children_box.y_max = box.y_min + b.y + b.dim_y - 1; + + const Box intersection = Box::Intersection(children_box, box); + child->SetBox(intersection); + + need_iteration_ |= (intersection != children_box); + } + } + + void Select(Selection& selection) override { + // If this Node box_ doesn't intersect with the selection, then no + // selection. + if (Box::Intersection(selection.GetBox(), box_).IsEmpty()) { + return; + } + + Selection selection_lines = IsColumnOriented() + ? selection.SaturateVertical(box_) + : selection.SaturateHorizontal(box_); + + size_t i = 0; + for (auto& line : global_.lines) { + Box box; + box.x_min = box_.x_min + line.x; + box.x_max = box_.x_min + line.x + line.dim_x - 1; + box.y_min = box_.y_min + line.y; + box.y_max = box_.y_min + line.y + line.dim_y - 1; + + // If the line box doesn't intersect with the selection, then no + // selection. + if (Box::Intersection(selection.GetBox(), box).IsEmpty()) { + continue; + } + + Selection selection_line = IsColumnOriented() + ? selection_lines.SaturateHorizontal(box) + : selection_lines.SaturateVertical(box); + + for (auto& block : line.blocks) { + std::ignore = block; + children_[i]->Select(selection_line); + i++; + } + } + } + + void Check(Status* status) override { + for (auto& child : children_) { + child->Check(status); + } + + if (status->iteration == 0) { + asked_ = 6000; // NOLINT + need_iteration_ = true; + } + + status->need_iteration |= need_iteration_; + } + + int asked_ = 6000; // NOLINT + bool need_iteration_ = true; + const FlexboxConfig config_; + const FlexboxConfig config_normalized_; + flexbox_helper::Global global_; +}; + +} // namespace + +/// @brief A container displaying elements on row/columns and capable of +/// wrapping on the next column/row when full. +/// @param children The elements in the container +/// @param config The option +/// @return The container. +/// +/// #### Example +/// +/// ```cpp +/// flexbox({ +/// text("element 1"), +/// text("element 2"), +/// text("element 3"), +/// }, FlexboxConfig() +// .Set(FlexboxConfig::Direction::Column) +// .Set(FlexboxConfig::Wrap::WrapInversed) +// .SetGapMainAxis(1) +// .SetGapCrossAxis(1) +// ) +/// ``` +Element flexbox(Elements children, FlexboxConfig config) { + return std::make_shared<Flexbox>(std::move(children), config); +} + +/// @brief A container displaying elements in rows from left to right. When +/// filled, it starts on a new row below. +/// @param children The elements in the container +/// @return The container. +/// +/// #### Example +/// +/// ```cpp +/// hflow({ +/// text("element 1"), +/// text("element 2"), +/// text("element 3"), +/// }); +/// ``` +Element hflow(Elements children) { + return flexbox(std::move(children), FlexboxConfig()); +} + +/// @brief A container displaying elements in rows from top to bottom. When +/// filled, it starts on a new columns on the right. +/// filled, it starts on a new row. +/// is full, it starts a new row. +/// @param children The elements in the container +/// @return The container. +/// +/// #### Example +/// +/// ```cpp +/// vflow({ +/// text("element 1"), +/// text("element 2"), +/// text("element 3"), +/// }); +/// ``` +Element vflow(Elements children) { + return flexbox(std::move(children), + FlexboxConfig().Set(FlexboxConfig::Direction::Column)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/flexbox_config.cpp b/contrib/libs/ftxui/src/ftxui/dom/flexbox_config.cpp new file mode 100644 index 00000000000..fbe78222266 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/flexbox_config.cpp @@ -0,0 +1,51 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include "ftxui/dom/flexbox_config.hpp" + +namespace ftxui { + +/// @brief Set the flexbox direction. +/// @ingroup dom +FlexboxConfig& FlexboxConfig::Set(FlexboxConfig::Direction d) { + this->direction = d; + return *this; +} + +/// @brief Set the flexbox wrap. +/// @ingroup dom +FlexboxConfig& FlexboxConfig::Set(FlexboxConfig::Wrap w) { + this->wrap = w; + return *this; +} + +/// @brief Set the flexbox justify content. +/// @ingroup dom +FlexboxConfig& FlexboxConfig::Set(FlexboxConfig::JustifyContent j) { + this->justify_content = j; + return *this; +} + +/// @brief Set the flexbox align items. +/// @ingroup dom +FlexboxConfig& FlexboxConfig::Set(FlexboxConfig::AlignItems a) { + this->align_items = a; + return *this; +} + +/// @brief Set the flexbox align content. +/// @ingroup dom +FlexboxConfig& FlexboxConfig::Set(FlexboxConfig::AlignContent a) { + this->align_content = a; + return *this; +} + +/// @brief Set the flexbox flex direction. +/// @ingroup dom +FlexboxConfig& FlexboxConfig::SetGap(int x, int y) { + this->gap_x = x; + this->gap_y = y; + return *this; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/flexbox_helper.cpp b/contrib/libs/ftxui/src/ftxui/dom/flexbox_helper.cpp new file mode 100644 index 00000000000..361ae89b98e --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/flexbox_helper.cpp @@ -0,0 +1,380 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include "ftxui/dom/flexbox_helper.hpp" + +#include <algorithm> // for max, min +#include <cstddef> // for size_t +#include <ftxui/dom/flexbox_config.hpp> // for FlexboxConfig, FlexboxConfig::Direction, FlexboxConfig::AlignContent, FlexboxConfig::JustifyContent, FlexboxConfig::Wrap, FlexboxConfig::Direction::RowInversed, FlexboxConfig::AlignItems, FlexboxConfig::Direction::Row, FlexboxConfig::Direction::Column, FlexboxConfig::Direction::ColumnInversed, FlexboxConfig::Wrap::WrapInversed, FlexboxConfig::AlignContent::Stretch, FlexboxConfig::JustifyContent::Stretch, FlexboxConfig::Wrap::Wrap, FlexboxConfig::AlignContent::Center, FlexboxConfig::AlignContent::FlexEnd, FlexboxConfig::AlignContent::FlexStart, FlexboxConfig::AlignContent::SpaceAround, FlexboxConfig::AlignContent::SpaceBetween, FlexboxConfig::AlignContent::SpaceEvenly, FlexboxConfig::AlignItems::Center, FlexboxConfig::AlignItems::FlexEnd, FlexboxConfig::AlignItems::FlexStart, FlexboxConfig::AlignItems::Stretch, FlexboxConfig::JustifyContent::Center, FlexboxConfig::JustifyContent::FlexEnd, FlexboxConfig::JustifyContent::FlexStart, FlexboxConfig::JustifyContent::SpaceAround, FlexboxConfig::JustifyContent::SpaceBetween, FlexboxConfig::JustifyContent::SpaceEvenly, FlexboxConfig::Wrap::NoWrap +#include <utility> // for swap, move +#include <vector> + +#include "ftxui/dom/box_helper.hpp" // for Element, Compute + +namespace ftxui::flexbox_helper { + +namespace { +void SymmetryXY(FlexboxConfig& c) { + std::swap(c.gap_x, c.gap_y); + switch (c.direction) { + case FlexboxConfig::Direction::Row: + c.direction = FlexboxConfig::Direction::Column; + break; + case FlexboxConfig::Direction::RowInversed: + c.direction = FlexboxConfig::Direction::ColumnInversed; + break; + case FlexboxConfig::Direction::Column: + c.direction = FlexboxConfig::Direction::Row; + break; + case FlexboxConfig::Direction::ColumnInversed: + c.direction = FlexboxConfig::Direction::RowInversed; + break; + } +} + +void SymmetryX(FlexboxConfig& c) { + switch (c.direction) { + case FlexboxConfig::Direction::Row: + c.direction = FlexboxConfig::Direction::RowInversed; + break; + case FlexboxConfig::Direction::RowInversed: + c.direction = FlexboxConfig::Direction::Row; + break; + default: + break; + } +} + +void SymmetryY(FlexboxConfig& c) { + switch (c.wrap) { + case FlexboxConfig::Wrap::NoWrap: + break; + case FlexboxConfig::Wrap::Wrap: + c.wrap = FlexboxConfig::Wrap::WrapInversed; + break; + case FlexboxConfig::Wrap::WrapInversed: + c.wrap = FlexboxConfig::Wrap::Wrap; + break; + } +} + +void SymmetryXY(Global& g) { + SymmetryXY(g.config); + std::swap(g.size_x, g.size_y); + for (auto& b : g.blocks) { + std::swap(b.min_size_x, b.min_size_y); + std::swap(b.flex_grow_x, b.flex_grow_y); + std::swap(b.flex_shrink_x, b.flex_shrink_y); + std::swap(b.x, b.y); + std::swap(b.dim_x, b.dim_y); + } + for (auto& l : g.lines) { + std::swap(l.x, l.y); + std::swap(l.dim_x, l.dim_y); + } +} + +void SymmetryX(Global& g) { + SymmetryX(g.config); + for (auto& b : g.blocks) { + b.x = g.size_x - b.x - b.dim_x; + } + for (auto& l : g.lines) { + l.x = g.size_x - l.x - l.dim_x; + } +} + +void SymmetryY(Global& g) { + SymmetryY(g.config); + for (auto& b : g.blocks) { + b.y = g.size_y - b.y - b.dim_y; + } + for (auto& l : g.lines) { + l.y = g.size_y - l.y - l.dim_y; + } +} + +void SetX(Global& global) { + for (auto& line : global.lines) { + std::vector<box_helper::Element> elements; + elements.reserve(line.blocks.size()); + for (auto* block : line.blocks) { + box_helper::Element element; + element.min_size = block->min_size_x; + element.flex_grow = + block->flex_grow_x != 0 || global.config.justify_content == + FlexboxConfig::JustifyContent::Stretch + ? 1 + : 0; + element.flex_shrink = block->flex_shrink_x; + elements.push_back(element); + } + + box_helper::Compute( + &elements, + global.size_x - global.config.gap_x * (int(line.blocks.size()) - 1)); + + int x = 0; + for (size_t i = 0; i < line.blocks.size(); ++i) { + line.blocks[i]->x = x; + line.blocks[i]->dim_x = elements[i].size; + x += elements[i].size; + x += global.config.gap_x; + } + } + + for (auto& line : global.lines) { + line.x = 0; + line.dim_x = global.size_x; + } +} + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +void SetY(Global& g) { + std::vector<box_helper::Element> elements; + elements.reserve(g.lines.size()); + for (auto& line : g.lines) { + box_helper::Element element; + element.flex_shrink = line.blocks.front()->flex_shrink_y; + element.flex_grow = line.blocks.front()->flex_grow_y; + for (auto* block : line.blocks) { + element.min_size = std::max(element.min_size, block->min_size_y); + element.flex_shrink = std::min(element.flex_shrink, block->flex_shrink_y); + element.flex_grow = std::min(element.flex_grow, block->flex_grow_y); + } + elements.push_back(element); + } + + // box_helper::Compute(&elements, g.size_y); + box_helper::Compute(&elements, 10000); // NOLINT + + // [Align-content] + std::vector<int> ys(elements.size()); + int y = 0; + for (size_t i = 0; i < elements.size(); ++i) { + ys[i] = y; + y += elements[i].size; + y += g.config.gap_y; + } + int remaining_space = std::max(0, g.size_y - y); + switch (g.config.align_content) { + case FlexboxConfig::AlignContent::FlexStart: { + break; + } + + case FlexboxConfig::AlignContent::FlexEnd: { + for (size_t i = 0; i < ys.size(); ++i) { // NOLINT + ys[i] += remaining_space; + } + break; + } + + case FlexboxConfig::AlignContent::Center: { + for (size_t i = 0; i < ys.size(); ++i) { // NOLINT + ys[i] += remaining_space / 2; + } + break; + } + + case FlexboxConfig::AlignContent::Stretch: { + for (int i = static_cast<int>(ys.size()) - 1; i >= 0; --i) { // NOLINT + const int shifted = remaining_space * (i + 0) / (i + 1); + ys[i] += shifted; + const int consumed = remaining_space - shifted; + elements[i].size += consumed; + remaining_space -= consumed; + } + break; + } + + case FlexboxConfig::AlignContent::SpaceBetween: { + for (int i = static_cast<int>(ys.size()) - 1; i >= 1; --i) { // NOLINT + ys[i] += remaining_space; + remaining_space = remaining_space * (i - 1) / i; + } + break; + } + + case FlexboxConfig::AlignContent::SpaceAround: { + for (int i = static_cast<int>(ys.size()) - 1; i >= 0; --i) { // NOLINT + ys[i] += remaining_space * (2 * i + 1) / (2 * i + 2); + remaining_space = remaining_space * (2 * i) / (2 * i + 2); + } + break; + } + + case FlexboxConfig::AlignContent::SpaceEvenly: { + for (int i = static_cast<int>(ys.size()) - 1; i >= 0; --i) { // NOLINT + ys[i] += remaining_space * (i + 1) / (i + 2); + remaining_space = remaining_space * (i + 1) / (i + 2); + } + break; + } + } + + // [Align items] + for (size_t i = 0; i < g.lines.size(); ++i) { + auto& element = elements[i]; + for (auto* block : g.lines[i].blocks) { + const bool stretch = + block->flex_grow_y != 0 || + g.config.align_content == FlexboxConfig::AlignContent::Stretch; + const int size = + stretch ? element.size : std::min(element.size, block->min_size_y); + switch (g.config.align_items) { + case FlexboxConfig::AlignItems::FlexStart: { + block->y = ys[i]; + block->dim_y = size; + break; + } + + case FlexboxConfig::AlignItems::Center: { + block->y = ys[i] + (element.size - size) / 2; + block->dim_y = size; + break; + } + + case FlexboxConfig::AlignItems::FlexEnd: { + block->y = ys[i] + element.size - size; + block->dim_y = size; + break; + } + + case FlexboxConfig::AlignItems::Stretch: { + block->y = ys[i]; + block->dim_y = element.size; + break; + } + } + } + } + + ys.push_back(g.size_y); + for (size_t i = 0; i < g.lines.size(); ++i) { + g.lines[i].y = ys[i]; + g.lines[i].dim_y = ys[i + 1] - ys[i]; + } +} + +void JustifyContent(Global& g) { + for (auto& line : g.lines) { + Block* last = line.blocks.back(); + int remaining_space = g.size_x - last->x - last->dim_x; + switch (g.config.justify_content) { + case FlexboxConfig::JustifyContent::FlexStart: + case FlexboxConfig::JustifyContent::Stretch: + break; + + case FlexboxConfig::JustifyContent::FlexEnd: { + for (auto* block : line.blocks) { + block->x += remaining_space; + } + break; + } + + case FlexboxConfig::JustifyContent::Center: { + for (auto* block : line.blocks) { + block->x += remaining_space / 2; + } + break; + } + + case FlexboxConfig::JustifyContent::SpaceBetween: { + for (int i = (int)line.blocks.size() - 1; i >= 1; --i) { + line.blocks[i]->x += remaining_space; + remaining_space = remaining_space * (i - 1) / i; + } + break; + } + + case FlexboxConfig::JustifyContent::SpaceAround: { + for (int i = (int)line.blocks.size() - 1; i >= 0; --i) { + line.blocks[i]->x += remaining_space * (2 * i + 1) / (2 * i + 2); + remaining_space = remaining_space * (2 * i) / (2 * i + 2); + } + break; + } + + case FlexboxConfig::JustifyContent::SpaceEvenly: { + for (int i = (int)line.blocks.size() - 1; i >= 0; --i) { + line.blocks[i]->x += remaining_space * (i + 1) / (i + 2); + remaining_space = remaining_space * (i + 1) / (i + 2); + } + break; + } + } + } +} + +void Compute1(Global& global); +void Compute2(Global& global); +void Compute3(Global& global); + +void Compute1(Global& global) { + if (global.config.direction == FlexboxConfig::Direction::RowInversed) { + SymmetryX(global); + Compute2(global); + SymmetryX(global); + return; + } + Compute2(global); +} + +void Compute2(Global& global) { + if (global.config.wrap == FlexboxConfig::Wrap::WrapInversed) { + SymmetryY(global); + Compute3(global); + SymmetryY(global); + return; + } + Compute3(global); +} + +void Compute3(Global& global) { + // Step 1: Lay out every elements into rows: + { + Line line; + int x = 0; + for (auto& block : global.blocks) { + // Does it fit the end of the row? + // No? Then we need to start a new one: + if (x + block.min_size_x > global.size_x) { + x = 0; + if (!line.blocks.empty()) { + global.lines.push_back(std::move(line)); + } + line = Line(); + } + + block.line = static_cast<int>(global.lines.size()); + block.line_position = static_cast<int>(line.blocks.size()); + line.blocks.push_back(&block); + x += block.min_size_x + global.config.gap_x; + } + if (!line.blocks.empty()) { + global.lines.push_back(std::move(line)); + } + } + + // Step 2: Set positions on the X axis. + SetX(global); + JustifyContent(global); // Distribute remaining space. + + // Step 3: Set positions on the Y axis. + SetY(global); +} + +} // namespace + +void Compute(Global& global) { + if (global.config.direction == FlexboxConfig::Direction::Column || + global.config.direction == FlexboxConfig::Direction::ColumnInversed) { + SymmetryXY(global); + Compute1(global); + SymmetryXY(global); + return; + } + Compute1(global); +} + +} // namespace ftxui::flexbox_helper diff --git a/contrib/libs/ftxui/src/ftxui/dom/flexbox_helper.hpp b/contrib/libs/ftxui/src/ftxui/dom/flexbox_helper.hpp new file mode 100644 index 00000000000..9f9313e4cb7 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/flexbox_helper.hpp @@ -0,0 +1,53 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_DOM_FLEXBOX_HELPER_HPP +#define FTXUI_DOM_FLEXBOX_HELPER_HPP + +#include <vector> +#include "ftxui/dom/flexbox_config.hpp" + +namespace ftxui::flexbox_helper { + +// A block is a rectangle in the flexbox. +struct Block { + // Input: + int min_size_x = 0; + int min_size_y = 0; + int flex_grow_x = 0; + int flex_grow_y = 0; + int flex_shrink_x = 0; + int flex_shrink_y = 0; + + // Output: + int line{}; + int line_position{}; + int x = 0; + int y = 0; + int dim_x = 0; + int dim_y = 0; + bool overflow = false; +}; + +// A line is a row of blocks. +struct Line { + std::vector<Block*> blocks; + int x = 0; + int y = 0; + int dim_x = 0; + int dim_y = 0; +}; + +struct Global { + std::vector<Block> blocks; + std::vector<Line> lines; + FlexboxConfig config; + int size_x; + int size_y; +}; + +void Compute(Global& global); + +} // namespace ftxui::flexbox_helper + +#endif /* end of include guard: FTXUI_DOM_FLEXBOX_HELPER_HPP*/ diff --git a/contrib/libs/ftxui/src/ftxui/dom/focus.cpp b/contrib/libs/ftxui/src/ftxui/dom/focus.cpp new file mode 100644 index 00000000000..647d4b3251b --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/focus.cpp @@ -0,0 +1,96 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <memory> // for make_shared +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Decorator, Element, focusPosition, focusPositionRelative +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/dom/requirement.hpp" // for Requirement, Requirement::NORMAL, Requirement::Selection +#include "ftxui/screen/box.hpp" // for Box + +namespace ftxui { + +/// @brief Used inside a `frame`, this force the view to be scrolled toward a +/// a given position. The position is expressed in proportion of the requested +/// size. +/// +/// For instance: +/// - (0, 0) means that the view is scrolled toward the upper left. +/// - (1, 0) means that the view is scrolled toward the upper right. +/// - (0, 1) means that the view is scrolled toward the bottom left. +/// @ingroup dom +/// +/// ### Example +/// +/// ```cpp +/// Element document = huge_document() +/// | focusPositionRelative(0.f, 1.f) +/// | frame; +/// ``` +Decorator focusPositionRelative(float x, float y) { + class Impl : public NodeDecorator { + public: + Impl(Element child, float x, float y) + : NodeDecorator(std::move(child)), x_(x), y_(y) {} + + void ComputeRequirement() override { + NodeDecorator::ComputeRequirement(); + requirement_.focused.enabled = true; + requirement_.focused.node = this; + requirement_.focused.box.x_min = int(float(requirement_.min_x) * x_); + requirement_.focused.box.y_min = int(float(requirement_.min_y) * y_); + requirement_.focused.box.x_max = int(float(requirement_.min_x) * x_); + requirement_.focused.box.y_max = int(float(requirement_.min_y) * y_); + } + + private: + const float x_; + const float y_; + }; + + return [x, y](Element child) { + return std::make_shared<Impl>(std::move(child), x, y); + }; +} + +/// @brief Used inside a `frame`, this force the view to be scrolled toward a +/// a given position. The position is expressed in the numbers of cells. +/// +/// @ingroup dom +/// +/// ### Example +/// +/// ```cpp +/// Element document = huge_document() +/// | focusPosition(10, 10) +/// | frame; +/// ``` +Decorator focusPosition(int x, int y) { + class Impl : public NodeDecorator { + public: + Impl(Element child, int x, int y) + : NodeDecorator(std::move(child)), x_(x), y_(y) {} + + void ComputeRequirement() override { + NodeDecorator::ComputeRequirement(); + requirement_.focused.enabled = false; + + Box& box = requirement_.focused.box; + box.x_min = x_; + box.y_min = y_; + box.x_max = x_; + box.y_max = y_; + } + + private: + const int x_; + const int y_; + }; + + return [x, y](Element child) { + return std::make_shared<Impl>(std::move(child), x, y); + }; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/frame.cpp b/contrib/libs/ftxui/src/ftxui/dom/frame.cpp new file mode 100644 index 00000000000..bbfb473678a --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/frame.cpp @@ -0,0 +1,222 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for max, min +#include <memory> // for make_shared, __shared_ptr_access +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, unpack, Elements, focus, frame, select, xframe, yframe +#include "ftxui/dom/node.hpp" // for Node, Elements +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Screen, Screen::Cursor +#include "ftxui/util/autoreset.hpp" // for AutoReset + +namespace ftxui { + +namespace { +class Focus : public Node { + public: + explicit Focus(Elements children) : Node(std::move(children)) {} + + void ComputeRequirement() override { + Node::ComputeRequirement(); + requirement_ = children_[0]->requirement(); + requirement_.focused.enabled = true; + requirement_.focused.node = this; + requirement_.focused.box.x_min = 0; + requirement_.focused.box.y_min = 0; + requirement_.focused.box.x_max = requirement_.min_x - 1; + requirement_.focused.box.y_max = requirement_.min_y - 1; + } + + void SetBox(Box box) override { + Node::SetBox(box); + children_[0]->SetBox(box); + } +}; + +class Frame : public Node { + public: + Frame(Elements children, bool x_frame, bool y_frame) + : Node(std::move(children)), x_frame_(x_frame), y_frame_(y_frame) {} + + void SetBox(Box box) override { + Node::SetBox(box); + auto& focused_box = requirement_.focused.box; + Box children_box = box; + + if (x_frame_) { + const int external_dimx = box.x_max - box.x_min; + const int internal_dimx = std::max(requirement_.min_x, external_dimx); + const int focused_dimx = focused_box.x_max - focused_box.x_min; + int dx = focused_box.x_min - external_dimx / 2 + focused_dimx / 2; + dx = std::max(0, std::min(internal_dimx - external_dimx - 1, dx)); + children_box.x_min = box.x_min - dx; + children_box.x_max = box.x_min + internal_dimx - dx; + } + + if (y_frame_) { + const int external_dimy = box.y_max - box.y_min; + const int internal_dimy = std::max(requirement_.min_y, external_dimy); + const int focused_dimy = focused_box.y_max - focused_box.y_min; + int dy = focused_box.y_min - external_dimy / 2 + focused_dimy / 2; + dy = std::max(0, std::min(internal_dimy - external_dimy - 1, dy)); + children_box.y_min = box.y_min - dy; + children_box.y_max = box.y_min + internal_dimy - dy; + } + + children_[0]->SetBox(children_box); + } + + void Render(Screen& screen) override { + const AutoReset<Box> stencil(&screen.stencil, + Box::Intersection(box_, screen.stencil)); + children_[0]->Render(screen); + } + + private: + bool x_frame_; + bool y_frame_; +}; + +class FocusCursor : public Focus { + public: + FocusCursor(Elements children, Screen::Cursor::Shape shape) + : Focus(std::move(children)), shape_(shape) {} + + private: + void ComputeRequirement() override { + Focus::ComputeRequirement(); // NOLINT + requirement_.focused.cursor_shape = shape_; + } + Screen::Cursor::Shape shape_; +}; + +} // namespace + +/// @brief Set the `child` to be the one focused among its siblings. +/// @param child The element to be focused. +/// @ingroup dom +Element focus(Element child) { + return std::make_shared<Focus>(unpack(std::move(child))); +} + +/// This is deprecated. Use `focus` instead. +/// @brief Set the `child` to be the one focused among its siblings. +/// @param child The element to be focused. +Element select(Element child) { + return focus(std::move(child)); +} + +/// @brief Allow an element to be displayed inside a 'virtual' area. It size can +/// be larger than its container. In this case only a smaller portion is +/// displayed. The view is scrollable to make the focused element visible. +/// @see frame +/// @see xframe +/// @see yframe +Element frame(Element child) { + return std::make_shared<Frame>(unpack(std::move(child)), true, true); +} + +/// @brief Same as `frame`, but only on the x-axis. +/// @see frame +/// @see xframe +/// @see yframe +Element xframe(Element child) { + return std::make_shared<Frame>(unpack(std::move(child)), true, false); +} + +/// @brief Same as `frame`, but only on the y-axis. +/// @see frame +/// @see xframe +/// @see yframe +Element yframe(Element child) { + return std::make_shared<Frame>(unpack(std::move(child)), false, true); +} + +/// @brief Same as `focus`, but set the cursor shape to be a still block. +/// @see focus +/// @see focusCursorBlock +/// @see focusCursorBlockBlinking +/// @see focusCursorBar +/// @see focusCursorBarBlinking +/// @see focusCursorUnderline +/// @see focusCursorUnderlineBlinking +/// @ingroup dom +Element focusCursorBlock(Element child) { + return std::make_shared<FocusCursor>(unpack(std::move(child)), + Screen::Cursor::Block); +} + +/// @brief Same as `focus`, but set the cursor shape to be a blinking block. +/// @see focus +/// @see focusCursorBlock +/// @see focusCursorBlockBlinking +/// @see focusCursorBar +/// @see focusCursorBarBlinking +/// @see focusCursorUnderline +/// @see focusCursorUnderlineBlinking +/// @ingroup dom +Element focusCursorBlockBlinking(Element child) { + return std::make_shared<FocusCursor>(unpack(std::move(child)), + Screen::Cursor::BlockBlinking); +} + +/// @brief Same as `focus`, but set the cursor shape to be a still block. +/// @see focus +/// @see focusCursorBlock +/// @see focusCursorBlockBlinking +/// @see focusCursorBar +/// @see focusCursorBarBlinking +/// @see focusCursorUnderline +/// @see focusCursorUnderlineBlinking +/// @ingroup dom +Element focusCursorBar(Element child) { + return std::make_shared<FocusCursor>(unpack(std::move(child)), + Screen::Cursor::Bar); +} + +/// @brief Same as `focus`, but set the cursor shape to be a blinking bar. +/// @see focus +/// @see focusCursorBlock +/// @see focusCursorBlockBlinking +/// @see focusCursorBar +/// @see focusCursorBarBlinking +/// @see focusCursorUnderline +/// @see focusCursorUnderlineBlinking +/// @ingroup dom +Element focusCursorBarBlinking(Element child) { + return std::make_shared<FocusCursor>(unpack(std::move(child)), + Screen::Cursor::BarBlinking); +} + +/// @brief Same as `focus`, but set the cursor shape to be a still underline. +/// @see focus +/// @see focusCursorBlock +/// @see focusCursorBlockBlinking +/// @see focusCursorBar +/// @see focusCursorBarBlinking +/// @see focusCursorUnderline +/// @see focusCursorUnderlineBlinking +/// @ingroup dom +Element focusCursorUnderline(Element child) { + return std::make_shared<FocusCursor>(unpack(std::move(child)), + Screen::Cursor::Underline); +} + +/// @brief Same as `focus`, but set the cursor shape to be a blinking underline. +/// @see focus +/// @see focusCursorBlock +/// @see focusCursorBlockBlinking +/// @see focusCursorBar +/// @see focusCursorBarBlinking +/// @see focusCursorUnderline +/// @see focusCursorUnderlineBlinking +/// @ingroup dom +Element focusCursorUnderlineBlinking(Element child) { + return std::make_shared<FocusCursor>(unpack(std::move(child)), + Screen::Cursor::UnderlineBlinking); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/gauge.cpp b/contrib/libs/ftxui/src/ftxui/dom/gauge.cpp new file mode 100644 index 00000000000..19926a2d8c0 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/gauge.cpp @@ -0,0 +1,297 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <ftxui/dom/direction.hpp> // for Direction, Direction::Down, Direction::Left, Direction::Right, Direction::Up +#include <memory> // for allocator, make_shared +#include <string> // for string + +#include "ftxui/dom/elements.hpp" // for Element, gauge, gaugeDirection, gaugeDown, gaugeLeft, gaugeRight, gaugeUp +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Screen, Pixel + +namespace ftxui { + +namespace { +// NOLINTNEXTLINE +static const std::string charset_horizontal[11] = { +#if defined(FTXUI_MICROSOFT_TERMINAL_FALLBACK) + // Microsoft's terminals often use fonts not handling the 8 unicode + // characters for representing the whole gauge. Fallback with less. + " ", " ", " ", " ", "▌", "▌", "▌", "█", "█", "█", +#else + " ", " ", "▏", "▎", "▍", "▌", "▋", "▊", "▉", "█", +#endif + // An extra character in case when the fuzzer manage to have: + // int(9 * (limit - limit_int) = 9 + "█"}; + +// NOLINTNEXTLINE +static const std::string charset_vertical[10] = { + "█", + "▇", + "▆", + "▅", + "▄", + "▃", + "▂", + "▁", + " ", + // An extra character in case when the fuzzer manage to have: + // int(8 * (limit - limit_int) = 8 + " ", +}; + +class Gauge : public Node { + public: + Gauge(float progress, Direction direction) + : progress_(progress), direction_(direction) { + // This handle NAN correctly: + if (!(progress_ > 0.F)) { + progress_ = 0.F; + } + if (!(progress_ < 1.F)) { + progress_ = 1.F; + } + } + + void ComputeRequirement() override { + switch (direction_) { + case Direction::Right: + case Direction::Left: + requirement_.flex_grow_x = 1; + requirement_.flex_grow_y = 0; + requirement_.flex_shrink_x = 1; + requirement_.flex_shrink_y = 0; + break; + case Direction::Up: + case Direction::Down: + requirement_.flex_grow_x = 0; + requirement_.flex_grow_y = 1; + requirement_.flex_shrink_x = 0; + requirement_.flex_shrink_y = 1; + break; + } + requirement_.min_x = 1; + requirement_.min_y = 1; + } + + void Render(Screen& screen) override { + switch (direction_) { + case Direction::Right: + RenderHorizontal(screen, /*invert=*/false); + break; + case Direction::Up: + RenderVertical(screen, /*invert=*/false); + break; + case Direction::Left: + RenderHorizontal(screen, /*invert=*/true); + break; + case Direction::Down: + RenderVertical(screen, /*invert=*/true); + break; + } + } + + void RenderHorizontal(Screen& screen, bool invert) { + const int y = box_.y_min; + if (y > box_.y_max) { + return; + } + + // Draw the progress bar horizontally. + { + const float progress = invert ? 1.F - progress_ : progress_; + const auto limit = + float(box_.x_min) + progress * float(box_.x_max - box_.x_min + 1); + const int limit_int = static_cast<int>(limit); + int x = box_.x_min; + while (x < limit_int) { + screen.at(x++, y) = charset_horizontal[9]; // NOLINT + } + // NOLINTNEXTLINE + screen.at(x++, y) = charset_horizontal[int(9 * (limit - limit_int))]; + while (x <= box_.x_max) { + screen.at(x++, y) = charset_horizontal[0]; + } + } + + if (invert) { + for (int x = box_.x_min; x <= box_.x_max; x++) { + screen.PixelAt(x, y).inverted ^= true; + } + } + } + + void RenderVertical(Screen& screen, bool invert) { + const int x = box_.x_min; + if (x > box_.x_max) { + return; + } + + // Draw the progress bar vertically: + { + const float progress = invert ? progress_ : 1.F - progress_; + const float limit = + float(box_.y_min) + progress * float(box_.y_max - box_.y_min + 1); + const int limit_int = static_cast<int>(limit); + int y = box_.y_min; + while (y < limit_int) { + screen.at(x, y++) = charset_vertical[8]; // NOLINT + } + // NOLINTNEXTLINE + screen.at(x, y++) = charset_vertical[int(8 * (limit - limit_int))]; + while (y <= box_.y_max) { + screen.at(x, y++) = charset_vertical[0]; + } + } + + if (invert) { + for (int y = box_.y_min; y <= box_.y_max; y++) { + screen.PixelAt(x, y).inverted ^= true; + } + } + } + + private: + float progress_; + Direction direction_; +}; + +} // namespace + +/// @brief Draw a high definition progress bar progressing in specified +/// direction. +/// @param progress The proportion of the area to be filled. Belong to [0,1]. +// @param direction Direction of progress bars progression. +/// @ingroup dom +Element gaugeDirection(float progress, Direction direction) { + return std::make_shared<Gauge>(progress, direction); +} + +/// @brief Draw a high definition progress bar progressing from left to right. +/// @param progress The proportion of the area to be filled. Belong to [0,1]. +/// @ingroup dom +/// +/// ### Example +/// +/// A gauge. It can be used to represent a progress bar. +/// ~~~cpp +/// border(gaugeRight(0.5)) +/// ~~~ +/// +/// #### Output +/// +/// ~~~bash +/// ┌──────────────────────────────────────────────────────────────────────────┐ +/// │█████████████████████████████████████ │ +/// └──────────────────────────────────────────────────────────────────────────┘ +/// ~~~ +Element gaugeRight(float progress) { + return gaugeDirection(progress, Direction::Right); +} + +/// @brief Draw a high definition progress bar progressing from right to left. +/// @param progress The proportion of the area to be filled. Belong to [0,1]. +/// @ingroup dom +/// +/// ### Example +/// +/// A gauge. It can be used to represent a progress bar. +/// ~~~cpp +/// border(gaugeLeft(0.5)) +/// ~~~ +/// +/// #### Output +/// +/// ~~~bash +/// ┌──────────────────────────────────────────────────────────────────────────┐ +/// │ █████████████████████████████████████│ +/// └──────────────────────────────────────────────────────────────────────────┘ +/// ~~~ +Element gaugeLeft(float progress) { + return gaugeDirection(progress, Direction::Left); +} + +/// @brief Draw a high definition progress bar progressing from bottom to top. +/// @param progress The proportion of the area to be filled. Belong to [0,1]. +/// @ingroup dom +/// +/// ### Example +/// +/// A gauge. It can be used to represent a progress bar. +/// ~~~cpp +/// border(gaugeUp(0.5)) +/// ~~~ +/// +/// #### Output +/// +/// ~~~bash +/// ┌─┐ +/// │ │ +/// │ │ +/// │ │ +/// │ │ +/// │█│ +/// │█│ +/// │█│ +/// │█│ +/// └─┘ +/// ~~~ +Element gaugeUp(float progress) { + return gaugeDirection(progress, Direction::Up); +} + +/// @brief Draw a high definition progress bar progressing from top to bottom. +/// @param progress The proportion of the area to be filled. Belong to [0,1]. +/// @ingroup dom +/// +/// ### Example +/// +/// A gauge. It can be used to represent a progress bar. +/// ~~~cpp +/// border(gaugeDown(0.5)) +/// ~~~ +/// +/// #### Output +/// +/// ~~~bash +/// ┌─┐ +/// │█│ +/// │█│ +/// │█│ +/// │█│ +/// │ │ +/// │ │ +/// │ │ +/// │ │ +/// └─┘ +/// ~~~ +Element gaugeDown(float progress) { + return gaugeDirection(progress, Direction::Down); +} + +/// @brief Draw a high definition progress bar. +/// @param progress The proportion of the area to be filled. Belong to [0,1]. +/// @ingroup dom +/// +/// ### Example +/// +/// A gauge. It can be used to represent a progress bar. +/// ~~~cpp +/// border(gauge(0.5)) +/// ~~~ +/// +/// #### Output +/// +/// ~~~bash +/// ┌──────────────────────────────────────────────────────────────────────────┐ +/// │█████████████████████████████████████ │ +/// └──────────────────────────────────────────────────────────────────────────┘ +/// ~~~ +Element gauge(float progress) { + return gaugeRight(progress); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/graph.cpp b/contrib/libs/ftxui/src/ftxui/dom/graph.cpp new file mode 100644 index 00000000000..927558ba384 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/graph.cpp @@ -0,0 +1,75 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <functional> // for function +#include <memory> // for allocator, make_shared +#include <string> // for string +#include <utility> // for move +#include <vector> // for vector + +#include "ftxui/dom/elements.hpp" // for GraphFunction, Element, graph +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Screen + +namespace ftxui { + +namespace { +// NOLINTNEXTLINE +static std::string charset[] = +#if defined(FTXUI_MICROSOFT_TERMINAL_FALLBACK) + // Microsoft's terminals often use fonts not handling the 8 unicode + // characters for representing the whole graph. Fallback with less. + {" ", " ", "█", " ", "█", "█", "█", "█", "█"}; +#else + {" ", "▗", "▐", "▖", "▄", "▟", "▌", "▙", "█"}; +#endif + +class Graph : public Node { + public: + explicit Graph(GraphFunction graph_function) + : graph_function_(std::move(graph_function)) {} + + void ComputeRequirement() override { + requirement_.flex_grow_x = 1; + requirement_.flex_grow_y = 1; + requirement_.flex_shrink_x = 1; + requirement_.flex_shrink_y = 1; + requirement_.min_x = 3; + requirement_.min_y = 3; + } + + void Render(Screen& screen) override { + const int width = (box_.x_max - box_.x_min + 1) * 2; + const int height = (box_.y_max - box_.y_min + 1) * 2; + if (width <= 0 || height <= 0) { + return; + } + auto data = graph_function_(width, height); + int i = 0; + for (int x = box_.x_min; x <= box_.x_max; ++x) { + const int height_1 = 2 * box_.y_max - data[i++]; + const int height_2 = 2 * box_.y_max - data[i++]; + for (int y = box_.y_min; y <= box_.y_max; ++y) { + const int yy = 2 * y; + int i_1 = yy < height_1 ? 0 : yy == height_1 ? 3 : 6; // NOLINT + int i_2 = yy < height_2 ? 0 : yy == height_2 ? 1 : 2; // NOLINT + screen.at(x, y) = charset[i_1 + i_2]; // NOLINT + } + } + } + + private: + GraphFunction graph_function_; +}; + +} // namespace + +/// @brief Draw a graph using a GraphFunction. +/// @param graph_function the function to be called to get the data. +Element graph(GraphFunction graph_function) { + return std::make_shared<Graph>(std::move(graph_function)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/gridbox.cpp b/contrib/libs/ftxui/src/ftxui/dom/gridbox.cpp new file mode 100644 index 00000000000..38b30b5f12e --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/gridbox.cpp @@ -0,0 +1,177 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for max, min +#include <cstddef> // for size_t +#include <memory> // for __shared_ptr_access, shared_ptr, make_shared, allocator_traits<>::value_type +#include <utility> // for move +#include <vector> // for vector, __alloc_traits<>::value_type + +#include "ftxui/dom/box_helper.hpp" // for Element, Compute +#include "ftxui/dom/elements.hpp" // for Elements, filler, Element, gridbox +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/box.hpp" // for Box + +namespace ftxui { +class Screen; + +namespace { + +// Accumulate the values of a list U[n] into v[n]. So that: +// V[0] = 0; +// V[n+1] = v[n] + U[n] +// return the sum of U[n]. +int Integrate(std::vector<int>& elements) { + int accu = 0; + for (auto& i : elements) { + const int old_accu = accu; + accu += i; + i = old_accu; + } + return accu; +} + +class GridBox : public Node { + public: + explicit GridBox(std::vector<Elements> lines) : lines_(std::move(lines)) { + y_size = static_cast<int>(lines_.size()); + for (const auto& line : lines_) { + x_size = std::max(x_size, int(line.size())); + } + + // Fill in empty cells, in case the user did not used the API correctly: + for (auto& line : lines_) { + while (line.size() < size_t(x_size)) { + line.push_back(filler()); + } + } + } + + void ComputeRequirement() override { + requirement_ = Requirement{}; + for (auto& line : lines_) { + for (auto& cell : line) { + cell->ComputeRequirement(); + } + } + + // Compute the size of each columns/row. + std::vector<int> size_x(x_size, 0); + std::vector<int> size_y(y_size, 0); + for (int x = 0; x < x_size; ++x) { + for (int y = 0; y < y_size; ++y) { + size_x[x] = std::max(size_x[x], lines_[y][x]->requirement().min_x); + size_y[y] = std::max(size_y[y], lines_[y][x]->requirement().min_y); + } + } + + requirement_.min_x = Integrate(size_x); + requirement_.min_y = Integrate(size_y); + + // Forward the focused/focused child state: + for (int x = 0; x < x_size; ++x) { + for (int y = 0; y < y_size; ++y) { + if (requirement_.focused.enabled || + !lines_[y][x]->requirement().focused.enabled) { + continue; + } + requirement_.focused = lines_[y][x]->requirement().focused; + requirement_.focused.box.Shift(size_x[x], size_y[y]); + } + } + } + + void SetBox(Box box) override { + Node::SetBox(box); + + box_helper::Element init; + init.min_size = 0; + init.flex_grow = 1024; // NOLINT + init.flex_shrink = 1024; // NOLINT + std::vector<box_helper::Element> elements_x(x_size, init); + std::vector<box_helper::Element> elements_y(y_size, init); + + for (int y = 0; y < y_size; ++y) { + for (int x = 0; x < x_size; ++x) { + const auto& cell = lines_[y][x]; + const auto& requirement = cell->requirement(); + auto& e_x = elements_x[x]; + auto& e_y = elements_y[y]; + e_x.min_size = std::max(e_x.min_size, requirement.min_x); + e_y.min_size = std::max(e_y.min_size, requirement.min_y); + e_x.flex_grow = std::min(e_x.flex_grow, requirement.flex_grow_x); + e_y.flex_grow = std::min(e_y.flex_grow, requirement.flex_grow_y); + e_x.flex_shrink = std::min(e_x.flex_shrink, requirement.flex_shrink_x); + e_y.flex_shrink = std::min(e_y.flex_shrink, requirement.flex_shrink_y); + } + } + + const int target_size_x = box.x_max - box.x_min + 1; + const int target_size_y = box.y_max - box.y_min + 1; + box_helper::Compute(&elements_x, target_size_x); + box_helper::Compute(&elements_y, target_size_y); + + Box box_y = box; + int y = box_y.y_min; + for (int iy = 0; iy < y_size; ++iy) { + box_y.y_min = y; + y += elements_y[iy].size; + box_y.y_max = y - 1; + + Box box_x = box_y; + int x = box_x.x_min; + for (int ix = 0; ix < x_size; ++ix) { + box_x.x_min = x; + x += elements_x[ix].size; + box_x.x_max = x - 1; + lines_[iy][ix]->SetBox(box_x); + } + } + } + + void Render(Screen& screen) override { + for (auto& line : lines_) { + for (auto& cell : line) { + cell->Render(screen); + } + } + } + + int x_size = 0; + int y_size = 0; + std::vector<Elements> lines_; +}; +} // namespace + // +/// @brief A container displaying a grid of elements. +/// @param lines A list of lines, each line being a list of elements. +/// @return The container. +/// +/// #### Example +/// +/// ```cpp +/// auto cell = [](const char* t) { return text(t) | border; }; +/// auto document = gridbox({ +/// {cell("north-west") , cell("north") , cell("north-east")} , +/// {cell("west") , cell("center") , cell("east")} , +/// {cell("south-west") , cell("south") , cell("south-east")} , +/// }); +/// ``` +/// Output: +/// ``` +/// ╭──────────╮╭──────╮╭──────────╮ +/// │north-west││north ││north-east│ +/// ╰──────────╯╰──────╯╰──────────╯ +/// ╭──────────╮╭──────╮╭──────────╮ +/// │west ││center││east │ +/// ╰──────────╯╰──────╯╰──────────╯ +/// ╭──────────╮╭──────╮╭──────────╮ +/// │south-west││south ││south-east│ +/// ╰──────────╯╰──────╯╰──────────╯ +/// ``` +Element gridbox(std::vector<Elements> lines) { + return std::make_shared<GridBox>(std::move(lines)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/hbox.cpp b/contrib/libs/ftxui/src/ftxui/dom/hbox.cpp new file mode 100644 index 00000000000..46f9ef815f8 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/hbox.cpp @@ -0,0 +1,98 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for max +#include <cstddef> // for size_t +#include <memory> // for __shared_ptr_access, shared_ptr, make_shared, allocator_traits<>::value_type +#include <utility> // for move +#include <vector> // for vector, __alloc_traits<>::value_type + +#include "ftxui/dom/box_helper.hpp" // for Element, Compute +#include "ftxui/dom/elements.hpp" // for Element, Elements, hbox +#include "ftxui/dom/node.hpp" // for Node, Elements +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/dom/selection.hpp" // for Selection +#include "ftxui/screen/box.hpp" // for Box +namespace ftxui { + +namespace { +class HBox : public Node { + public: + explicit HBox(Elements children) : Node(std::move(children)) {} + + private: + void ComputeRequirement() override { + requirement_ = Requirement{}; + + for (auto& child : children_) { + child->ComputeRequirement(); + + // Propagate the focused requirement. + if (requirement_.focused.Prefer(child->requirement().focused)) { + requirement_.focused = child->requirement().focused; + requirement_.focused.box.Shift(requirement_.min_x, 0); + } + + // Extend the min_x and min_y to contain all the children + requirement_.min_x += child->requirement().min_x; + requirement_.min_y = + std::max(requirement_.min_y, child->requirement().min_y); + } + } + + void SetBox(Box box) override { + Node::SetBox(box); + + std::vector<box_helper::Element> elements(children_.size()); + for (size_t i = 0; i < children_.size(); ++i) { + auto& element = elements[i]; + const auto& requirement = children_[i]->requirement(); + element.min_size = requirement.min_x; + element.flex_grow = requirement.flex_grow_x; + element.flex_shrink = requirement.flex_shrink_x; + } + const int target_size = box.x_max - box.x_min + 1; + box_helper::Compute(&elements, target_size); + + int x = box.x_min; + for (size_t i = 0; i < children_.size(); ++i) { + box.x_min = x; + box.x_max = x + elements[i].size - 1; + children_[i]->SetBox(box); + x = box.x_max + 1; + } + } + + void Select(Selection& selection) override { + // If this Node box_ doesn't intersect with the selection, then no + // selection. + if (Box::Intersection(selection.GetBox(), box_).IsEmpty()) { + return; + } + + Selection selection_saturated = selection.SaturateHorizontal(box_); + for (auto& child : children_) { + child->Select(selection_saturated); + } + } +}; + +} // namespace + +/// @brief A container displaying elements horizontally one by one. +/// @param children The elements in the container +/// @return The container. +/// +/// #### Example +/// +/// ```cpp +/// hbox({ +/// text("Left"), +/// text("Right"), +/// }); +/// ``` +Element hbox(Elements children) { + return std::make_shared<HBox>(std::move(children)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/hyperlink.cpp b/contrib/libs/ftxui/src/ftxui/dom/hyperlink.cpp new file mode 100644 index 00000000000..f14502d2df8 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/hyperlink.cpp @@ -0,0 +1,74 @@ +// Copyright 2023 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <cstdint> // for uint8_t +#include <memory> // for make_shared +#include <string> // for string +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, Decorator, hyperlink +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Screen, Pixel + +namespace ftxui { + +namespace { +class Hyperlink : public NodeDecorator { + public: + Hyperlink(Element child, std::string link) + : NodeDecorator(std::move(child)), link_(std::move(link)) {} + + void Render(Screen& screen) override { + const uint8_t hyperlink_id = screen.RegisterHyperlink(link_); + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, y).hyperlink = hyperlink_id; + } + } + NodeDecorator::Render(screen); + } + + std::string link_; +}; +} // namespace + +/// @brief Make the rendered area clickable using a web browser. +/// The link will be opened when the user click on it. +/// This is supported only on a limited set of terminal emulator. +/// List: https://github.com/Alhadis/OSC8-Adoption/ +/// @param link The link +/// @param child The input element. +/// @return The output element with the link. +/// @ingroup dom +/// +/// ### Example +/// +/// ```cpp +/// Element document = +/// hyperlink("https://github.com/ArthurSonzogni/FTXUI", "link"); +/// ``` +Element hyperlink(std::string link, Element child) { + return std::make_shared<Hyperlink>(std::move(child), std::move(link)); +} + +/// @brief Decorate using an hyperlink. +/// The link will be opened when the user click on it. +/// This is supported only on a limited set of terminal emulator. +/// List: https://github.com/Alhadis/OSC8-Adoption/ +/// @param link The link to redirect the users to. +/// @return The Decorator applying the hyperlink. +/// @ingroup dom +/// +/// ### Example +/// +/// ```cpp +/// Element document = +/// text("red") | hyperlink("https://github.com/Arthursonzogni/FTXUI"); +/// ``` +// NOLINTNEXTLINE +Decorator hyperlink(std::string link) { + return [link](Element child) { return hyperlink(link, std::move(child)); }; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/inverted.cpp b/contrib/libs/ftxui/src/ftxui/dom/inverted.cpp new file mode 100644 index 00000000000..c69329915e7 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/inverted.cpp @@ -0,0 +1,38 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <memory> // for make_shared +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, inverted +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Pixel, Screen + +namespace ftxui { + +namespace { +class Inverted : public NodeDecorator { + public: + using NodeDecorator::NodeDecorator; + + void Render(Screen& screen) override { + Node::Render(screen); + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, y).inverted ^= true; + } + } + } +}; +} // namespace + +/// @brief Add a filter that will invert the foreground and the background +/// colors. +/// @ingroup dom +Element inverted(Element child) { + return std::make_shared<Inverted>(std::move(child)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/italic.cpp b/contrib/libs/ftxui/src/ftxui/dom/italic.cpp new file mode 100644 index 00000000000..e1a1f6e534b --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/italic.cpp @@ -0,0 +1,35 @@ +// Copyright 2025 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <memory> // for make_shared +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, underlinedDouble +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Pixel, Screen + +namespace ftxui { + +/// @brief Apply a underlinedDouble to text. +/// @ingroup dom +Element italic(Element child) { + class Impl : public NodeDecorator { + public: + using NodeDecorator::NodeDecorator; + + void Render(Screen& screen) override { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, y).italic = true; + } + } + Node::Render(screen); + } + }; + + return std::make_shared<Impl>(std::move(child)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/linear_gradient.cpp b/contrib/libs/ftxui/src/ftxui/dom/linear_gradient.cpp new file mode 100644 index 00000000000..5dda646de91 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/linear_gradient.cpp @@ -0,0 +1,302 @@ +// Copyright 2023 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for max, min, sort, copy +#include <cmath> // for fmod, cos, sin +#include <cstddef> // for size_t +#include <ftxui/dom/linear_gradient.hpp> // for LinearGradient::Stop, LinearGradient +#include <memory> // for allocator_traits<>::value_type, make_shared +#include <optional> // for optional, operator!=, operator< +#include <utility> // for move +#include <vector> // for vector + +#include "ftxui/dom/elements.hpp" // for Element, Decorator, bgcolor, color +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/color.hpp" // for Color, Color::Default, Color::Blue +#include "ftxui/screen/screen.hpp" // for Pixel, Screen + +namespace ftxui { +namespace { + +struct LinearGradientNormalized { + float angle = 0.F; + std::vector<Color> colors; + std::vector<float> positions; // Sorted. +}; + +// Convert a LinearGradient to a normalized version. +LinearGradientNormalized Normalize(LinearGradient gradient) { + // Handle gradient of size 0. + if (gradient.stops.empty()) { + return LinearGradientNormalized{ + 0.F, + {Color::Default, Color::Default}, + {0.F, 1.F}, + }; + } + + // Fill in the two extent, if not provided. + if (!gradient.stops.front().position) { + gradient.stops.front().position = 0.F; + } + if (!gradient.stops.back().position) { + gradient.stops.back().position = 1.F; + } + + // Fill in the blank, by interpolating positions. + size_t last_checkpoint = 0; + for (size_t i = 1; i < gradient.stops.size(); ++i) { + if (!gradient.stops[i].position) { + continue; + } + + if (i - last_checkpoint >= 2) { + const float min = gradient.stops[i].position.value(); // NOLINT + const float max = + gradient.stops[last_checkpoint].position.value(); // NOLINT + for (size_t j = last_checkpoint + 1; j < i; ++j) { + gradient.stops[j].position = min + (max - min) * + float(j - last_checkpoint) / + float(i - last_checkpoint); + } + } + + last_checkpoint = i; + } + + // Sort the stops by position. + std::sort( + gradient.stops.begin(), gradient.stops.end(), + [](const auto& a, const auto& b) { return a.position < b.position; }); + + // If we don't being with zero, add a stop at zero. + if (gradient.stops.front().position != 0) { + gradient.stops.insert(gradient.stops.begin(), + {gradient.stops.front().color, 0.F}); + } + // If we don't end with one, add a stop at one. + if (gradient.stops.back().position != 1) { + gradient.stops.push_back({gradient.stops.back().color, 1.F}); + } + + // Normalize the angle. + LinearGradientNormalized normalized; + const float modulo = 360.F; + normalized.angle = + std::fmod(std::fmod(gradient.angle, modulo) + modulo, modulo); + for (auto& stop : gradient.stops) { + normalized.colors.push_back(stop.color); + // NOLINTNEXTLINE + normalized.positions.push_back(stop.position.value()); + } + return normalized; +} + +Color Interpolate(const LinearGradientNormalized& gradient, float t) { + // Find the right color in the gradient's stops. + size_t i = 1; + while (true) { + // Note that `t` might be slightly greater than 1.0 due to floating point + // precision. This is why we need to handle the case where `t` is greater + // than the last stop's position. + // See https://github.com/ArthurSonzogni/FTXUI/issues/998 + if (i >= gradient.positions.size()) { + const float half = 0.5F; + return Color::Interpolate(half, gradient.colors.back(), + gradient.colors.back()); + } + if (t <= gradient.positions[i]) { + break; + } + ++i; + } + + const float t0 = gradient.positions[i - 1]; + const float t1 = gradient.positions[i - 0]; + const float tt = (t - t0) / (t1 - t0); + + const Color& c0 = gradient.colors[i - 1]; + const Color& c1 = gradient.colors[i - 0]; + const Color& cc = Color::Interpolate(tt, c0, c1); + + return cc; +} + +class LinearGradientColor : public NodeDecorator { + public: + explicit LinearGradientColor(Element child, + const LinearGradient& gradient, + bool background_color) + : NodeDecorator(std::move(child)), + gradient_(Normalize(gradient)), + background_color_{background_color} {} + + private: + void Render(Screen& screen) override { + const float degtorad = 0.01745329251F; + const float dx = std::cos(gradient_.angle * degtorad); + const float dy = std::sin(gradient_.angle * degtorad); + + // Project every corner to get the extent of the gradient. + const float p1 = float(box_.x_min) * dx + float(box_.y_min) * dy; + const float p2 = float(box_.x_min) * dx + float(box_.y_max) * dy; + const float p3 = float(box_.x_max) * dx + float(box_.y_min) * dy; + const float p4 = float(box_.x_max) * dx + float(box_.y_max) * dy; + const float min = std::min({p1, p2, p3, p4}); + const float max = std::max({p1, p2, p3, p4}); + + // Renormalize the projection to [0, 1] using the extent and projective + // geometry. + const float dX = dx / (max - min); + const float dY = dy / (max - min); + const float dZ = -min / (max - min); + + // Project every pixel to get the color. + if (background_color_) { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + const float t = float(x) * dX + float(y) * dY + dZ; + screen.PixelAt(x, y).background_color = Interpolate(gradient_, t); + } + } + } else { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + const float t = float(x) * dX + float(y) * dY + dZ; + screen.PixelAt(x, y).foreground_color = Interpolate(gradient_, t); + } + } + } + + NodeDecorator::Render(screen); + } + + LinearGradientNormalized gradient_; + bool background_color_; +}; + +} // namespace + +/// @brief Build the "empty" gradient. This is often followed by calls to +/// LinearGradient::Angle() and LinearGradient::Stop(). +/// Example: +/// ```cpp +/// auto gradient = +/// LinearGradient() +/// .Angle(45) +/// .Stop(Color::Red, 0.0) +/// .Stop(Color::Green, 0.5) +/// .Stop(Color::Blue, 1.0);; +/// ``` +/// @ingroup dom +LinearGradient::LinearGradient() = default; + +/// @brief Build a gradient with two colors. +/// @param begin The color at the beginning of the gradient. +/// @param end The color at the end of the gradient. +/// @ingroup dom +LinearGradient::LinearGradient(Color begin, Color end) + : LinearGradient(0, begin, end) {} + +/// @brief Build a gradient with two colors and an angle. +/// @param a The angle of the gradient. +/// @param begin The color at the beginning of the gradient. +/// @param end The color at the end of the gradient. +/// @ingroup dom +LinearGradient::LinearGradient(float a, Color begin, Color end) : angle(a) { + stops.push_back({begin, {}}); + stops.push_back({end, {}}); +} + +/// @brief Set the angle of the gradient. +/// @param a The angle of the gradient. +/// @return The gradient. +/// @ingroup dom +LinearGradient& LinearGradient::Angle(float a) { + angle = a; + return *this; +} + +/// @brief Add a color stop to the gradient. +/// @param c The color of the stop. +/// @param p The position of the stop. +/// @return The gradient. +LinearGradient& LinearGradient::Stop(Color c, float p) { + stops.push_back({c, p}); + return *this; +} + +/// @brief Add a color stop to the gradient. +/// @param c The color of the stop. +/// @return The gradient. +/// @ingroup dom +/// @note The position of the stop is interpolated from nearby stops. +LinearGradient& LinearGradient::Stop(Color c) { + stops.push_back({c, {}}); + return *this; +} + +/// @brief Set the foreground color of an element with linear-gradient effect. +/// @param gradient The gradient effect to be applied on the output element. +/// @param child The input element. +/// @return The output element colored. +/// @ingroup dom +/// +/// ### Example +/// +/// ```cpp +/// color(LinearGradient{0, {Color::Red, Color::Blue}}, text("Hello")) +/// ``` +Element color(const LinearGradient& gradient, Element child) { + return std::make_shared<LinearGradientColor>(std::move(child), gradient, + /*background_color*/ false); +} + +/// @brief Set the background color of an element with linear-gradient effect. +/// @param gradient The gradient effect to be applied on the output element. +/// @param child The input element. +/// @return The output element colored. +/// @ingroup dom +/// +/// ### Example +/// +/// ```cpp +/// bgcolor(LinearGradient{0, {Color::Red, Color::Blue}}, text("Hello")) +/// ``` +Element bgcolor(const LinearGradient& gradient, Element child) { + return std::make_shared<LinearGradientColor>(std::move(child), gradient, + /*background_color*/ true); +} + +/// @brief Decorate using a linear-gradient effect on the foreground color. +/// @param gradient The gradient effect to be applied on the output element. +/// @return The Decorator applying the color. +/// @ingroup dom +/// +/// ### Example +/// +/// ```cpp +/// text("Hello") | color(LinearGradient{0, {Color::Red, Color::Blue}}) +/// ``` +Decorator color(const LinearGradient& gradient) { + return + [gradient](Element child) { return color(gradient, std::move(child)); }; +} + +/// @brief Decorate using a linear-gradient effect on the background color. +/// @param gradient The gradient effect to be applied on the output element. +/// @return The Decorator applying the color. +/// @ingroup dom +/// +/// ### Example +/// +/// ```cpp +/// text("Hello") | color(LinearGradient{0, {Color::Red, Color::Blue}}) +/// ``` +Decorator bgcolor(const LinearGradient& gradient) { + return + [gradient](Element child) { return bgcolor(gradient, std::move(child)); }; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/node.cpp b/contrib/libs/ftxui/src/ftxui/dom/node.cpp new file mode 100644 index 00000000000..e610a2fb290 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/node.cpp @@ -0,0 +1,204 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <ftxui/screen/box.hpp> // for Box +#include <string> +#include <utility> // for move + +#include <cstddef> +#include "ftxui/dom/node.hpp" +#include "ftxui/dom/selection.hpp" // for Selection +#include "ftxui/screen/screen.hpp" // for Screen + +namespace ftxui { + +Node::Node() = default; +Node::Node(Elements children) : children_(std::move(children)) {} +Node::~Node() = default; + +/// @brief Compute how much space an elements needs. +/// @ingroup dom +void Node::ComputeRequirement() { + if (children_.empty()) { + return; + } + for (auto& child : children_) { + child->ComputeRequirement(); + } + + // By default, the requirement is the one of the first child. + requirement_ = children_[0]->requirement(); + + // Propagate the focused requirement. + for (size_t i = 1; i < children_.size(); ++i) { + if (!requirement_.focused.enabled && + children_[i]->requirement().focused.enabled) { + requirement_.focused = children_[i]->requirement().focused; + } + } +} + +/// @brief Assign a position and a dimension to an element for drawing. +/// @ingroup dom +void Node::SetBox(Box box) { + box_ = box; +} + +/// @brief Compute the selection of an element. +/// @ingroup dom +void Node::Select(Selection& selection) { + // If this Node box_ doesn't intersect with the selection, then no selection. + if (Box::Intersection(selection.GetBox(), box_).IsEmpty()) { + return; + } + + // By default we defer the selection to the children. + for (auto& child : children_) { + child->Select(selection); + } +} + +/// @brief Display an element on a ftxui::Screen. +/// @ingroup dom +void Node::Render(Screen& screen) { + for (auto& child : children_) { + child->Render(screen); + } +} + +void Node::Check(Status* status) { + for (auto& child : children_) { + child->Check(status); + } + status->need_iteration |= (status->iteration == 0); +} + +std::string Node::GetSelectedContent(Selection& selection) { + std::string content; + + for (auto& child : children_) { + content += child->GetSelectedContent(selection); + } + + return content; +} + +/// @brief Display an element on a ftxui::Screen. +/// @ingroup dom +void Render(Screen& screen, const Element& element) { + Selection selection; + Render(screen, element.get(), selection); +} + +/// @brief Display an element on a ftxui::Screen. +/// @ingroup dom +void Render(Screen& screen, Node* node) { + Selection selection; + Render(screen, node, selection); +} + +void Render(Screen& screen, Node* node, Selection& selection) { + Box box; + box.x_min = 0; + box.y_min = 0; + box.x_max = screen.dimx() - 1; + box.y_max = screen.dimy() - 1; + + Node::Status status; + node->Check(&status); + const int max_iterations = 20; + while (status.need_iteration && status.iteration < max_iterations) { + // Step 1: Find what dimension this elements wants to be. + node->ComputeRequirement(); + + // Step 2: Assign a dimension to the element. + node->SetBox(box); + + // Check if the element needs another iteration of the layout algorithm. + status.need_iteration = false; + status.iteration++; + node->Check(&status); + } + + // Step 3: Selection + if (!selection.IsEmpty()) { + node->Select(selection); + } + + if (node->requirement().focused.enabled +#if defined(FTXUI_MICROSOFT_TERMINAL_FALLBACK) + // Setting the cursor to the right position allow folks using CJK (China, + // Japanese, Korean, ...) characters to see their [input method editor] + // displayed at the right location. See [issue]. + // + // [input method editor]: + // https://en.wikipedia.org/wiki/Input_method + // + // [issue]: + // https://github.com/ArthurSonzogni/FTXUI/issues/2#issuecomment-505282355 + // + // Unfortunately, Microsoft terminal do not handle properly hiding the + // cursor. Instead the character under the cursor is hidden, which is a + // big problem. As a result, we can't enable setting cursor to the right + // location. It will be displayed at the bottom right corner. + // See: + // https://github.com/microsoft/terminal/issues/1203 + // https://github.com/microsoft/terminal/issues/3093 + && + node->requirement().focused.cursor_shape != Screen::Cursor::Shape::Hidden +#endif + ) { + screen.SetCursor(Screen::Cursor{ + node->requirement().focused.node->box_.x_max, + node->requirement().focused.node->box_.y_max, + node->requirement().focused.cursor_shape, + }); + } else { + screen.SetCursor(Screen::Cursor{ + screen.dimx() - 1, + screen.dimy() - 1, + Screen::Cursor::Shape::Hidden, + }); + } + + // Step 4: Draw the element. + screen.stencil = box; + node->Render(screen); + + // Step 5: Apply shaders + screen.ApplyShader(); +} + +std::string GetNodeSelectedContent(Screen& screen, + Node* node, + Selection& selection) { + Box box; + box.x_min = 0; + box.y_min = 0; + box.x_max = screen.dimx() - 1; + box.y_max = screen.dimy() - 1; + + Node::Status status; + node->Check(&status); + const int max_iterations = 20; + while (status.need_iteration && status.iteration < max_iterations) { + // Step 1: Find what dimension this elements wants to be. + node->ComputeRequirement(); + + // Step 2: Assign a dimension to the element. + node->SetBox(box); + + // Check if the element needs another iteration of the layout algorithm. + status.need_iteration = false; + status.iteration++; + node->Check(&status); + } + + // Step 3: Selection + node->Select(selection); + + // Step 4: get the selected content. + return node->GetSelectedContent(selection); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/node_decorator.cpp b/contrib/libs/ftxui/src/ftxui/dom/node_decorator.cpp new file mode 100644 index 00000000000..db53f6b7371 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/node_decorator.cpp @@ -0,0 +1,22 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <ftxui/dom/node.hpp> // for Node, Elements + +#include "ftxui/dom/node_decorator.hpp" +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/box.hpp" // for Box + +namespace ftxui { + +void NodeDecorator::ComputeRequirement() { + Node::ComputeRequirement(); + requirement_ = children_[0]->requirement(); +} + +void NodeDecorator::SetBox(Box box) { + Node::SetBox(box); + children_[0]->SetBox(box); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/node_decorator.hpp b/contrib/libs/ftxui/src/ftxui/dom/node_decorator.hpp new file mode 100644 index 00000000000..a37751f4b19 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/node_decorator.hpp @@ -0,0 +1,25 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_DOM_NODE_DECORATOR_H_ +#define FTXUI_DOM_NODE_DECORATOR_H_ + +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, unpack +#include "ftxui/dom/node.hpp" // for Node + +namespace ftxui { +struct Box; + +// Helper class. +class NodeDecorator : public Node { + public: + explicit NodeDecorator(Element child) : Node(unpack(std::move(child))) {} + void ComputeRequirement() override; + void SetBox(Box box) override; +}; + +} // namespace ftxui + +#endif /* end of include guard: FTXUI_DOM_NODE_DECORATOR_H_ */ diff --git a/contrib/libs/ftxui/src/ftxui/dom/paragraph.cpp b/contrib/libs/ftxui/src/ftxui/dom/paragraph.cpp new file mode 100644 index 00000000000..4d2c154c405 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/paragraph.cpp @@ -0,0 +1,95 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <functional> // for function +#include <sstream> // for basic_istream, stringstream +#include <string> // for string, allocator, getline +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for flexbox, Element, text, Elements, operator|, xflex, paragraph, paragraphAlignCenter, paragraphAlignJustify, paragraphAlignLeft, paragraphAlignRight +#include "ftxui/dom/flexbox_config.hpp" // for FlexboxConfig, FlexboxConfig::JustifyContent, FlexboxConfig::JustifyContent::Center, FlexboxConfig::JustifyContent::FlexEnd, FlexboxConfig::JustifyContent::SpaceBetween + +namespace ftxui { + +namespace { +Elements Split(const std::string& the_text) { + Elements output; + std::stringstream ss(the_text); + std::string word; + while (std::getline(ss, word, ' ')) { + output.push_back(text(word)); + } + return output; +} + +Element Split(const std::string& paragraph, + const std::function<Element(std::string)>& f) { + Elements output; + std::stringstream ss(paragraph); + std::string line; + while (std::getline(ss, line, '\n')) { + output.push_back(f(line)); + } + return vbox(std::move(output)); +} + +} // namespace + +/// @brief Return an element drawing the paragraph on multiple lines. +/// @ingroup dom +/// @see flexbox. +Element paragraph(const std::string& the_text) { + return paragraphAlignLeft(the_text); +} + +/// @brief Return an element drawing the paragraph on multiple lines, aligned on +/// the left. +/// @ingroup dom +/// @see flexbox. +Element paragraphAlignLeft(const std::string& the_text) { + return Split(the_text, [](const std::string& line) { + static const auto config = FlexboxConfig().SetGap(1, 0); + return flexbox(Split(line), config); + }); +}; + +/// @brief Return an element drawing the paragraph on multiple lines, aligned on +/// the right. +/// @ingroup dom +/// @see flexbox. +Element paragraphAlignRight(const std::string& the_text) { + return Split(the_text, [](const std::string& line) { + static const auto config = FlexboxConfig().SetGap(1, 0).Set( + FlexboxConfig::JustifyContent::FlexEnd); + return flexbox(Split(line), config); + }); +} + +/// @brief Return an element drawing the paragraph on multiple lines, aligned on +/// the center. +/// @ingroup dom +/// @see flexbox. +Element paragraphAlignCenter(const std::string& the_text) { + return Split(the_text, [](const std::string& line) { + static const auto config = + FlexboxConfig().SetGap(1, 0).Set(FlexboxConfig::JustifyContent::Center); + return flexbox(Split(line), config); + }); +} + +/// @brief Return an element drawing the paragraph on multiple lines, aligned +/// using a justified alignment. +/// the center. +/// @ingroup dom +/// @see flexbox. +Element paragraphAlignJustify(const std::string& the_text) { + return Split(the_text, [](const std::string& line) { + static const auto config = FlexboxConfig().SetGap(1, 0).Set( + FlexboxConfig::JustifyContent::SpaceBetween); + Elements words = Split(line); + words.push_back(text("") | xflex); + return flexbox(std::move(words), config); + }); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/reflect.cpp b/contrib/libs/ftxui/src/ftxui/dom/reflect.cpp new file mode 100644 index 00000000000..5c5754fbbc2 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/reflect.cpp @@ -0,0 +1,49 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <memory> // for make_shared, __shared_ptr_access +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, unpack, Decorator, reflect +#include "ftxui/dom/node.hpp" // for Node, Elements +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Screen + +namespace ftxui { +namespace { + +// Helper class. +class Reflect : public Node { + public: + Reflect(Element child, Box& box) + : Node(unpack(std::move(child))), reflected_box_(box) {} + + void ComputeRequirement() final { + Node::ComputeRequirement(); + requirement_ = children_[0]->requirement(); + } + + void SetBox(Box box) final { + reflected_box_ = box; + Node::SetBox(box); + children_[0]->SetBox(box); + } + + void Render(Screen& screen) final { + reflected_box_ = Box::Intersection(screen.stencil, reflected_box_); + Node::Render(screen); + } + + private: + Box& reflected_box_; +}; +} // namespace + +Decorator reflect(Box& box) { + return [&](Element child) -> Element { + return std::make_shared<Reflect>(std::move(child), box); + }; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/scroll_indicator.cpp b/contrib/libs/ftxui/src/ftxui/dom/scroll_indicator.cpp new file mode 100644 index 00000000000..dd652cd4930 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/scroll_indicator.cpp @@ -0,0 +1,129 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for max +#include <memory> // for make_shared, __shared_ptr_access +#include <string> // for string +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, vscroll_indicator, hscroll_indicator +#include "ftxui/dom/node.hpp" // for Node, Elements +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Screen, Pixel + +namespace ftxui { + +/// @brief Display a vertical scrollbar to the right. +/// colors. +/// @ingroup dom +Element vscroll_indicator(Element child) { + class Impl : public NodeDecorator { + using NodeDecorator::NodeDecorator; + + void ComputeRequirement() override { + NodeDecorator::ComputeRequirement(); + requirement_ = children_[0]->requirement(); + requirement_.min_x++; + } + + void SetBox(Box box) override { + box_ = box; + box.x_max--; + children_[0]->SetBox(box); + } + + void Render(Screen& screen) final { + NodeDecorator::Render(screen); + + const Box& stencil = screen.stencil; + + const int size_inner = box_.y_max - box_.y_min; + if (size_inner <= 0) { + return; + } + const int size_outter = stencil.y_max - stencil.y_min + 1; + if (size_outter >= size_inner) { + return; + } + + int size = 2 * size_outter * size_outter / size_inner; + size = std::max(size, 1); + + const int start_y = + 2 * stencil.y_min + // + 2 * (stencil.y_min - box_.y_min) * size_outter / size_inner; + + const int x = stencil.x_max; + for (int y = stencil.y_min; y <= stencil.y_max; ++y) { + const int y_up = 2 * y + 0; + const int y_down = 2 * y + 1; + const bool up = (start_y <= y_up) && (y_up <= start_y + size); + const bool down = (start_y <= y_down) && (y_down <= start_y + size); + + const char* c = up ? (down ? "┃" : "╹") : (down ? "╻" : " "); // NOLINT + screen.PixelAt(x, y).character = c; + } + } + }; + return std::make_shared<Impl>(std::move(child)); +} + +/// @brief Display an horizontal scrollbar to the bottom. +/// colors. +/// @ingroup dom +Element hscroll_indicator(Element child) { + class Impl : public NodeDecorator { + using NodeDecorator::NodeDecorator; + + void ComputeRequirement() override { + NodeDecorator::ComputeRequirement(); + requirement_ = children_[0]->requirement(); + requirement_.min_y++; + } + + void SetBox(Box box) override { + box_ = box; + box.y_max--; + children_[0]->SetBox(box); + } + + void Render(Screen& screen) final { + NodeDecorator::Render(screen); + + const Box& stencil = screen.stencil; + + const int size_inner = box_.x_max - box_.x_min; + if (size_inner <= 0) { + return; + } + const int size_outter = stencil.x_max - stencil.x_min + 1; + if (size_outter >= size_inner) { + return; + } + + int size = 2 * size_outter * size_outter / size_inner; + size = std::max(size, 1); + + const int start_x = + 2 * stencil.x_min + // + 2 * (stencil.x_min - box_.x_min) * size_outter / size_inner; + + const int y = stencil.y_max; + for (int x = stencil.x_min; x <= stencil.x_max; ++x) { + const int x_left = 2 * x + 0; + const int x_right = 2 * x + 1; + const bool left = (start_x <= x_left) && (x_left <= start_x + size); + const bool right = (start_x <= x_right) && (x_right <= start_x + size); + + const char* c = + left ? (right ? "─" : "╴") : (right ? "╶" : " "); // NOLINT + screen.PixelAt(x, y).character = c; + } + } + }; + return std::make_shared<Impl>(std::move(child)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/selection.cpp b/contrib/libs/ftxui/src/ftxui/dom/selection.cpp new file mode 100644 index 00000000000..ff55fc414a6 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/selection.cpp @@ -0,0 +1,173 @@ +// Copyright 2024 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. + +#include "ftxui/dom/selection.hpp" // for Selection +#include <algorithm> // for max, min +#include <string> // for string +#include <tuple> // for ignore + +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator + +namespace ftxui { + +namespace { +class Unselectable : public NodeDecorator { + public: + using NodeDecorator::NodeDecorator; + + void Select(Selection& ignored) override { + std::ignore = ignored; + // Overwrite the select method to do nothing. + } +}; +} // namespace + +/// @brief Create an empty selection. +Selection::Selection() = default; + +/// @brief Create a selection. +/// @param start_x The x coordinate of the start of the selection. +/// @param start_y The y coordinate of the start of the selection. +/// @param end_x The x coordinate of the end of the selection. +/// @param end_y The y coordinate of the end of the selection. +Selection::Selection(int start_x, int start_y, int end_x, int end_y) + : start_x_(start_x), + start_y_(start_y), + end_x_(end_x), + end_y_(end_y), + box_{ + std::min(start_x, end_x), + std::max(start_x, end_x), + std::min(start_y, end_y), + std::max(start_y, end_y), + }, + empty_(false) {} + +Selection::Selection(int start_x, + int start_y, + int end_x, + int end_y, + Selection* parent) + : start_x_(start_x), + start_y_(start_y), + end_x_(end_x), + end_y_(end_y), + box_{ + std::min(start_x, end_x), + std::max(start_x, end_x), + std::min(start_y, end_y), + std::max(start_y, end_y), + }, + parent_(parent), + empty_(false) {} + +/// @brief Get the box of the selection. +/// @return The box of the selection. +const Box& Selection::GetBox() const { + return box_; +} + +/// @brief Saturate the selection to be inside the box. +/// This is called by `hbox` to propagate the selection to its children. +/// @param box The box to saturate the selection in. +/// @return The saturated selection. +Selection Selection::SaturateHorizontal(Box box) { + int start_x = start_x_; + int start_y = start_y_; + int end_x = end_x_; + int end_y = end_y_; + + const bool start_outside = !box.Contain(start_x, start_y); + const bool end_outside = !box.Contain(end_x, end_y); + const bool properly_ordered = + start_y < end_y || (start_y == end_y && start_x <= end_x); + if (properly_ordered) { + if (start_outside) { + start_x = box.x_min; + start_y = box.y_min; + } + if (end_outside) { + end_x = box.x_max; + end_y = box.y_max; + } + } else { + if (start_outside) { + start_x = box.x_max; + start_y = box.y_max; + } + if (end_outside) { + end_x = box.x_min; + end_y = box.y_min; + } + } + return { + start_x, start_y, end_x, end_y, parent_, + }; +} + +/// @brief Saturate the selection to be inside the box. +/// This is called by `vbox` to propagate the selection to its children. +/// @param box The box to saturate the selection in. +/// @return The saturated selection. +Selection Selection::SaturateVertical(Box box) { + int start_x = start_x_; + int start_y = start_y_; + int end_x = end_x_; + int end_y = end_y_; + + const bool start_outside = !box.Contain(start_x, start_y); + const bool end_outside = !box.Contain(end_x, end_y); + const bool properly_ordered = + start_y < end_y || (start_y == end_y && start_x <= end_x); + + if (properly_ordered) { + if (start_outside) { + start_x = box.x_min; + start_y = box.y_min; + } + if (end_outside) { + end_x = box.x_max; + end_y = box.y_max; + } + } else { + if (start_outside) { + start_x = box.x_max; + start_y = box.y_max; + } + if (end_outside) { + end_x = box.x_min; + end_y = box.y_min; + } + } + return {start_x, start_y, end_x, end_y, parent_}; +} + +void Selection::AddPart(const std::string& part, int y, int left, int right) { + if (parent_ != this) { + parent_->AddPart(part, y, left, right); + return; + } + [&] { + if (parts_.str().empty()) { + parts_ << part; + return; + } + + if (y_ != y) { + parts_ << '\n' << part; + return; + } + + if (x_ == left + 1) { + parts_ << part; + return; + } + + parts_ << part; + }(); + y_ = y; + x_ = right; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/selection_style.cpp b/contrib/libs/ftxui/src/ftxui/dom/selection_style.cpp new file mode 100644 index 00000000000..202c3ba7a0a --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/selection_style.cpp @@ -0,0 +1,92 @@ +// Copyright 2024 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <functional> // for function +#include <memory> // for make_shared +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, Decorator, bgcolor, color +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/color.hpp" // for Color +#include "ftxui/screen/pixel.hpp" // for Pixel +#include "ftxui/screen/screen.hpp" // for Screen + +namespace ftxui { + +namespace { + +class SelectionStyleReset : public NodeDecorator { + public: + explicit SelectionStyleReset(Element child) + : NodeDecorator(std::move(child)) {} + + void Render(Screen& screen) final { + auto old_style = screen.GetSelectionStyle(); + screen.SetSelectionStyle([](Pixel&) {}); + NodeDecorator::Render(screen); + screen.SetSelectionStyle(old_style); + } +}; + +class SelectionStyle : public NodeDecorator { + public: + SelectionStyle(Element child, const std::function<void(Pixel&)>& style) + : NodeDecorator(std::move(child)), style_(style) {} + + void Render(Screen& screen) final { + auto old_style = screen.GetSelectionStyle(); + auto new_style = [&, old_style](Pixel& pixel) { + old_style(pixel); + style_(pixel); + }; + screen.SetSelectionStyle(new_style); + NodeDecorator::Render(screen); + screen.SetSelectionStyle(old_style); + } + + std::function<void(Pixel&)> style_; +}; + +} // namespace + +/// @brief Reset the selection style of an element. +/// @param child The input element. +/// @return The output element with the selection style reset. +Element selectionStyleReset(Element child) { + return std::make_shared<SelectionStyleReset>(std::move(child)); +} + +/// @brief Set the background color of an element when selected. +/// Note that the style is applied on top of the existing style. +Decorator selectionBackgroundColor(Color foreground) { + return selectionStyle([foreground](Pixel& pixel) { // + pixel.background_color = foreground; + }); +} + +/// @brief Set the foreground color of an element when selected. +/// Note that the style is applied on top of the existing style. +Decorator selectionForegroundColor(Color foreground) { + return selectionStyle([foreground](Pixel& pixel) { // + pixel.foreground_color = foreground; + }); +} + +/// @brief Set the color of an element when selected. +/// @param foreground The color to be applied. +/// Note that the style is applied on top of the existing style. +Decorator selectionColor(Color foreground) { + return selectionForegroundColor(foreground); +} + +/// @brief Set the style of an element when selected. +/// @param style The style to be applied. +/// Note that the style is applied on top of the existing style. +// NOLINTNEXTLINE +Decorator selectionStyle(std::function<void(Pixel&)> style) { + return [style](Element child) -> Element { + return std::make_shared<SelectionStyle>(std::move(child), style); + }; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/separator.cpp b/contrib/libs/ftxui/src/ftxui/dom/separator.cpp new file mode 100644 index 00000000000..4f68dbaead3 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/separator.cpp @@ -0,0 +1,570 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <array> // for array, array<>::value_type +#include <memory> // for make_shared, allocator +#include <string> // for basic_string, string +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, BorderStyle, LIGHT, separator, DOUBLE, EMPTY, HEAVY, separatorCharacter, separatorDouble, separatorEmpty, separatorHSelector, separatorHeavy, separatorLight, separatorStyled, separatorVSelector +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/color.hpp" // for Color +#include "ftxui/screen/pixel.hpp" // for Pixel +#include "ftxui/screen/screen.hpp" // for Pixel, Screen + +namespace ftxui { + +namespace { +using Charset = std::array<std::string, 2>; // NOLINT +using Charsets = std::array<Charset, 6>; // NOLINT +// NOLINTNEXTLINE +const Charsets charsets = { + Charset{"│", "─"}, // LIGHT + Charset{"╏", "╍"}, // DASHED + Charset{"┃", "━"}, // HEAVY + Charset{"║", "═"}, // DOUBLE + Charset{"│", "─"}, // ROUNDED + Charset{" ", " "}, // EMPTY +}; + +class Separator : public Node { + public: + explicit Separator(std::string value) : value_(std::move(value)) {} + + void ComputeRequirement() override { + requirement_.min_x = 1; + requirement_.min_y = 1; + } + + void Render(Screen& screen) override { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + Pixel& pixel = screen.PixelAt(x, y); + pixel.character = value_; + pixel.automerge = true; + } + } + } + + std::string value_; +}; + +class SeparatorAuto : public Node { + public: + explicit SeparatorAuto(BorderStyle style) : style_(style) {} + + void ComputeRequirement() override { + requirement_.min_x = 1; + requirement_.min_y = 1; + } + + void Render(Screen& screen) override { + const bool is_column = (box_.x_max == box_.x_min); + const bool is_line = (box_.y_min == box_.y_max); + + const std::string c = + charsets[style_][int(is_line && !is_column)]; // NOLINT + + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + Pixel& pixel = screen.PixelAt(x, y); + pixel.character = c; + pixel.automerge = true; + } + } + } + + BorderStyle style_; +}; + +class SeparatorWithPixel : public SeparatorAuto { + public: + explicit SeparatorWithPixel(Pixel pixel) + : SeparatorAuto(LIGHT), pixel_(std::move(pixel)) { + pixel_.automerge = true; + } + void Render(Screen& screen) override { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, y) = pixel_; + } + } + } + + private: + Pixel pixel_; +}; +} // namespace + +/// @brief Draw a vertical or horizontal separation in between two other +/// elements. +/// @ingroup dom +/// @see separator +/// @see separatorLight +/// @see separatorDashed +/// @see separatorDouble +/// @see separatorHeavy +/// @see separatorEmpty +/// @see separatorRounded +/// @see separatorStyled +/// @see separatorCharacter +/// +/// Add a visual separation in between two elements. +/// +/// ### Example +/// +/// ```cpp +/// // Use 'border' as a function... +/// Element document = vbox({ +/// text("up"), +/// separator(), +/// text("down"), +/// }); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// up +/// ──── +/// down +/// ``` +Element separator() { + return std::make_shared<SeparatorAuto>(LIGHT); +} + +/// @brief Draw a vertical or horizontal separation in between two other +/// elements. +/// @param style the style of the separator. +/// @ingroup dom +/// @see separator +/// @see separatorLight +/// @see separatorDashed +/// @see separatorDouble +/// @see separatorHeavy +/// @see separatorEmpty +/// @see separatorRounded +/// @see separatorStyled +/// @see separatorCharacter +/// +/// Add a visual separation in between two elements. +/// +/// ### Example +/// +/// ```cpp +/// // Use 'border' as a function... +/// Element document = vbox({ +/// text("up"), +/// separatorStyled(DOUBLE), +/// text("down"), +/// }); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// up +/// ════ +/// down +/// ``` +Element separatorStyled(BorderStyle style) { + return std::make_shared<SeparatorAuto>(style); +} + +/// @brief Draw a vertical or horizontal separation in between two other +/// elements, using the LIGHT style. +/// @ingroup dom +/// @see separator +/// @see separatorLight +/// @see separatorDashed +/// @see separatorDouble +/// @see separatorHeavy +/// @see separatorEmpty +/// @see separatorRounded +/// @see separatorStyled +/// @see separatorCharacter +/// +/// Add a visual separation in between two elements. +/// +/// ### Example +/// +/// ```cpp +/// // Use 'border' as a function... +/// Element document = vbox({ +/// text("up"), +/// separatorLight(), +/// text("down"), +/// }); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// up +/// ──── +/// down +/// ``` +Element separatorLight() { + return std::make_shared<SeparatorAuto>(LIGHT); +} + +/// @brief Draw a vertical or horizontal separation in between two other +/// elements, using the DASHED style. +/// @ingroup dom +/// @see separator +/// @see separatorLight +/// @see separatorDashed +/// @see separatorDouble +/// @see separatorHeavy +/// @see separatorEmpty +/// @see separatorRounded +/// @see separatorStyled +/// @see separatorCharacter +/// +/// Add a visual separation in between two elements. +/// +/// ### Example +/// +/// ```cpp +/// // Use 'border' as a function... +/// Element document = vbox({ +/// text("up"), +/// separatorLight(), +/// text("down"), +/// }); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// up +/// ╍╍╍╍ +/// down +/// ``` +Element separatorDashed() { + return std::make_shared<SeparatorAuto>(DASHED); +} + +/// @brief Draw a vertical or horizontal separation in between two other +/// elements, using the HEAVY style. +/// @ingroup dom +/// @see separator +/// @see separatorLight +/// @see separatorDashed +/// @see separatorDouble +/// @see separatorHeavy +/// @see separatorEmpty +/// @see separatorRounded +/// @see separatorStyled +/// @see separatorCharacter +/// +/// Add a visual separation in between two elements. +/// +/// ### Example +/// +/// ```cpp +/// // Use 'border' as a function... +/// Element document = vbox({ +/// text("up"), +/// separatorHeavy(), +/// text("down"), +/// }); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// up +/// ━━━━ +/// down +/// ``` +Element separatorHeavy() { + return std::make_shared<SeparatorAuto>(HEAVY); +} + +/// @brief Draw a vertical or horizontal separation in between two other +/// elements, using the DOUBLE style. +/// @ingroup dom +/// @see separator +/// @see separatorLight +/// @see separatorDashed +/// @see separatorDouble +/// @see separatorHeavy +/// @see separatorEmpty +/// @see separatorRounded +/// @see separatorStyled +/// @see separatorCharacter +/// +/// Add a visual separation in between two elements. +/// +/// ### Example +/// +/// ```cpp +/// // Use 'border' as a function... +/// Element document = vbox({ +/// text("up"), +/// separatorDouble(), +/// text("down"), +/// }); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// up +/// ════ +/// down +/// ``` +Element separatorDouble() { + return std::make_shared<SeparatorAuto>(DOUBLE); +} + +/// @brief Draw a vertical or horizontal separation in between two other +/// elements, using the EMPTY style. +/// @ingroup dom +/// @see separator +/// @see separatorLight +/// @see separatorDashed +/// @see separatorDouble +/// @see separatorHeavy +/// @see separatorEmpty +/// @see separatorRounded +/// @see separatorStyled +/// @see separatorCharacter +/// +/// Add a visual separation in between two elements. +/// +/// ### Example +/// +/// ```cpp +/// // Use 'border' as a function... +/// Element document = vbox({ +/// text("up"), +/// separator(), +/// text("down"), +/// }); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// up +/// +/// down +/// ``` +Element separatorEmpty() { + return std::make_shared<SeparatorAuto>(EMPTY); +} + +/// @brief Draw a vertical or horizontal separation in between two other +/// elements. +/// @param value the character to fill the separator area. +/// @ingroup dom +/// @see separator +/// @see separatorLight +/// @see separatorDashed +/// @see separatorDouble +/// @see separatorHeavy +/// @see separatorEmpty +/// @see separatorRounded +/// @see separatorStyled +/// @see separatorCharacter +/// +/// Add a visual separation in between two elements. +/// +/// ### Example +/// +/// ```cpp +/// // Use 'border' as a function... +/// Element document = vbox({ +/// text("up"), +/// separator(), +/// text("down"), +/// }); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// up +/// ──── +/// down +/// ``` +Element separatorCharacter(std::string value) { + return std::make_shared<Separator>(std::move(value)); +} + +/// @brief Draw a separator in between two element filled with a given pixel. +/// @ingroup dom +/// @see separator +/// @see separatorLight +/// @see separatorDashed +/// @see separatorHeavy +/// @see separatorDouble +/// @see separatorStyled +/// +/// ### Example +/// +/// ```cpp +/// Pixel empty; +/// Element document = vbox({ +/// text("Up"), +/// separator(empty), +/// text("Down"), +/// }) +/// ``` +/// +/// ### Output +/// +/// ```bash +/// Up +/// +/// Down +/// ``` +Element separator(Pixel pixel) { + return std::make_shared<SeparatorWithPixel>(std::move(pixel)); +} + +/// @brief Draw an horizontal bar, with the area in between left/right colored +/// differently. +/// @param left the left limit of the active area. +/// @param right the right limit of the active area. +/// @param selected_color the color of the selected area. +/// @param unselected_color the color of the unselected area. +/// +/// ### Example +/// +/// ```cpp +/// Element document = separatorHSelector(2,5, Color::White, Color::Blue); +/// ``` +Element separatorHSelector(float left, + float right, + Color unselected_color, + Color selected_color) { + class Impl : public Node { + public: + Impl(float left, float right, Color selected_color, Color unselected_color) + : left_(left), + right_(right), + unselected_color_(unselected_color), + selected_color_(selected_color) {} + void ComputeRequirement() override { + requirement_.min_x = 1; + requirement_.min_y = 1; + } + + void Render(Screen& screen) override { + if (box_.y_max < box_.y_min) { + return; + } + + // This are the two location with an empty demi-cell. + int demi_cell_left = int(left_ * 2.F - 1.F); // NOLINT + int demi_cell_right = int(right_ * 2.F + 2.F); // NOLINT + + const int y = box_.y_min; + for (int x = box_.x_min; x <= box_.x_max; ++x) { + Pixel& pixel = screen.PixelAt(x, y); + + const int a = (x - box_.x_min) * 2; + const int b = a + 1; + const bool a_empty = demi_cell_left == a || demi_cell_right == a; + const bool b_empty = demi_cell_left == b || demi_cell_right == b; + + if (!a_empty && !b_empty) { + pixel.character = "─"; + pixel.automerge = true; + } else { + pixel.character = a_empty ? "╶" : "╴"; // NOLINT + pixel.automerge = false; + } + + if (demi_cell_left <= a && b <= demi_cell_right) { + pixel.foreground_color = selected_color_; + } else { + pixel.foreground_color = unselected_color_; + } + } + } + + float left_; + float right_; + Color unselected_color_; + Color selected_color_; + }; + return std::make_shared<Impl>(left, right, unselected_color, selected_color); +} + +/// @brief Draw an vertical bar, with the area in between up/downcolored +/// differently. +/// @param up the left limit of the active area. +/// @param down the right limit of the active area. +/// @param selected_color the color of the selected area. +/// @param unselected_color the color of the unselected area. +/// +/// ### Example +/// +/// ```cpp +/// Element document = separatorHSelector(2,5, Color::White, Color::Blue); +/// ``` +Element separatorVSelector(float up, + float down, + Color unselected_color, + Color selected_color) { + class Impl : public Node { + public: + Impl(float up, float down, Color unselected_color, Color selected_color) + : up_(up), + down_(down), + unselected_color_(unselected_color), + selected_color_(selected_color) {} + void ComputeRequirement() override { + requirement_.min_x = 1; + requirement_.min_y = 1; + } + + void Render(Screen& screen) override { + if (box_.x_max < box_.x_min) { + return; + } + + // This are the two location with an empty demi-cell. + const int demi_cell_up = int(up_ * 2 - 1); + const int demi_cell_down = int(down_ * 2 + 2); + + const int x = box_.x_min; + for (int y = box_.y_min; y <= box_.y_max; ++y) { + Pixel& pixel = screen.PixelAt(x, y); + + const int a = (y - box_.y_min) * 2; + const int b = a + 1; + const bool a_empty = demi_cell_up == a || demi_cell_down == a; + const bool b_empty = demi_cell_up == b || demi_cell_down == b; + + if (!a_empty && !b_empty) { + pixel.character = "│"; + pixel.automerge = true; + } else { + pixel.character = a_empty ? "╷" : "╵"; // NOLINT + pixel.automerge = false; + } + + if (demi_cell_up <= a && b <= demi_cell_down) { + pixel.foreground_color = selected_color_; + } else { + pixel.foreground_color = unselected_color_; + } + } + } + + float up_; + float down_; + Color unselected_color_; + Color selected_color_; + }; + return std::make_shared<Impl>(up, down, unselected_color, selected_color); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/size.cpp b/contrib/libs/ftxui/src/ftxui/dom/size.cpp new file mode 100644 index 00000000000..a805c0981bf --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/size.cpp @@ -0,0 +1,95 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for min, max +#include <memory> // for make_shared, __shared_ptr_access +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Constraint, WidthOrHeight, EQUAL, GREATER_THAN, LESS_THAN, WIDTH, unpack, Decorator, Element, size +#include "ftxui/dom/node.hpp" // for Node, Elements +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/box.hpp" // for Box + +namespace ftxui { + +namespace { +class Size : public Node { + public: + Size(Element child, WidthOrHeight direction, Constraint constraint, int value) + : Node(unpack(std::move(child))), + direction_(direction), + constraint_(constraint), + value_(value) {} + + void ComputeRequirement() override { + Node::ComputeRequirement(); + requirement_ = children_[0]->requirement(); + + auto& value = direction_ == WIDTH ? requirement_.min_x : requirement_.min_y; + + switch (constraint_) { + case LESS_THAN: + value = std::min(value, value_); + break; + case EQUAL: + value = value_; + break; + case GREATER_THAN: + value = std::max(value, value_); + break; + } + + if (direction_ == WIDTH) { + requirement_.flex_grow_x = 0; + requirement_.flex_shrink_x = 0; + } else { + requirement_.flex_grow_y = 0; + requirement_.flex_shrink_y = 0; + } + } + + void SetBox(Box box) override { + Node::SetBox(box); + + if (direction_ == WIDTH) { + switch (constraint_) { + case LESS_THAN: + case EQUAL: + box.x_max = std::min(box.x_min + value_ + 1, box.x_max); + break; + case GREATER_THAN: + break; + } + } else { + switch (constraint_) { + case LESS_THAN: + case EQUAL: + box.y_max = std::min(box.y_min + value_ + 1, box.y_max); + break; + case GREATER_THAN: + break; + } + } + children_[0]->SetBox(box); + } + + private: + WidthOrHeight direction_; + Constraint constraint_; + int value_; +}; +} // namespace + +/// @brief Apply a constraint on the size of an element. +/// @param direction Whether the WIDTH of the HEIGHT of the element must be +/// constrained. +/// @param constraint The type of constaint. +/// @param value The value. +/// @ingroup dom +Decorator size(WidthOrHeight direction, Constraint constraint, int value) { + return [=](Element e) { + return std::make_shared<Size>(std::move(e), direction, constraint, value); + }; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/spinner.cpp b/contrib/libs/ftxui/src/ftxui/dom/spinner.cpp new file mode 100644 index 00000000000..4ef08c43db0 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/spinner.cpp @@ -0,0 +1,300 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <cstddef> // for size_t +#include <string> // for basic_string, string +#include <utility> // for move +#include <vector> // for vector, __alloc_traits<>::value_type + +#include "ftxui/dom/elements.hpp" // for Element, gauge, text, vbox, spinner + +namespace ftxui { + +namespace { +// NOLINTNEXTLINE +const std::vector<std::vector<std::vector<std::string>>> elements = { + { + {"Replaced by the gauge"}, + }, + { + {". "}, + {".. "}, + {"..."}, + }, + { + {"|"}, + {"/"}, + {"-"}, + {"\\"}, + }, + { + {"+"}, + {"x"}, + }, + { + {"| "}, + {"|| "}, + {"|||"}, + }, + { + {"←"}, + {"↖"}, + {"↑"}, + {"↗"}, + {"→"}, + {"↘"}, + {"↓"}, + {"↙"}, + }, + { + {"▁"}, + {"▂"}, + {"▃"}, + {"▄"}, + {"▅"}, + {"▆"}, + {"▇"}, + {"█"}, + {"▇"}, + {"▆"}, + {"▅"}, + {"▄"}, + {"▃"}, + {"▁"}, + }, + { + {"▉"}, + {"▊"}, + {"▋"}, + {"▌"}, + {"▍"}, + {"▎"}, + {"▏"}, + {"▎"}, + {"▍"}, + {"▌"}, + {"▋"}, + {"▊"}, + }, + { + {"▖"}, + {"▘"}, + {"▝"}, + {"▗"}, + }, + { + {"◢"}, + {"◣"}, + {"◤"}, + {"◥"}, + }, + { + {"◰"}, + {"◳"}, + {"◲"}, + {"◱"}, + }, + { + {"◴"}, + {"◷"}, + {"◶"}, + {"◵"}, + }, + { + {"◐"}, + {"◓"}, + {"◑"}, + {"◒"}, + }, + { + {"◡"}, + {"⊙"}, + {"◠"}, + }, + { + {"⠁"}, + {"⠂"}, + {"⠄"}, + {"⡀"}, + {"⢀"}, + {"⠠"}, + {"⠐"}, + {"⠈"}, + }, + { + {"⠋"}, + {"⠙"}, + {"⠹"}, + {"⠸"}, + {"⠼"}, + {"⠴"}, + {"⠦"}, + {"⠧"}, + {"⠇"}, + {"⠏"}, + }, + { + {"(*----------)"}, {"(-*---------)"}, {"(--*--------)"}, + {"(---*-------)"}, {"(----*------)"}, {"(-----*-----)"}, + {"(------*----)"}, {"(-------*---)"}, {"(--------*--)"}, + {"(---------*-)"}, {"(----------*)"}, {"(---------*-)"}, + {"(--------*--)"}, {"(-------*---)"}, {"(------*----)"}, + {"(-----*-----)"}, {"(----*------)"}, {"(---*-------)"}, + {"(--*--------)"}, {"(-*---------)"}, + }, + { + {"[ ]"}, + {"[= ]"}, + {"[== ]"}, + {"[=== ]"}, + {"[==== ]"}, + {"[===== ]"}, + {"[======]"}, + {"[===== ]"}, + {"[==== ]"}, + {"[=== ]"}, + {"[== ]"}, + {"[= ]"}, + }, + { + {"[ ]"}, + {"[= ]"}, + {"[== ]"}, + {"[=== ]"}, + {"[==== ]"}, + {"[===== ]"}, + {"[======]"}, + {"[ =====]"}, + {"[ ====]"}, + {"[ ===]"}, + {"[ ==]"}, + {"[ =]"}, + }, + { + {"[== ]"}, + {"[== ]"}, + {"[== ]"}, + {"[== ]"}, + {"[== ]"}, + {" [== ]"}, + {"[ == ]"}, + {"[ == ]"}, + {"[ ==]"}, + {"[ ==]"}, + {"[ ==]"}, + {"[ ==]"}, + {"[ ==]"}, + {"[ ==] "}, + {"[ == ]"}, + {"[ == ]"}, + }, + { + { + " ─╮", + " │", + " ", + }, + { + " ╮", + " │", + " ╯", + }, + { + " ", + " │", + " ─╯", + }, + { + " ", + " ", + "╰─╯", + }, + { + " ", + "│ ", + "╰─ ", + }, + { + "╭ ", + "│ ", + "╰ ", + }, + { + "╭─ ", + "│ ", + " ", + }, + { + "╭─╮", + " ", + " ", + }, + }, + { + { + " /\\O ", + " /\\/", + " /\\ ", + " / \\ ", + "LOL LOL", + }, + { + " _O ", + " //|_ ", + " | ", + " /| ", + " LLOL ", + }, + { + " O ", + " /_ ", + " |\\ ", + " / | ", + " LOLLOL ", + }, + }, + { + {" ", "_______", " "}, + {" ", "______/", " "}, + {" _", "_____/ ", " "}, + {" _ ", "____/ \\", " "}, + {" _ ", "___/ \\ ", " \\"}, + {" _ ", "__/ \\ ", " \\_"}, + {" _ ", "_/ \\ ", " \\_/"}, + {" _ ", "/ \\ _", " \\_/ "}, + {"_ ", " \\ __", " \\_/ "}, + {" ", "\\ ___", " \\_/ "}, + {" ", " ___", "\\_/ "}, + {" ", " _____", "_/ "}, + {" ", " ______", "/ "}, + {" ", "_______", " "}, + }, +}; + +} // namespace + +/// @brief Useful to represent the effect of time and/or events. This display an +/// ASCII art "video". +/// @param charset_index The type of "video". +/// @param image_index The "frame" of the video. You need to increase this for +/// every "step". +/// @ingroup dom +Element spinner(int charset_index, size_t image_index) { + if (charset_index <= 0) { + const int progress_size = 40; + image_index %= progress_size; + if (image_index > progress_size / 2) { + image_index = progress_size - image_index; + } + return gauge(float(image_index) * 0.05F); // NOLINT + } + charset_index %= int(elements.size()); + image_index %= int(elements[charset_index].size()); + std::vector<Element> lines; + for (const auto& it : elements[charset_index][image_index]) { + lines.push_back(text(it)); + } + return vbox(std::move(lines)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/strikethrough.cpp b/contrib/libs/ftxui/src/ftxui/dom/strikethrough.cpp new file mode 100644 index 00000000000..f5eba40c777 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/strikethrough.cpp @@ -0,0 +1,35 @@ +// Copyright 2023 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <memory> // for make_shared +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, strikethrough +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Pixel, Screen + +namespace ftxui { + +/// @brief Apply a strikethrough to text. +/// @ingroup dom +Element strikethrough(Element child) { + class Impl : public NodeDecorator { + public: + using NodeDecorator::NodeDecorator; + + void Render(Screen& screen) override { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, y).strikethrough = true; + } + } + Node::Render(screen); + } + }; + + return std::make_shared<Impl>(std::move(child)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/table.cpp b/contrib/libs/ftxui/src/ftxui/dom/table.cpp new file mode 100644 index 00000000000..52bc8094a1f --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/table.cpp @@ -0,0 +1,463 @@ +// Copyright 2021 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include "ftxui/dom/table.hpp" + +#include <algorithm> // for max +#include <initializer_list> // for initializer_list +#include <memory> // for allocator, shared_ptr, allocator_traits<>::value_type +#include <utility> // for move, swap +#include <vector> // for vector + +#include "ftxui/dom/elements.hpp" // for Element, operator|, text, separatorCharacter, Elements, BorderStyle, Decorator, emptyElement, size, gridbox, EQUAL, flex, flex_shrink, HEIGHT, WIDTH + +namespace ftxui { +namespace { + +bool IsCell(int x, int y) { + return x % 2 == 1 && y % 2 == 1; +} + +// NOLINTNEXTLINE +static std::string charset[6][6] = { + {"┌", "┐", "└", "┘", "─", "│"}, // LIGHT + {"┏", "┓", "┗", "┛", "╍", "╏"}, // DASHED + {"┏", "┓", "┗", "┛", "━", "┃"}, // HEAVY + {"╔", "╗", "╚", "╝", "═", "║"}, // DOUBLE + {"╭", "╮", "╰", "╯", "─", "│"}, // ROUNDED + {" ", " ", " ", " ", " ", " "}, // EMPTY +}; + +int Wrap(int input, int modulo) { + input %= modulo; + input += modulo; + input %= modulo; + return input; +} + +void Order(int& a, int& b) { + if (a >= b) { + std::swap(a, b); + } +} + +} // namespace + +/// @brief Create an empty table. +/// @ingroup dom +Table::Table() { + Initialize({}); +} + +/// @brief Create a table from a vector of vector of string. +/// @param input The input data. +/// @ingroup dom +Table::Table(std::vector<std::vector<std::string>> input) { + std::vector<std::vector<Element>> output; + output.reserve(input.size()); + for (auto& row : input) { + output.emplace_back(); + auto& output_row = output.back(); + output_row.reserve(row.size()); + for (auto& cell : row) { + output_row.push_back(text(std::move(cell))); + } + } + Initialize(std::move(output)); +} + +/// @brief Create a table from a vector of vector of Element +/// @param input The input elements. +/// @ingroup dom +Table::Table(std::vector<std::vector<Element>> input) { + Initialize(std::move(input)); +} + +// @brief Create a table from a list of list of string. +// @param init The input data. +// @ingroup dom +Table::Table(std::initializer_list<std::vector<std::string>> init) { + std::vector<std::vector<Element>> input; + for (const auto& row : init) { + std::vector<Element> output_row; + output_row.reserve(row.size()); + for (const auto& cell : row) { + output_row.push_back(text(cell)); + } + input.push_back(std::move(output_row)); + } + Initialize(std::move(input)); +} + +// private +void Table::Initialize(std::vector<std::vector<Element>> input) { + input_dim_y_ = static_cast<int>(input.size()); + input_dim_x_ = 0; + for (auto& row : input) { + input_dim_x_ = std::max(input_dim_x_, int(row.size())); + } + + dim_y_ = 2 * input_dim_y_ + 1; + dim_x_ = 2 * input_dim_x_ + 1; + + // Reserve space. + elements_.resize(dim_y_); + for (int y = 0; y < dim_y_; ++y) { + elements_[y].resize(dim_x_); + } + + // Transfert elements_ from |input| toward |elements_|. + { + int y = 1; + for (auto& row : input) { + int x = 1; + for (auto& cell : row) { + elements_[y][x] = std::move(cell); + x += 2; + } + y += 2; + } + } + + // Add empty element for the border. + for (int y = 0; y < dim_y_; ++y) { + for (int x = 0; x < dim_x_; ++x) { + auto& element = elements_[y][x]; + + if (IsCell(x, y)) { + if (!element) { + element = emptyElement(); + } + continue; + } + + element = emptyElement(); + } + } +} + +/// @brief Select a row of the table. +/// @param index The index of the row to select. +/// @note You can use negative index to select from the end. +/// @ingroup dom +TableSelection Table::SelectRow(int index) { + return SelectRectangle(0, -1, index, index); +} + +/// @brief Select a range of rows of the table. +/// @param row_min The first row to select. +/// @param row_max The last row to select. +/// @note You can use negative index to select from the end. +/// @ingroup dom +TableSelection Table::SelectRows(int row_min, int row_max) { + return SelectRectangle(0, -1, row_min, row_max); +} + +/// @brief Select a column of the table. +/// @param index The index of the column to select. +/// @note You can use negative index to select from the end. +/// @ingroup dom +TableSelection Table::SelectColumn(int index) { + return SelectRectangle(index, index, 0, -1); +} + +/// @brief Select a range of columns of the table. +/// @param column_min The first column to select. +/// @param column_max The last column to select. +/// @note You can use negative index to select from the end. +/// @ingroup dom +TableSelection Table::SelectColumns(int column_min, int column_max) { + return SelectRectangle(column_min, column_max, 0, -1); +} + +/// @brief Select a cell of the table. +/// @param column The column of the cell to select. +/// @param row The row of the cell to select. +/// @note You can use negative index to select from the end. +/// @ingroup dom +TableSelection Table::SelectCell(int column, int row) { + return SelectRectangle(column, column, row, row); +} + +/// @brief Select a rectangle of the table. +/// @param column_min The first column to select. +/// @param column_max The last column to select. +/// @param row_min The first row to select. +/// @param row_max The last row to select. +/// @note You can use negative index to select from the end. +/// @ingroup dom +TableSelection Table::SelectRectangle(int column_min, + int column_max, + int row_min, + int row_max) { + column_min = Wrap(column_min, input_dim_x_); + column_max = Wrap(column_max, input_dim_x_); + Order(column_min, column_max); + row_min = Wrap(row_min, input_dim_y_); + row_max = Wrap(row_max, input_dim_y_); + Order(row_min, row_max); + + TableSelection output; // NOLINT + output.table_ = this; + output.x_min_ = 2 * column_min; + output.x_max_ = 2 * column_max + 2; + output.y_min_ = 2 * row_min; + output.y_max_ = 2 * row_max + 2; + return output; +} + +/// @brief Select all the table. +/// @ingroup dom +TableSelection Table::SelectAll() { + TableSelection output; // NOLINT + output.table_ = this; + output.x_min_ = 0; + output.x_max_ = dim_x_ - 1; + output.y_min_ = 0; + output.y_max_ = dim_y_ - 1; + return output; +} + +/// @brief Render the table. +/// @return The rendered table. This is an element you can draw. +/// @ingroup dom +Element Table::Render() { + for (int y = 0; y < dim_y_; ++y) { + for (int x = 0; x < dim_x_; ++x) { + auto& it = elements_[y][x]; + + // Line + if ((x + y) % 2 == 1) { + it = std::move(it) | flex; + continue; + } + + // Cells + if ((x % 2) == 1 && (y % 2) == 1) { + it = std::move(it) | flex_shrink; + continue; + } + + // Corners + it = std::move(it) | size(WIDTH, EQUAL, 0) | size(HEIGHT, EQUAL, 0); + } + } + dim_x_ = 0; + dim_y_ = 0; + return gridbox(std::move(elements_)); +} + +/// @brief Apply the `decorator` to the selection. +/// This decorate both the cells, the lines and the corners. +/// @param decorator The decorator to apply. +/// @ingroup dom +// NOLINTNEXTLINE +void TableSelection::Decorate(Decorator decorator) { + for (int y = y_min_; y <= y_max_; ++y) { + for (int x = x_min_; x <= x_max_; ++x) { + Element& e = table_->elements_[y][x]; + e = std::move(e) | decorator; + } + } +} + +/// @brief Apply the `decorator` to the selection. +/// @param decorator The decorator to apply. +/// This decorate only the cells. +/// @ingroup dom +// NOLINTNEXTLINE +void TableSelection::DecorateCells(Decorator decorator) { + for (int y = y_min_; y <= y_max_; ++y) { + for (int x = x_min_; x <= x_max_; ++x) { + if (y % 2 == 1 && x % 2 == 1) { + Element& e = table_->elements_[y][x]; + e = std::move(e) | decorator; + } + } + } +} + +/// @brief Apply the `decorator` to the selection. +/// This decorate only the lines modulo `modulo` with a shift of `shift`. +/// @param decorator The decorator to apply. +/// @param modulo The modulo of the lines to decorate. +/// @param shift The shift of the lines to decorate. +/// @ingroup dom +// NOLINTNEXTLINE +void TableSelection::DecorateAlternateColumn(Decorator decorator, + int modulo, + int shift) { + for (int y = y_min_; y <= y_max_; ++y) { + for (int x = x_min_; x <= x_max_; ++x) { + if (y % 2 == 1 && (x / 2) % modulo == shift) { + Element& e = table_->elements_[y][x]; + e = std::move(e) | decorator; + } + } + } +} + +/// @brief Apply the `decorator` to the selection. +/// This decorate only the lines modulo `modulo` with a shift of `shift`. +/// @param decorator The decorator to apply. +/// @param modulo The modulo of the lines to decorate. +/// @param shift The shift of the lines to decorate. +/// @ingroup dom +// NOLINTNEXTLINE +void TableSelection::DecorateAlternateRow(Decorator decorator, + int modulo, + int shift) { + for (int y = y_min_ + 1; y <= y_max_ - 1; ++y) { + for (int x = x_min_; x <= x_max_; ++x) { + if (y % 2 == 1 && (y / 2) % modulo == shift) { + Element& e = table_->elements_[y][x]; + e = std::move(e) | decorator; + } + } + } +} + +/// @brief Apply the `decorator` to the selection. +/// This decorate only the corners modulo `modulo` with a shift of `shift`. +/// @param decorator The decorator to apply. +/// @param modulo The modulo of the corners to decorate. +/// @param shift The shift of the corners to decorate. +/// @ingroup dom +// NOLINTNEXTLINE +void TableSelection::DecorateCellsAlternateColumn(Decorator decorator, + int modulo, + int shift) { + for (int y = y_min_; y <= y_max_; ++y) { + for (int x = x_min_; x <= x_max_; ++x) { + if (y % 2 == 1 && x % 2 == 1 && ((x / 2) % modulo == shift)) { + Element& e = table_->elements_[y][x]; + e = std::move(e) | decorator; + } + } + } +} + +/// @brief Apply the `decorator` to the selection. +/// This decorate only the corners modulo `modulo` with a shift of `shift`. +/// @param decorator The decorator to apply. +/// @param modulo The modulo of the corners to decorate. +/// @param shift The shift of the corners to decorate. +/// @ingroup dom +// NOLINTNEXTLINE +void TableSelection::DecorateCellsAlternateRow(Decorator decorator, + int modulo, + int shift) { + for (int y = y_min_; y <= y_max_; ++y) { + for (int x = x_min_; x <= x_max_; ++x) { + if (y % 2 == 1 && x % 2 == 1 && ((y / 2) % modulo == shift)) { + Element& e = table_->elements_[y][x]; + e = std::move(e) | decorator; + } + } + } +} + +/// @brief Apply a `border` around the selection. +/// @param border The border style to apply. +/// @ingroup dom +void TableSelection::Border(BorderStyle border) { + BorderLeft(border); + BorderRight(border); + BorderTop(border); + BorderBottom(border); + + // NOLINTNEXTLINE + table_->elements_[y_min_][x_min_] = text(charset[border][0]) | automerge; + // NOLINTNEXTLINE + table_->elements_[y_min_][x_max_] = text(charset[border][1]) | automerge; + // NOLINTNEXTLINE + table_->elements_[y_max_][x_min_] = text(charset[border][2]) | automerge; + // NOLINTNEXTLINE + table_->elements_[y_max_][x_max_] = text(charset[border][3]) | automerge; +} + +/// @brief Draw some separator lines in the selection. +/// @param border The border style to apply. +/// @ingroup dom +void TableSelection::Separator(BorderStyle border) { + for (int y = y_min_ + 1; y <= y_max_ - 1; ++y) { + for (int x = x_min_ + 1; x <= x_max_ - 1; ++x) { + if (y % 2 == 0 || x % 2 == 0) { + Element& e = table_->elements_[y][x]; + e = (y % 2 == 1) + ? separatorCharacter(charset[border][5]) | automerge // NOLINT + : separatorCharacter(charset[border][4]) | automerge; // NOLINT + } + } + } +} + +/// @brief Draw some vertical separator lines in the selection. +/// @param border The border style to apply. +/// @ingroup dom +void TableSelection::SeparatorVertical(BorderStyle border) { + for (int y = y_min_ + 1; y <= y_max_ - 1; ++y) { + for (int x = x_min_ + 1; x <= x_max_ - 1; ++x) { + if (x % 2 == 0) { + table_->elements_[y][x] = + separatorCharacter(charset[border][5]) | automerge; // NOLINT + } + } + } +} + +/// @brief Draw some horizontal separator lines in the selection. +/// @param border The border style to apply. +/// @ingroup dom +void TableSelection::SeparatorHorizontal(BorderStyle border) { + for (int y = y_min_ + 1; y <= y_max_ - 1; ++y) { + for (int x = x_min_ + 1; x <= x_max_ - 1; ++x) { + if (y % 2 == 0) { + table_->elements_[y][x] = + separatorCharacter(charset[border][4]) | automerge; // NOLINT + } + } + } +} + +/// @brief Draw some separator lines to the left side of the selection. +/// @param border The border style to apply. +/// @ingroup dom +void TableSelection::BorderLeft(BorderStyle border) { + for (int y = y_min_; y <= y_max_; y++) { + table_->elements_[y][x_min_] = + separatorCharacter(charset[border][5]) | automerge; // NOLINT + } +} + +/// @brief Draw some separator lines to the right side of the selection. +/// @param border The border style to apply. +/// @ingroup dom +void TableSelection::BorderRight(BorderStyle border) { + for (int y = y_min_; y <= y_max_; y++) { + table_->elements_[y][x_max_] = + separatorCharacter(charset[border][5]) | automerge; // NOLINT + } +} + +/// @brief Draw some separator lines to the top side of the selection. +/// @param border The border style to apply. +/// @ingroup dom +void TableSelection::BorderTop(BorderStyle border) { + for (int x = x_min_; x <= x_max_; x++) { + table_->elements_[y_min_][x] = + separatorCharacter(charset[border][4]) | automerge; // NOLINT + } +} + +/// @brief Draw some separator lines to the bottom side of the selection. +/// @param border The border style to apply. +/// @ingroup dom +void TableSelection::BorderBottom(BorderStyle border) { + for (int x = x_min_; x <= x_max_; x++) { + table_->elements_[y_max_][x] = + separatorCharacter(charset[border][4]) | automerge; // NOLINT + } +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/text.cpp b/contrib/libs/ftxui/src/ftxui/dom/text.cpp new file mode 100644 index 00000000000..4b7a483c25b --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/text.cpp @@ -0,0 +1,224 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for min +#include <memory> // for make_shared +#include <sstream> +#include <string> // for string, wstring +#include <utility> // for move + +#include "ftxui/dom/deprecated.hpp" // for text, vtext +#include "ftxui/dom/elements.hpp" // for Element, text, vtext +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/dom/selection.hpp" // for Selection +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Pixel, Screen +#include "ftxui/screen/string.hpp" // for string_width, Utf8ToGlyphs, to_string + +namespace ftxui { + +namespace { +using ftxui::Screen; + +class Text : public Node { + public: + explicit Text(std::string text) : text_(std::move(text)) {} + + void ComputeRequirement() override { + requirement_.min_x = string_width(text_); + requirement_.min_y = 1; + has_selection = false; + } + + void Select(Selection& selection) override { + if (Box::Intersection(selection.GetBox(), box_).IsEmpty()) { + return; + } + + const Selection selection_saturated = selection.SaturateHorizontal(box_); + + has_selection = true; + selection_start_ = selection_saturated.GetBox().x_min; + selection_end_ = selection_saturated.GetBox().x_max; + + std::stringstream ss; + int x = box_.x_min; + for (const auto& cell : Utf8ToGlyphs(text_)) { + if (cell == "\n") { + continue; + } + if (selection_start_ <= x && x <= selection_end_) { + ss << cell; + } + x++; + } + selection.AddPart(ss.str(), box_.y_min, selection_start_, selection_end_); + } + + void Render(Screen& screen) override { + int x = box_.x_min; + const int y = box_.y_min; + + if (y > box_.y_max) { + return; + } + + for (const auto& cell : Utf8ToGlyphs(text_)) { + if (x > box_.x_max) { + break; + } + if (cell == "\n") { + continue; + } + screen.PixelAt(x, y).character = cell; + + if (has_selection) { + auto selectionTransform = screen.GetSelectionStyle(); + if ((x >= selection_start_) && (x <= selection_end_)) { + selectionTransform(screen.PixelAt(x, y)); + } + } + + ++x; + } + } + + private: + std::string text_; + bool has_selection = false; + int selection_start_ = 0; + int selection_end_ = -1; +}; + +class VText : public Node { + public: + explicit VText(std::string text) + : text_(std::move(text)), width_{std::min(string_width(text_), 1)} {} + + void ComputeRequirement() override { + requirement_.min_x = width_; + requirement_.min_y = string_width(text_); + } + + void Render(Screen& screen) override { + const int x = box_.x_min; + int y = box_.y_min; + if (x + width_ - 1 > box_.x_max) { + return; + } + for (const auto& it : Utf8ToGlyphs(text_)) { + if (y > box_.y_max) { + return; + } + screen.PixelAt(x, y).character = it; + y += 1; + } + } + + private: + std::string text_; + int width_ = 1; +}; + +} // namespace + +/// @brief Display a piece of UTF8 encoded unicode text. +/// @ingroup dom +/// @see ftxui::to_wstring +/// +/// ### Example +/// +/// ```cpp +/// Element document = text("Hello world!"); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// Hello world! +/// ``` +Element text(std::string text) { + return std::make_shared<Text>(std::move(text)); +} + +/// @brief Display a piece of unicode text. +/// @ingroup dom +/// @see ftxui::to_wstring +/// +/// ### Example +/// +/// ```cpp +/// Element document = text(L"Hello world!"); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// Hello world! +/// ``` +Element text(std::wstring text) { // NOLINT + return std::make_shared<Text>(to_string(text)); +} + +/// @brief Display a piece of unicode text vertically. +/// @ingroup dom +/// @see ftxui::to_wstring +/// +/// ### Example +/// +/// ```cpp +/// Element document = vtext("Hello world!"); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// H +/// e +/// l +/// l +/// o +/// +/// w +/// o +/// r +/// l +/// d +/// ! +/// ``` +Element vtext(std::string text) { + return std::make_shared<VText>(std::move(text)); +} + +/// @brief Display a piece unicode text vertically. +/// @ingroup dom +/// @see ftxui::to_wstring +/// +/// ### Example +/// +/// ```cpp +/// Element document = vtext(L"Hello world!"); +/// ``` +/// +/// ### Output +/// +/// ```bash +/// H +/// e +/// l +/// l +/// o +/// +/// w +/// o +/// r +/// l +/// d +/// ! +/// ``` +Element vtext(std::wstring text) { // NOLINT + return std::make_shared<VText>(to_string(text)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/underlined.cpp b/contrib/libs/ftxui/src/ftxui/dom/underlined.cpp new file mode 100644 index 00000000000..9d703bcc3fb --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/underlined.cpp @@ -0,0 +1,37 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <memory> // for make_shared +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, underlined +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Pixel, Screen + +namespace ftxui { + +namespace { +class Underlined : public NodeDecorator { + public: + using NodeDecorator::NodeDecorator; + + void Render(Screen& screen) override { + Node::Render(screen); + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, y).underlined = true; + } + } + } +}; +} // namespace + +/// @brief Make the underlined element to be underlined. +/// @ingroup dom +Element underlined(Element child) { + return std::make_shared<Underlined>(std::move(child)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/underlined_double.cpp b/contrib/libs/ftxui/src/ftxui/dom/underlined_double.cpp new file mode 100644 index 00000000000..e51ab787872 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/underlined_double.cpp @@ -0,0 +1,35 @@ +// Copyright 2023 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <memory> // for make_shared +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, underlinedDouble +#include "ftxui/dom/node.hpp" // for Node +#include "ftxui/dom/node_decorator.hpp" // for NodeDecorator +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Pixel, Screen + +namespace ftxui { + +/// @brief Apply a underlinedDouble to text. +/// @ingroup dom +Element underlinedDouble(Element child) { + class Impl : public NodeDecorator { + public: + using NodeDecorator::NodeDecorator; + + void Render(Screen& screen) override { + for (int y = box_.y_min; y <= box_.y_max; ++y) { + for (int x = box_.x_min; x <= box_.x_max; ++x) { + screen.PixelAt(x, y).underlined_double = true; + } + } + Node::Render(screen); + } + }; + + return std::make_shared<Impl>(std::move(child)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/util.cpp b/contrib/libs/ftxui/src/ftxui/dom/util.cpp new file mode 100644 index 00000000000..9bf2086e912 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/util.cpp @@ -0,0 +1,150 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for min +#include <functional> // for function +#include <memory> // for __shared_ptr_access, make_unique +#include <utility> // for move + +#include "ftxui/dom/elements.hpp" // for Element, Decorator, Elements, operator|, Fit, emptyElement, nothing, operator|= +#include "ftxui/dom/node.hpp" // for Node, Node::Status +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/screen/box.hpp" // for Box +#include "ftxui/screen/screen.hpp" // for Full +#include "ftxui/screen/terminal.hpp" // for Dimensions + +namespace ftxui { + +namespace { +Decorator compose(Decorator a, Decorator b) { + return [a = std::move(a), b = std::move(b)](Element element) { + return b(a(std::move(element))); + }; +} +} // namespace + +/// @brief A decoration doing absolutely nothing. +/// @ingroup dom +Element nothing(Element element) { + return element; +} + +/// @brief Compose two decorator into one. +/// @ingroup dom +/// +/// ### Example +/// +/// ```cpp +/// auto decorator = bold | blink; +/// ``` +Decorator operator|(Decorator a, Decorator b) { + return compose(std::move(a), // + std::move(b)); +} + +/// @brief From a set of element, apply a decorator to every elements. +/// @return the set of decorated element. +/// @ingroup dom +Elements operator|(Elements elements, Decorator decorator) { // NOLINT + Elements output; + output.reserve(elements.size()); + for (auto& it : elements) { + output.push_back(std::move(it) | decorator); + } + return output; +} + +/// @brief From an element, apply a decorator. +/// @return the decorated element. +/// @ingroup dom +/// +/// ### Example +/// +/// Both of these are equivalent: +/// ```cpp +/// bold(text("Hello")); +/// ``` +/// ```cpp +/// text("Hello") | bold; +/// ``` +Element operator|(Element element, Decorator decorator) { // NOLINT + return decorator(std::move(element)); +} + +/// @brief Apply a decorator to an element. +/// @return the decorated element. +/// @ingroup dom +/// +/// ### Example +/// +/// Both of these are equivalent: +/// ```cpp +/// auto element = text("Hello"); +/// element |= bold; +/// ``` +Element& operator|=(Element& e, Decorator d) { + e = e | std::move(d); + return e; +} + +/// The minimal dimension that will fit the given element. +/// @see Fixed +/// @see Full +Dimensions Dimension::Fit(Element& e, bool extend_beyond_screen) { + const Dimensions fullsize = Dimension::Full(); + Box box; + box.x_min = 0; + box.y_min = 0; + box.x_max = fullsize.dimx; + box.y_max = fullsize.dimy; + + Node::Status status; + e->Check(&status); + const int max_iteration = 20; + while (status.need_iteration && status.iteration < max_iteration) { + e->ComputeRequirement(); + + // Don't give the element more space than it needs: + box.x_max = std::min(box.x_max, e->requirement().min_x); + box.y_max = e->requirement().min_y; + if (!extend_beyond_screen) { + box.y_max = std::min(box.y_max, fullsize.dimy); + } + + e->SetBox(box); + status.need_iteration = false; + status.iteration++; + e->Check(&status); + + if (!status.need_iteration) { + break; + } + // Increase the size of the box until it fits... + box.x_max = std::min(e->requirement().min_x, fullsize.dimx); + box.y_max = e->requirement().min_y; + + // ... but don't go beyond the screen size: + if (!extend_beyond_screen) { + box.y_max = std::min(box.y_max, fullsize.dimy); + } + } + + return { + box.x_max, + box.y_max, + }; +} + +/// An element of size 0x0 drawing nothing. +/// @ingroup dom +Element emptyElement() { + class Impl : public Node { + void ComputeRequirement() override { + requirement_.min_x = 0; + requirement_.min_y = 0; + } + }; + return std::make_unique<Impl>(); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/dom/vbox.cpp b/contrib/libs/ftxui/src/ftxui/dom/vbox.cpp new file mode 100644 index 00000000000..4b936d17eda --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/dom/vbox.cpp @@ -0,0 +1,100 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <algorithm> // for max +#include <cstddef> // for size_t +#include <memory> // for __shared_ptr_access, shared_ptr, make_shared, allocator_traits<>::value_type +#include <utility> // for move +#include <vector> // for vector, __alloc_traits<>::value_type + +#include "ftxui/dom/box_helper.hpp" // for Element, Compute +#include "ftxui/dom/elements.hpp" // for Element, Elements, vbox +#include "ftxui/dom/node.hpp" // for Node, Elements +#include "ftxui/dom/requirement.hpp" // for Requirement +#include "ftxui/dom/selection.hpp" // for Selection +#include "ftxui/screen/box.hpp" // for Box + +namespace ftxui { + +namespace { +class VBox : public Node { + public: + explicit VBox(Elements children) : Node(std::move(children)) {} + + private: + void ComputeRequirement() override { + requirement_ = Requirement{}; + + for (auto& child : children_) { + child->ComputeRequirement(); + + // Propagate the focused requirement. + if (requirement_.focused.Prefer(child->requirement().focused)) { + requirement_.focused = child->requirement().focused; + requirement_.focused.box.Shift(0, requirement_.min_y); + } + + // Extend the min_x and min_y to contain all the children + requirement_.min_y += child->requirement().min_y; + requirement_.min_x = + std::max(requirement_.min_x, child->requirement().min_x); + } + } + + void SetBox(Box box) override { + Node::SetBox(box); + + std::vector<box_helper::Element> elements(children_.size()); + for (size_t i = 0; i < children_.size(); ++i) { + auto& element = elements[i]; + const auto& requirement = children_[i]->requirement(); + element.min_size = requirement.min_y; + element.flex_grow = requirement.flex_grow_y; + element.flex_shrink = requirement.flex_shrink_y; + } + const int target_size = box.y_max - box.y_min + 1; + box_helper::Compute(&elements, target_size); + + int y = box.y_min; + for (size_t i = 0; i < children_.size(); ++i) { + box.y_min = y; + box.y_max = y + elements[i].size - 1; + children_[i]->SetBox(box); + y = box.y_max + 1; + } + } + + void Select(Selection& selection) override { + // If this Node box_ doesn't intersect with the selection, then no + // selection. + if (Box::Intersection(selection.GetBox(), box_).IsEmpty()) { + return; + } + + Selection selection_saturated = selection.SaturateVertical(box_); + + for (auto& child : children_) { + child->Select(selection_saturated); + } + } +}; +} // namespace + +/// @brief A container displaying elements vertically one by one. +/// @param children The elements in the container +/// @return The container. +/// @ingroup dom +/// +/// #### Example +/// +/// ```cpp +/// vbox({ +/// text("Up"), +/// text("Down"), +/// }); +/// ``` +Element vbox(Elements children) { + return std::make_shared<VBox>(std::move(children)); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/screen/box.cpp b/contrib/libs/ftxui/src/ftxui/screen/box.cpp new file mode 100644 index 00000000000..19f59ba5ba2 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/screen/box.cpp @@ -0,0 +1,72 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include "ftxui/screen/box.hpp" + +#include <algorithm> + +namespace ftxui { +/// @return the biggest Box contained in both |a| and |b|. +/// @ingroup screen +// static +Box Box::Intersection(Box a, Box b) { + return Box{ + std::max(a.x_min, b.x_min), + std::min(a.x_max, b.x_max), + std::max(a.y_min, b.y_min), + std::min(a.y_max, b.y_max), + }; +} + +/// @return the smallest Box containing both |a| and |b|. +/// @ingroup screen +// static +Box Box::Union(Box a, Box b) { + return Box{ + std::min(a.x_min, b.x_min), + std::max(a.x_max, b.x_max), + std::min(a.y_min, b.y_min), + std::max(a.y_max, b.y_max), + }; +} + +/// Shift the box by (x,y). +/// @param x horizontal shift. +/// @param y vertical shift. +/// @ingroup screen +void Box::Shift(int x, int y) { + x_min += x; + x_max += x; + y_min += y; + y_max += y; +} + +/// @return whether (x,y) is contained inside the box. +/// @ingroup screen +bool Box::Contain(int x, int y) const { + return x_min <= x && // + x_max >= x && // + y_min <= y && // + y_max >= y; +} + +/// @return whether the box is empty. +/// @ingroup screen +bool Box::IsEmpty() const { + return x_min > x_max || y_min > y_max; +} + +/// @return whether |other| is the same as |this| +/// @ingroup screen +bool Box::operator==(const Box& other) const { + return (x_min == other.x_min) && (x_max == other.x_max) && + (y_min == other.y_min) && (y_max == other.y_max); +} + +/// @return whether |other| and |this| are different. +/// @ingroup screen +bool Box::operator!=(const Box& other) const { + return !operator==(other); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/screen/color.cpp b/contrib/libs/ftxui/src/ftxui/screen/color.cpp new file mode 100644 index 00000000000..43e78983c77 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/screen/color.cpp @@ -0,0 +1,299 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include "ftxui/screen/color.hpp" + +#include <array> // for array +#include <cmath> +#include <cstdint> +#include <string> + +#include "ftxui/screen/color_info.hpp" // for GetColorInfo, ColorInfo +#include "ftxui/screen/terminal.hpp" // for ColorSupport, Color, Palette256, TrueColor + +namespace ftxui { +namespace { +const std::array<const char*, 33> palette16code = { + "30", "40", // + "31", "41", // + "32", "42", // + "33", "43", // + "34", "44", // + "35", "45", // + "36", "46", // + "37", "47", // + "90", "100", // + "91", "101", // + "92", "102", // + "93", "103", // + "94", "104", // + "95", "105", // + "96", "106", // + "97", "107", // +}; + +} // namespace + +bool Color::operator==(const Color& rhs) const { + return red_ == rhs.red_ && green_ == rhs.green_ && blue_ == rhs.blue_ && + type_ == rhs.type_; +} + +bool Color::operator!=(const Color& rhs) const { + return !operator==(rhs); +} + +std::string Color::Print(bool is_background_color) const { + if (is_background_color) { + switch (type_) { + case ColorType::Palette1: + return "49"; + case ColorType::Palette16: + return palette16code[2 * red_ + 1]; // NOLINT + case ColorType::Palette256: + return "48;5;" + std::to_string(red_); + case ColorType::TrueColor: + return "48;2;" + std::to_string(red_) + ";" + std::to_string(green_) + + ";" + std::to_string(blue_); + } + } else { + switch (type_) { + case ColorType::Palette1: + return "39"; + case ColorType::Palette16: + return palette16code[2 * red_]; // NOLINT + case ColorType::Palette256: + return "38;5;" + std::to_string(red_); + case ColorType::TrueColor: + return "38;2;" + std::to_string(red_) + ";" + std::to_string(green_) + + ";" + std::to_string(blue_); + } + } + // NOTREACHED(); + return ""; +} + +/// @brief Build a transparent color. +/// @ingroup screen +Color::Color() = default; + +/// @brief Build a transparent color. +/// @ingroup screen +Color::Color(Palette1 /*value*/) : Color() {} + +/// @brief Build a color using the Palette16 colors. +/// @ingroup screen +Color::Color(Palette16 index) + : type_(ColorType::Palette16), red_(index), alpha_(255) {} + +/// @brief Build a color using Palette256 colors. +/// @ingroup screen +Color::Color(Palette256 index) + : type_(ColorType::Palette256), red_(index), alpha_(255) { + if (Terminal::ColorSupport() >= Terminal::Color::Palette256) { + return; + } + type_ = ColorType::Palette16; + red_ = GetColorInfo(Color::Palette256(red_)).index_16; +} + +/// @brief Build a Color from its RGB representation. +/// https://en.wikipedia.org/wiki/RGB_color_model +/// +/// @param red The quantity of red [0,255] +/// @param green The quantity of green [0,255] +/// @param blue The quantity of blue [0,255] +/// @param alpha The quantity of alpha [0,255] +/// @ingroup screen +Color::Color(uint8_t red, uint8_t green, uint8_t blue, uint8_t alpha) + : type_(ColorType::TrueColor), + red_(red), + green_(green), + blue_(blue), + alpha_(alpha) { + if (Terminal::ColorSupport() == Terminal::Color::TrueColor) { + return; + } + + // Find the closest Color from the database: + const int max_distance = 256 * 256 * 3; + int closest = max_distance; + int best = 0; + const int database_begin = 16; + const int database_end = 256; + for (int i = database_begin; i < database_end; ++i) { + const ColorInfo color_info = GetColorInfo(Color::Palette256(i)); + const int dr = color_info.red - red; + const int dg = color_info.green - green; + const int db = color_info.blue - blue; + const int dist = dr * dr + dg * dg + db * db; + if (closest > dist) { + closest = dist; + best = i; + } + } + + if (Terminal::ColorSupport() == Terminal::Color::Palette256) { + type_ = ColorType::Palette256; + red_ = best; + } else { + type_ = ColorType::Palette16; + red_ = GetColorInfo(Color::Palette256(best)).index_16; + } +} + +/// @brief Build a Color from its RGB representation. +/// https://en.wikipedia.org/wiki/RGB_color_model +/// +/// @param red The quantity of red [0,255] +/// @param green The quantity of green [0,255] +/// @param blue The quantity of blue [0,255] +/// @ingroup screen +// static +Color Color::RGB(uint8_t red, uint8_t green, uint8_t blue) { + return RGBA(red, green, blue, 255); +} + +/// @brief Build a Color from its RGBA representation. +/// https://en.wikipedia.org/wiki/RGB_color_model +/// @param red The quantity of red [0,255] +/// @param green The quantity of green [0,255] +/// @param blue The quantity of blue [0,255] +/// @param alpha The quantity of alpha [0,255] +/// @ingroup screen +/// @see Color::RGB +// static +Color Color::RGBA(uint8_t red, uint8_t green, uint8_t blue, uint8_t alpha) { + return {red, green, blue, alpha}; +} + +/// @brief Build a Color from its HSV representation. +/// https://en.wikipedia.org/wiki/HSL_and_HSV +/// +/// @param h The hue of the color [0,255] +/// @param s The "colorfulness" [0,255]. +/// @param v The "Lightness" [0,255] +/// @param alpha The quantity of alpha [0,255] +/// @ingroup screen +// static +Color Color::HSVA(uint8_t h, uint8_t s, uint8_t v, uint8_t alpha) { + uint8_t region = h / 43; // NOLINT + uint8_t remainder = (h - (region * 43)) * 6; // NOLINT + uint8_t p = (v * (255 - s)) >> 8; // NOLINT + uint8_t q = (v * (255 - ((s * remainder) >> 8))) >> 8; // NOLINT + uint8_t t = (v * (255 - ((s * (255 - remainder)) >> 8))) >> 8; // NOLINT + + // clang-format off + switch (region) { // NOLINT + case 0: return Color(v,t,p, alpha); // NOLINT + case 1: return Color(q,v,p, alpha); // NOLINT + case 2: return Color(p,v,t, alpha); // NOLINT + case 3: return Color(p,q,v, alpha); // NOLINT + case 4: return Color(t,p,v, alpha); // NOLINT + case 5: return Color(v,p,q, alpha); // NOLINT + } // NOLINT + // clang-format on + return {0, 0, 0, alpha}; +} + +/// @brief Build a Color from its HSV representation. +/// https://en.wikipedia.org/wiki/HSL_and_HSV +/// +/// @param h The hue of the color [0,255] +/// @param s The "colorfulness" [0,255]. +/// @param v The "Lightness" [0,255] +/// @ingroup screen +// static +Color Color::HSV(uint8_t h, uint8_t s, uint8_t v) { + return HSVA(h, s, v, 255); +} + +// static +Color Color::Interpolate(float t, const Color& a, const Color& b) { + if (a.type_ == ColorType::Palette1 || // + b.type_ == ColorType::Palette1) { + if (t < 0.5F) { // NOLINT + return a; + } else { + return b; + } + } + + auto get_color = [](const Color& color, // + uint8_t* red, uint8_t* green, uint8_t* blue) { + switch (color.type_) { + case ColorType::Palette1: { + return; + } + + case ColorType::Palette16: { + const ColorInfo info = GetColorInfo(Color::Palette16(color.red_)); + *red = info.red; + *green = info.green; + *blue = info.blue; + return; + } + + case ColorType::Palette256: { + const ColorInfo info = GetColorInfo(Color::Palette256(color.red_)); + *red = info.red; + *green = info.green; + *blue = info.blue; + return; + } + + case ColorType::TrueColor: + default: { + *red = color.red_; + *green = color.green_; + *blue = color.blue_; + return; + } + } + }; + + uint8_t a_r = 0; + uint8_t a_g = 0; + uint8_t a_b = 0; + uint8_t b_r = 0; + uint8_t b_g = 0; + uint8_t b_b = 0; + get_color(a, &a_r, &a_g, &a_b); + get_color(b, &b_r, &b_g, &b_b); + + // Gamma correction: + // https://en.wikipedia.org/wiki/Gamma_correction + auto interp = [t](uint8_t a_u, uint8_t b_u) { + constexpr float gamma = 2.2F; + const float a_f = powf(a_u, gamma); + const float b_f = powf(b_u, gamma); + const float c_f = a_f * (1.0F - t) + // + b_f * t; + return static_cast<uint8_t>(powf(c_f, 1.F / gamma)); + }; + return Color::RGB(interp(a_r, b_r), // + interp(a_g, b_g), // + interp(a_b, b_b)); // +} + +/// @brief Blend two colors together using the alpha channel. +// static +Color Color::Blend(const Color& lhs, const Color& rhs) { + Color out = Interpolate(float(rhs.alpha_) / 255.F, lhs, rhs); + out.alpha_ = lhs.alpha_ + rhs.alpha_ - lhs.alpha_ * rhs.alpha_ / 255; + return out; +} + +inline namespace literals { + +Color operator""_rgb(unsigned long long int combined) { + // assert(combined <= 0xffffffU); + auto const red = static_cast<uint8_t>(combined >> 16U); + auto const green = static_cast<uint8_t>(combined >> 8U); + auto const blue = static_cast<uint8_t>(combined); + return {red, green, blue}; +} + +} // namespace literals + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/screen/color_info.cpp b/contrib/libs/ftxui/src/ftxui/screen/color_info.cpp new file mode 100644 index 00000000000..54663ec871f --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/screen/color_info.cpp @@ -0,0 +1,281 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include "ftxui/screen/color_info.hpp" + +#include <array> + +#include "ftxui/screen/color.hpp" // for Color, Color::Palette16, Color::Palette256 + +namespace ftxui { + +// clang-format off +const std::array<ColorInfo, 256> palette256 = {{ + {"Black" , 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 } , + {"Red" , 1 , 1 , 128 , 0 , 0 , 0 , 255 , 128 } , + {"Green" , 2 , 2 , 0 , 128 , 0 , 85 , 255 , 128 } , + {"Yellow" , 3 , 3 , 128 , 128 , 0 , 43 , 255 , 128 } , + {"Blue" , 4 , 4 , 0 , 0 , 128 , 171 , 255 , 128 } , + {"Magenta" , 5 , 5 , 128 , 0 , 128 , 213 , 255 , 128 } , + {"Cyan" , 6 , 6 , 0 , 128 , 128 , 128 , 255 , 128 } , + {"GrayLight" , 7 , 7 , 192 , 192 , 192 , 0 , 0 , 192 } , + {"GrayDark" , 8 , 8 , 128 , 128 , 128 , 0 , 0 , 128 } , + {"RedLight" , 9 , 9 , 255 , 0 , 0 , 0 , 255 , 255 } , + {"GreenLight" , 10 , 10 , 0 , 255 , 0 , 85 , 255 , 255 } , + {"YellowLight" , 11 , 11 , 255 , 255 , 0 , 43 , 255 , 255 } , + {"BlueLight" , 12 , 12 , 0 , 0 , 255 , 171 , 255 , 255 } , + {"MagentaLight" , 13 , 13 , 255 , 0 , 255 , 213 , 255 , 255 } , + {"CyanLight" , 14 , 14 , 0 , 255 , 255 , 128 , 255 , 255 } , + {"White" , 15 , 15 , 255 , 255 , 255 , 0 , 0 , 255 } , + {"Grey0" , 16 , 0 , 0 , 0 , 0 , 0 , 0 , 0 } , + {"NavyBlue" , 17 , 4 , 0 , 0 , 95 , 171 , 255 , 95 } , + {"DarkBlue" , 18 , 4 , 0 , 0 , 135 , 171 , 255 , 135 } , + {"Blue3" , 19 , 4 , 0 , 0 , 175 , 171 , 255 , 175 } , + {"Blue3Bis" , 20 , 12 , 0 , 0 , 215 , 171 , 255 , 215 } , + {"Blue1" , 21 , 12 , 0 , 0 , 255 , 171 , 255 , 255 } , + {"DarkGreen" , 22 , 2 , 0 , 95 , 0 , 85 , 255 , 95 } , + {"DeepSkyBlue4" , 23 , 6 , 0 , 95 , 95 , 128 , 255 , 95 } , + {"DeepSkyBlue4Bis" , 24 , 6 , 0 , 95 , 135 , 141 , 255 , 135 } , + {"DeepSkyBlue4Ter" , 25 , 6 , 0 , 95 , 175 , 148 , 255 , 175 } , + {"DodgerBlue3" , 26 , 12 , 0 , 95 , 215 , 152 , 255 , 215 } , + {"DodgerBlue2" , 27 , 12 , 0 , 95 , 255 , 155 , 255 , 255 } , + {"Green4" , 28 , 2 , 0 , 135 , 0 , 85 , 255 , 135 } , + {"SpringGreen4" , 29 , 6 , 0 , 135 , 95 , 115 , 255 , 135 } , + {"Turquoise4" , 30 , 6 , 0 , 135 , 135 , 128 , 255 , 135 } , + {"DeepSkyBlue3" , 31 , 6 , 0 , 135 , 175 , 138 , 255 , 175 } , + {"DeepSkyBlue3Bis" , 32 , 14 , 0 , 135 , 215 , 144 , 255 , 215 } , + {"DodgerBlue1" , 33 , 14 , 0 , 135 , 255 , 149 , 255 , 255 } , + {"Green3" , 34 , 2 , 0 , 175 , 0 , 85 , 255 , 175 } , + {"SpringGreen3" , 35 , 6 , 0 , 175 , 95 , 108 , 255 , 175 } , + {"DarkCyan" , 36 , 6 , 0 , 175 , 135 , 118 , 255 , 175 } , + {"LightSeaGreen" , 37 , 6 , 0 , 175 , 175 , 128 , 255 , 175 } , + {"DeepSkyBlue2" , 38 , 14 , 0 , 175 , 215 , 136 , 255 , 215 } , + {"DeepSkyBlue1" , 39 , 14 , 0 , 175 , 255 , 142 , 255 , 255 } , + {"Green3Bis" , 40 , 10 , 0 , 215 , 0 , 85 , 255 , 215 } , + {"SpringGreen3Bis" , 41 , 10 , 0 , 215 , 95 , 104 , 255 , 215 } , + {"SpringGreen2" , 42 , 14 , 0 , 215 , 135 , 112 , 255 , 215 } , + {"Cyan3" , 43 , 14 , 0 , 215 , 175 , 120 , 255 , 215 } , + {"DarkTurquoise" , 44 , 14 , 0 , 215 , 215 , 128 , 255 , 215 } , + {"Turquoise2" , 45 , 14 , 0 , 215 , 255 , 135 , 255 , 255 } , + {"Green1" , 46 , 10 , 0 , 255 , 0 , 85 , 255 , 255 } , + {"SpringGreen2Bis" , 47 , 10 , 0 , 255 , 95 , 101 , 255 , 255 } , + {"SpringGreen1" , 48 , 14 , 0 , 255 , 135 , 107 , 255 , 255 } , + {"MediumSpringGreen" , 49 , 14 , 0 , 255 , 175 , 114 , 255 , 255 } , + {"Cyan2" , 50 , 14 , 0 , 255 , 215 , 121 , 255 , 255 } , + {"Cyan1" , 51 , 14 , 0 , 255 , 255 , 128 , 255 , 255 } , + {"DarkRed" , 52 , 1 , 95 , 0 , 0 , 0 , 255 , 95 } , + {"DeepPink4Ter" , 53 , 5 , 95 , 0 , 95 , 213 , 255 , 95 } , + {"Purple4" , 54 , 5 , 95 , 0 , 135 , 201 , 255 , 135 } , + {"Purple4Bis" , 55 , 5 , 95 , 0 , 175 , 194 , 255 , 175 } , + {"Purple3" , 56 , 12 , 95 , 0 , 215 , 190 , 255 , 215 } , + {"BlueViolet" , 57 , 12 , 95 , 0 , 255 , 187 , 255 , 255 } , + {"Orange4" , 58 , 3 , 95 , 95 , 0 , 43 , 255 , 95 } , + {"Grey37" , 59 , 8 , 95 , 95 , 95 , 0 , 0 , 95 } , + {"MediumPurple4" , 60 , 4 , 95 , 95 , 135 , 171 , 75 , 135 } , + {"SlateBlue3" , 61 , 4 , 95 , 95 , 175 , 171 , 116 , 175 } , + {"SlateBlue3Bis" , 62 , 12 , 95 , 95 , 215 , 171 , 142 , 215 } , + {"RoyalBlue1" , 63 , 12 , 95 , 95 , 255 , 171 , 160 , 255 } , + {"Chartreuse4" , 64 , 3 , 95 , 135 , 0 , 55 , 255 , 135 } , + {"DarkSeaGreen4" , 65 , 8 , 95 , 135 , 95 , 85 , 75 , 135 } , + {"PaleTurquoise4" , 66 , 6 , 95 , 135 , 135 , 128 , 75 , 135 } , + {"SteelBlue" , 67 , 4 , 95 , 135 , 175 , 150 , 116 , 175 } , + {"SteelBlue3" , 68 , 12 , 95 , 135 , 215 , 157 , 142 , 215 } , + {"CornflowerBlue" , 69 , 12 , 95 , 135 , 255 , 161 , 160 , 255 } , + {"Chartreuse3" , 70 , 3 , 95 , 175 , 0 , 62 , 255 , 175 } , + {"DarkSeaGreen4Bis" , 71 , 2 , 95 , 175 , 95 , 85 , 116 , 175 } , + {"CadetBlue" , 72 , 2 , 95 , 175 , 135 , 106 , 116 , 175 } , + {"CadetBlueBis" , 73 , 6 , 95 , 175 , 175 , 128 , 116 , 175 } , + {"SkyBlue3" , 74 , 14 , 95 , 175 , 215 , 143 , 142 , 215 } , + {"SteelBlue1" , 75 , 12 , 95 , 175 , 255 , 150 , 160 , 255 } , + {"Chartreuse3Bis" , 76 , 10 , 95 , 215 , 0 , 66 , 255 , 215 } , + {"PaleGreen3Bis" , 77 , 10 , 95 , 215 , 95 , 85 , 142 , 215 } , + {"SeaGreen3" , 78 , 10 , 95 , 215 , 135 , 99 , 142 , 215 } , + {"Aquamarine3" , 79 , 14 , 95 , 215 , 175 , 113 , 142 , 215 } , + {"MediumTurquoise" , 80 , 14 , 95 , 215 , 215 , 128 , 142 , 215 } , + {"SteelBlue1Bis" , 81 , 14 , 95 , 215 , 255 , 139 , 160 , 255 } , + {"Chartreuse2Bis" , 82 , 10 , 95 , 255 , 0 , 69 , 255 , 255 } , + {"SeaGreen2" , 83 , 10 , 95 , 255 , 95 , 85 , 160 , 255 } , + {"SeaGreen1" , 84 , 10 , 95 , 255 , 135 , 95 , 160 , 255 } , + {"SeaGreen1Bis" , 85 , 10 , 95 , 255 , 175 , 106 , 160 , 255 } , + {"Aquamarine1Bis" , 86 , 14 , 95 , 255 , 215 , 117 , 160 , 255 } , + {"DarkSlateGray2" , 87 , 14 , 95 , 255 , 255 , 128 , 160 , 255 } , + {"DarkRedBis" , 88 , 1 , 135 , 0 , 0 , 0 , 255 , 135 } , + {"DeepPink4Bis" , 89 , 5 , 135 , 0 , 95 , 226 , 255 , 135 } , + {"DarkMagenta" , 90 , 5 , 135 , 0 , 135 , 213 , 255 , 135 } , + {"DarkMagentaBis" , 91 , 5 , 135 , 0 , 175 , 204 , 255 , 175 } , + {"DarkVioletBis" , 92 , 13 , 135 , 0 , 215 , 198 , 255 , 215 } , + {"PurpleBis" , 93 , 13 , 135 , 0 , 255 , 193 , 255 , 255 } , + {"Orange4Bis" , 94 , 3 , 135 , 95 , 0 , 30 , 255 , 135 } , + {"LightPink4" , 95 , 8 , 135 , 95 , 95 , 0 , 75 , 135 } , + {"Plum4" , 96 , 5 , 135 , 95 , 135 , 213 , 75 , 135 } , + {"MediumPurple3" , 97 , 4 , 135 , 95 , 175 , 192 , 116 , 175 } , + {"MediumPurple3Bis" , 98 , 12 , 135 , 95 , 215 , 185 , 142 , 215 } , + {"SlateBlue1" , 99 , 12 , 135 , 95 , 255 , 181 , 160 , 255 } , + {"Yellow4" , 100 , 3 , 135 , 135 , 0 , 43 , 255 , 135 } , + {"Wheat4" , 101 , 8 , 135 , 135 , 95 , 43 , 75 , 135 } , + {"Grey53" , 102 , 8 , 135 , 135 , 135 , 0 , 0 , 135 } , + {"LightSlateGrey" , 103 , 4 , 135 , 135 , 175 , 171 , 58 , 175 } , + {"MediumPurple" , 104 , 12 , 135 , 135 , 215 , 171 , 94 , 215 } , + {"LightSlateBlue" , 105 , 12 , 135 , 135 , 255 , 171 , 120 , 255 } , + {"Yellow4Bis" , 106 , 3 , 135 , 175 , 0 , 52 , 255 , 175 } , + {"DarkOliveGreen3" , 107 , 7 , 135 , 175 , 95 , 64 , 116 , 175 } , + {"DarkSeaGreen" , 108 , 7 , 135 , 175 , 135 , 85 , 58 , 175 } , + {"LightSkyBlue3" , 109 , 7 , 135 , 175 , 175 , 128 , 58 , 175 } , + {"LightSkyBlue3Bis" , 110 , 12 , 135 , 175 , 215 , 150 , 94 , 215 } , + {"SkyBlue2" , 111 , 12 , 135 , 175 , 255 , 157 , 120 , 255 } , + {"Chartreuse2" , 112 , 11 , 135 , 215 , 0 , 58 , 255 , 215 } , + {"DarkOliveGreen3Bis" , 113 , 10 , 135 , 215 , 95 , 71 , 142 , 215 } , + {"PaleGreen3" , 114 , 7 , 135 , 215 , 135 , 85 , 94 , 215 } , + {"DarkSeaGreen3" , 115 , 10 , 135 , 215 , 175 , 106 , 94 , 215 } , + {"DarkSlateGray3" , 116 , 14 , 135 , 215 , 215 , 128 , 94 , 215 } , + {"SkyBlue1" , 117 , 14 , 135 , 215 , 255 , 143 , 120 , 255 } , + {"Chartreuse1" , 118 , 11 , 135 , 255 , 0 , 63 , 255 , 255 } , + {"LightGreen" , 119 , 10 , 135 , 255 , 95 , 75 , 160 , 255 } , + {"LightGreenBis" , 120 , 10 , 135 , 255 , 135 , 85 , 120 , 255 } , + {"PaleGreen1" , 121 , 10 , 135 , 255 , 175 , 99 , 120 , 255 } , + {"Aquamarine1" , 122 , 14 , 135 , 255 , 215 , 113 , 120 , 255 } , + {"DarkSlateGray1" , 123 , 14 , 135 , 255 , 255 , 128 , 120 , 255 } , + {"Red3" , 124 , 1 , 175 , 0 , 0 , 0 , 255 , 175 } , + {"DeepPink4" , 125 , 5 , 175 , 0 , 95 , 233 , 255 , 175 } , + {"MediumVioletRed" , 126 , 5 , 175 , 0 , 135 , 223 , 255 , 175 } , + {"Magenta3" , 127 , 5 , 175 , 0 , 175 , 213 , 255 , 175 } , + {"DarkViolet" , 128 , 13 , 175 , 0 , 215 , 206 , 255 , 215 } , + {"Purple" , 129 , 13 , 175 , 0 , 255 , 200 , 255 , 255 } , + {"DarkOrange3" , 130 , 3 , 175 , 95 , 0 , 23 , 255 , 175 } , + {"IndianRed" , 131 , 7 , 175 , 95 , 95 , 0 , 116 , 175 } , + {"HotPink3" , 132 , 5 , 175 , 95 , 135 , 235 , 116 , 175 } , + {"MediumOrchid3" , 133 , 5 , 175 , 95 , 175 , 213 , 116 , 175 } , + {"MediumOrchid" , 134 , 13 , 175 , 95 , 215 , 199 , 142 , 215 } , + {"MediumPurple2" , 135 , 12 , 175 , 95 , 255 , 192 , 160 , 255 } , + {"DarkGoldenrod" , 136 , 3 , 175 , 135 , 0 , 33 , 255 , 175 } , + {"LightSalmon3" , 137 , 7 , 175 , 135 , 95 , 21 , 116 , 175 } , + {"RosyBrown" , 138 , 7 , 175 , 135 , 135 , 0 , 58 , 175 } , + {"Grey63" , 139 , 5 , 175 , 135 , 175 , 213 , 58 , 175 } , + {"MediumPurple2Bis" , 140 , 12 , 175 , 135 , 215 , 192 , 94 , 215 } , + {"MediumPurple1" , 141 , 12 , 175 , 135 , 255 , 185 , 120 , 255 } , + {"Gold3" , 142 , 3 , 175 , 175 , 0 , 43 , 255 , 175 } , + {"DarkKhaki" , 143 , 7 , 175 , 175 , 95 , 43 , 116 , 175 } , + {"NavajoWhite3" , 144 , 7 , 175 , 175 , 135 , 43 , 58 , 175 } , + {"Grey69" , 145 , 7 , 175 , 175 , 175 , 0 , 0 , 175 } , + {"LightSteelBlue3" , 146 , 12 , 175 , 175 , 215 , 171 , 47 , 215 } , + {"LightSteelBlue" , 147 , 12 , 175 , 175 , 255 , 171 , 80 , 255 } , + {"Yellow3" , 148 , 11 , 175 , 215 , 0 , 50 , 255 , 215 } , + {"DarkOliveGreen3Ter" , 149 , 11 , 175 , 215 , 95 , 57 , 142 , 215 } , + {"DarkSeaGreen3Bis" , 150 , 7 , 175 , 215 , 135 , 64 , 94 , 215 } , + {"DarkSeaGreen2" , 151 , 7 , 175 , 215 , 175 , 85 , 47 , 215 } , + {"LightCyan3" , 152 , 7 , 175 , 215 , 215 , 128 , 47 , 215 } , + {"LightSkyBlue1" , 153 , 12 , 175 , 215 , 255 , 150 , 80 , 255 } , + {"GreenYellow" , 154 , 11 , 175 , 255 , 0 , 56 , 255 , 255 } , + {"DarkOliveGreen2" , 155 , 10 , 175 , 255 , 95 , 64 , 160 , 255 } , + {"PaleGreen1Bis" , 156 , 10 , 175 , 255 , 135 , 71 , 120 , 255 } , + {"DarkSeaGreen2Bis" , 157 , 15 , 175 , 255 , 175 , 85 , 80 , 255 } , + {"DarkSeaGreen1" , 158 , 15 , 175 , 255 , 215 , 106 , 80 , 255 } , + {"PaleTurquoise1" , 159 , 14 , 175 , 255 , 255 , 128 , 80 , 255 } , + {"Red3Bis" , 160 , 9 , 215 , 0 , 0 , 0 , 255 , 215 } , + {"DeepPink3" , 161 , 13 , 215 , 0 , 95 , 237 , 255 , 215 } , + {"DeepPink3Bis" , 162 , 13 , 215 , 0 , 135 , 229 , 255 , 215 } , + {"Magenta3Bis" , 163 , 13 , 215 , 0 , 175 , 221 , 255 , 215 } , + {"Magenta3Ter" , 164 , 13 , 215 , 0 , 215 , 213 , 255 , 215 } , + {"Magenta2" , 165 , 13 , 215 , 0 , 255 , 207 , 255 , 255 } , + {"DarkOrange3Bis" , 166 , 9 , 215 , 95 , 0 , 19 , 255 , 215 } , + {"IndianRedBis" , 167 , 9 , 215 , 95 , 95 , 0 , 142 , 215 } , + {"HotPink3Bis" , 168 , 13 , 215 , 95 , 135 , 242 , 142 , 215 } , + {"HotPink2" , 169 , 13 , 215 , 95 , 175 , 228 , 142 , 215 } , + {"Orchid" , 170 , 13 , 215 , 95 , 215 , 213 , 142 , 215 } , + {"MediumOrchid1" , 171 , 13 , 215 , 95 , 255 , 203 , 160 , 255 } , + {"Orange3" , 172 , 11 , 215 , 135 , 0 , 27 , 255 , 215 } , + {"LightSalmon3" , 173 , 9 , 215 , 135 , 95 , 14 , 142 , 215 } , + {"LightPink3" , 174 , 7 , 215 , 135 , 135 , 0 , 94 , 215 } , + {"Pink3" , 175 , 13 , 215 , 135 , 175 , 235 , 94 , 215 } , + {"Plum3" , 176 , 13 , 215 , 135 , 215 , 213 , 94 , 215 } , + {"Violet" , 177 , 13 , 215 , 135 , 255 , 199 , 120 , 255 } , + {"Gold3Bis" , 178 , 11 , 215 , 175 , 0 , 35 , 255 , 215 } , + {"LightGoldenrod3" , 179 , 11 , 215 , 175 , 95 , 28 , 142 , 215 } , + {"Tan" , 180 , 7 , 215 , 175 , 135 , 21 , 94 , 215 } , + {"MistyRose3" , 181 , 7 , 215 , 175 , 175 , 0 , 47 , 215 } , + {"Thistle3" , 182 , 13 , 215 , 175 , 215 , 213 , 47 , 215 } , + {"Plum2" , 183 , 12 , 215 , 175 , 255 , 192 , 80 , 255 } , + {"Yellow3Bis" , 184 , 11 , 215 , 215 , 0 , 43 , 255 , 215 } , + {"Khaki3" , 185 , 11 , 215 , 215 , 95 , 43 , 142 , 215 } , + {"LightGoldenrod2" , 186 , 7 , 215 , 215 , 135 , 43 , 94 , 215 } , + {"LightYellow3" , 187 , 7 , 215 , 215 , 175 , 43 , 47 , 215 } , + {"Grey84" , 188 , 7 , 215 , 215 , 215 , 0 , 0 , 215 } , + {"LightSteelBlue1" , 189 , 12 , 215 , 215 , 255 , 171 , 40 , 255 } , + {"Yellow2" , 190 , 11 , 215 , 255 , 0 , 49 , 255 , 255 } , + {"DarkOliveGreen1" , 191 , 11 , 215 , 255 , 95 , 53 , 160 , 255 } , + {"DarkOliveGreen1Bis" , 192 , 11 , 215 , 255 , 135 , 57 , 120 , 255 } , + {"DarkSeaGreen1Bis" , 193 , 15 , 215 , 255 , 175 , 64 , 80 , 255 } , + {"Honeydew2" , 194 , 15 , 215 , 255 , 215 , 85 , 40 , 255 } , + {"LightCyan1Bis" , 195 , 15 , 215 , 255 , 255 , 128 , 40 , 255 } , + {"Red1" , 196 , 9 , 255 , 0 , 0 , 0 , 255 , 255 } , + {"DeepPink2" , 197 , 13 , 255 , 0 , 95 , 240 , 255 , 255 } , + {"DeepPink1" , 198 , 13 , 255 , 0 , 135 , 234 , 255 , 255 } , + {"DeepPink1Bis" , 199 , 13 , 255 , 0 , 175 , 227 , 255 , 255 } , + {"Magenta2Bis" , 200 , 13 , 255 , 0 , 215 , 220 , 255 , 255 } , + {"Magenta1" , 201 , 13 , 255 , 0 , 255 , 213 , 255 , 255 } , + {"OrangeRed1" , 202 , 9 , 255 , 95 , 0 , 16 , 255 , 255 } , + {"IndianRed1" , 203 , 9 , 255 , 95 , 95 , 0 , 160 , 255 } , + {"IndianRed1Bis" , 204 , 13 , 255 , 95 , 135 , 246 , 160 , 255 } , + {"HotPink" , 205 , 13 , 255 , 95 , 175 , 235 , 160 , 255 } , + {"HotPinkBis" , 206 , 13 , 255 , 95 , 215 , 224 , 160 , 255 } , + {"MediumOrchid1Bis" , 207 , 13 , 255 , 95 , 255 , 213 , 160 , 255 } , + {"DarkOrange" , 208 , 11 , 255 , 135 , 0 , 22 , 255 , 255 } , + {"Salmon1" , 209 , 9 , 255 , 135 , 95 , 10 , 160 , 255 } , + {"LightCoral" , 210 , 15 , 255 , 135 , 135 , 0 , 120 , 255 } , + {"PaleVioletRed1" , 211 , 13 , 255 , 135 , 175 , 242 , 120 , 255 } , + {"Orchid2" , 212 , 13 , 255 , 135 , 215 , 228 , 120 , 255 } , + {"Orchid1" , 213 , 13 , 255 , 135 , 255 , 213 , 120 , 255 } , + {"Orange1" , 214 , 11 , 255 , 175 , 0 , 29 , 255 , 255 } , + {"SandyBrown" , 215 , 9 , 255 , 175 , 95 , 21 , 160 , 255 } , + {"LightSalmon1" , 216 , 15 , 255 , 175 , 135 , 14 , 120 , 255 } , + {"LightPink1" , 217 , 15 , 255 , 175 , 175 , 0 , 80 , 255 } , + {"Pink1" , 218 , 13 , 255 , 175 , 215 , 235 , 80 , 255 } , + {"Plum1" , 219 , 13 , 255 , 175 , 255 , 213 , 80 , 255 } , + {"Gold1" , 220 , 11 , 255 , 215 , 0 , 36 , 255 , 255 } , + {"LightGoldenrod2Bis" , 221 , 11 , 255 , 215 , 95 , 32 , 160 , 255 } , + {"LightGoldenrod2Ter" , 222 , 15 , 255 , 215 , 135 , 28 , 120 , 255 } , + {"NavajoWhite1" , 223 , 15 , 255 , 215 , 175 , 21 , 80 , 255 } , + {"MistyRose1" , 224 , 15 , 255 , 215 , 215 , 0 , 40 , 255 } , + {"Thistle1" , 225 , 13 , 255 , 215 , 255 , 213 , 40 , 255 } , + {"Yellow1" , 226 , 11 , 255 , 255 , 0 , 43 , 255 , 255 } , + {"LightGoldenrod1" , 227 , 11 , 255 , 255 , 95 , 43 , 160 , 255 } , + {"Khaki1" , 228 , 15 , 255 , 255 , 135 , 43 , 120 , 255 } , + {"Wheat1" , 229 , 15 , 255 , 255 , 175 , 43 , 80 , 255 } , + {"Cornsilk1" , 230 , 15 , 255 , 255 , 215 , 43 , 40 , 255 } , + {"Grey100" , 231 , 15 , 255 , 255 , 255 , 0 , 0 , 255 } , + {"Grey3" , 232 , 0 , 8 , 8 , 8 , 0 , 0 , 8 } , + {"Grey7" , 233 , 0 , 18 , 18 , 18 , 0 , 0 , 18 } , + {"Grey11" , 234 , 0 , 28 , 28 , 28 , 0 , 0 , 28 } , + {"Grey15" , 235 , 0 , 38 , 38 , 38 , 0 , 0 , 38 } , + {"Grey19" , 236 , 0 , 48 , 48 , 48 , 0 , 0 , 48 } , + {"Grey23" , 237 , 0 , 58 , 58 , 58 , 0 , 0 , 58 } , + {"Grey27" , 238 , 8 , 68 , 68 , 68 , 0 , 0 , 68 } , + {"Grey30" , 239 , 8 , 78 , 78 , 78 , 0 , 0 , 78 } , + {"Grey35" , 240 , 8 , 88 , 88 , 88 , 0 , 0 , 88 } , + {"Grey39" , 241 , 8 , 98 , 98 , 98 , 0 , 0 , 98 } , + {"Grey42" , 242 , 8 , 108 , 108 , 108 , 0 , 0 , 108 } , + {"Grey46" , 243 , 8 , 118 , 118 , 118 , 0 , 0 , 118 } , + {"Grey50" , 244 , 8 , 128 , 128 , 128 , 0 , 0 , 128 } , + {"Grey54" , 245 , 8 , 138 , 138 , 138 , 0 , 0 , 138 } , + {"Grey58" , 246 , 8 , 148 , 148 , 148 , 0 , 0 , 148 } , + {"Grey62" , 247 , 8 , 158 , 158 , 158 , 0 , 0 , 158 } , + {"Grey66" , 248 , 7 , 168 , 168 , 168 , 0 , 0 , 168 } , + {"Grey70" , 249 , 7 , 178 , 178 , 178 , 0 , 0 , 178 } , + {"Grey74" , 250 , 7 , 188 , 188 , 188 , 0 , 0 , 188 } , + {"Grey78" , 251 , 7 , 198 , 198 , 198 , 0 , 0 , 198 } , + {"Grey82" , 252 , 7 , 208 , 208 , 208 , 0 , 0 , 208 } , + {"Grey85" , 253 , 7 , 218 , 218 , 218 , 0 , 0 , 218 } , + {"Grey89" , 254 , 15 , 228 , 228 , 228 , 0 , 0 , 228 } , + {"Grey93" , 255 , 15 , 238 , 238 , 238 , 0 , 0 , 238 } , +}}; + +ColorInfo GetColorInfo(Color::Palette256 index) { + return palette256[index]; // NOLINT; +} + +ColorInfo GetColorInfo(Color::Palette16 index) { + return palette256[index]; // NOLINT; +} +// clang-format off + +} diff --git a/contrib/libs/ftxui/src/ftxui/screen/image.cpp b/contrib/libs/ftxui/src/ftxui/screen/image.cpp new file mode 100644 index 00000000000..55e4b135ff5 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/screen/image.cpp @@ -0,0 +1,63 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <sstream> // IWYU pragma: keep +#include <string> +#include <vector> + +#include "ftxui/screen/image.hpp" +#include "ftxui/screen/pixel.hpp" + +namespace ftxui { + +namespace { +Pixel& dev_null_pixel() { + static Pixel pixel; + return pixel; +} +} // namespace + +Image::Image(int dimx, int dimy) + : stencil{0, dimx - 1, 0, dimy - 1}, + dimx_(dimx), + dimy_(dimy), + pixels_(dimy, std::vector<Pixel>(dimx)) {} + +/// @brief Access a character in a cell at a given position. +/// @param x The cell position along the x-axis. +/// @param y The cell position along the y-axis. +std::string& Image::at(int x, int y) { + return PixelAt(x, y).character; +} + +/// @brief Access a character in a cell at a given position. +/// @param x The cell position along the x-axis. +/// @param y The cell position along the y-axis. +const std::string& Image::at(int x, int y) const { + return PixelAt(x, y).character; +} + +/// @brief Access a cell (Pixel) at a given position. +/// @param x The cell position along the x-axis. +/// @param y The cell position along the y-axis. +Pixel& Image::PixelAt(int x, int y) { + return stencil.Contain(x, y) ? pixels_[y][x] : dev_null_pixel(); +} + +/// @brief Access a cell (Pixel) at a given position. +/// @param x The cell position along the x-axis. +/// @param y The cell position along the y-axis. +const Pixel& Image::PixelAt(int x, int y) const { + return stencil.Contain(x, y) ? pixels_[y][x] : dev_null_pixel(); +} + +/// @brief Clear all the pixel from the screen. +void Image::Clear() { + for (auto& line : pixels_) { + for (auto& cell : line) { + cell = Pixel(); + } + } +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/screen/screen.cpp b/contrib/libs/ftxui/src/ftxui/screen/screen.cpp new file mode 100644 index 00000000000..ab087fcc991 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/screen/screen.cpp @@ -0,0 +1,565 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <cstddef> // for size_t +#include <cstdint> +#include <iostream> // for operator<<, stringstream, basic_ostream, flush, cout, ostream +#include <limits> +#include <map> // for _Rb_tree_const_iterator, map, operator!=, operator== +#include <sstream> // IWYU pragma: keep +#include <utility> // for pair + +#include "ftxui/screen/image.hpp" // for Image +#include "ftxui/screen/pixel.hpp" // for Pixel +#include "ftxui/screen/screen.hpp" +#include "ftxui/screen/string.hpp" // for string_width +#include "ftxui/screen/terminal.hpp" // for Dimensions, Size + +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include <windows.h> +#endif + +// Macro for hinting that an expression is likely to be false. +#if !defined(FTXUI_UNLIKELY) +#if defined(COMPILER_GCC) || defined(__clang__) +#define FTXUI_UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define FTXUI_UNLIKELY(x) (x) +#endif // defined(COMPILER_GCC) +#endif // !defined(FTXUI_UNLIKELY) + +#if !defined(FTXUI_LIKELY) +#if defined(COMPILER_GCC) || defined(__clang__) +#define FTXUI_LIKELY(x) __builtin_expect(!!(x), 1) +#else +#define FTXUI_LIKELY(x) (x) +#endif // defined(COMPILER_GCC) +#endif // !defined(FTXUI_LIKELY) + +namespace ftxui { + +namespace { + +#if defined(_WIN32) +void WindowsEmulateVT100Terminal() { + static bool done = false; + if (done) + return; + done = true; + + // Enable VT processing on stdout and stdin + auto stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE); + + DWORD out_mode = 0; + GetConsoleMode(stdout_handle, &out_mode); + + // https://docs.microsoft.com/en-us/windows/console/setconsolemode + const int enable_virtual_terminal_processing = 0x0004; + const int disable_newline_auto_return = 0x0008; + out_mode |= enable_virtual_terminal_processing; + out_mode |= disable_newline_auto_return; + + SetConsoleMode(stdout_handle, out_mode); +} +#endif + +// NOLINTNEXTLINE(readability-function-cognitive-complexity) +void UpdatePixelStyle(const Screen* screen, + std::stringstream& ss, + const Pixel& prev, + const Pixel& next) { + // See https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda + if (FTXUI_UNLIKELY(next.hyperlink != prev.hyperlink)) { + ss << "\x1B]8;;" << screen->Hyperlink(next.hyperlink) << "\x1B\\"; + } + + // Bold + if (FTXUI_UNLIKELY((next.bold ^ prev.bold) | (next.dim ^ prev.dim))) { + // BOLD_AND_DIM_RESET: + ss << ((prev.bold && !next.bold) || (prev.dim && !next.dim) ? "\x1B[22m" + : ""); + ss << (next.bold ? "\x1B[1m" : ""); // BOLD_SET + ss << (next.dim ? "\x1B[2m" : ""); // DIM_SET + } + + // Underline + if (FTXUI_UNLIKELY(next.underlined != prev.underlined || + next.underlined_double != prev.underlined_double)) { + ss << (next.underlined ? "\x1B[4m" // UNDERLINE + : next.underlined_double ? "\x1B[21m" // UNDERLINE_DOUBLE + : "\x1B[24m"); // UNDERLINE_RESET + } + + // Blink + if (FTXUI_UNLIKELY(next.blink != prev.blink)) { + ss << (next.blink ? "\x1B[5m" // BLINK_SET + : "\x1B[25m"); // BLINK_RESET + } + + // Inverted + if (FTXUI_UNLIKELY(next.inverted != prev.inverted)) { + ss << (next.inverted ? "\x1B[7m" // INVERTED_SET + : "\x1B[27m"); // INVERTED_RESET + } + + // Italics + if (FTXUI_UNLIKELY(next.italic != prev.italic)) { + ss << (next.italic ? "\x1B[3m" // ITALIC_SET + : "\x1B[23m"); // ITALIC_RESET + } + + // StrikeThrough + if (FTXUI_UNLIKELY(next.strikethrough != prev.strikethrough)) { + ss << (next.strikethrough ? "\x1B[9m" // CROSSED_OUT + : "\x1B[29m"); // CROSSED_OUT_RESET + } + + if (FTXUI_UNLIKELY(next.foreground_color != prev.foreground_color || + next.background_color != prev.background_color)) { + ss << "\x1B[" + next.foreground_color.Print(false) + "m"; + ss << "\x1B[" + next.background_color.Print(true) + "m"; + } +} + +struct TileEncoding { + std::uint8_t left : 2; + std::uint8_t top : 2; + std::uint8_t right : 2; + std::uint8_t down : 2; + std::uint8_t round : 1; + + // clang-format off + bool operator<(const TileEncoding& other) const { + if (left < other.left) { return true; } + if (left > other.left) { return false; } + if (top < other.top) { return true; } + if (top > other.top) { return false; } + if (right < other.right) { return true; } + if (right > other.right) { return false; } + if (down < other.down) { return true; } + if (down > other.down) { return false; } + if (round < other.round) { return true; } + if (round > other.round) { return false; } + return false; + } + // clang-format on +}; + +// clang-format off +const std::map<std::string, TileEncoding> tile_encoding = { // NOLINT + {"─", {1, 0, 1, 0, 0}}, + {"━", {2, 0, 2, 0, 0}}, + {"╍", {2, 0, 2, 0, 0}}, + + {"│", {0, 1, 0, 1, 0}}, + {"┃", {0, 2, 0, 2, 0}}, + {"╏", {0, 2, 0, 2, 0}}, + + {"┌", {0, 0, 1, 1, 0}}, + {"┍", {0, 0, 2, 1, 0}}, + {"┎", {0, 0, 1, 2, 0}}, + {"┏", {0, 0, 2, 2, 0}}, + + {"┐", {1, 0, 0, 1, 0}}, + {"┑", {2, 0, 0, 1, 0}}, + {"┒", {1, 0, 0, 2, 0}}, + {"┓", {2, 0, 0, 2, 0}}, + + {"└", {0, 1, 1, 0, 0}}, + {"┕", {0, 1, 2, 0, 0}}, + {"┖", {0, 2, 1, 0, 0}}, + {"┗", {0, 2, 2, 0, 0}}, + + {"┘", {1, 1, 0, 0, 0}}, + {"┙", {2, 1, 0, 0, 0}}, + {"┚", {1, 2, 0, 0, 0}}, + {"┛", {2, 2, 0, 0, 0}}, + + {"├", {0, 1, 1, 1, 0}}, + {"┝", {0, 1, 2, 1, 0}}, + {"┞", {0, 2, 1, 1, 0}}, + {"┟", {0, 1, 1, 2, 0}}, + {"┠", {0, 2, 1, 2, 0}}, + {"┡", {0, 2, 2, 1, 0}}, + {"┢", {0, 1, 2, 2, 0}}, + {"┣", {0, 2, 2, 2, 0}}, + + {"┤", {1, 1, 0, 1, 0}}, + {"┥", {2, 1, 0, 1, 0}}, + {"┦", {1, 2, 0, 1, 0}}, + {"┧", {1, 1, 0, 2, 0}}, + {"┨", {1, 2, 0, 2, 0}}, + {"┩", {2, 2, 0, 1, 0}}, + {"┪", {2, 1, 0, 2, 0}}, + {"┫", {2, 2, 0, 2, 0}}, + + {"┬", {1, 0, 1, 1, 0}}, + {"┭", {2, 0, 1, 1, 0}}, + {"┮", {1, 0, 2, 1, 0}}, + {"┯", {2, 0, 2, 1, 0}}, + {"┰", {1, 0, 1, 2, 0}}, + {"┱", {2, 0, 1, 2, 0}}, + {"┲", {1, 0, 2, 2, 0}}, + {"┳", {2, 0, 2, 2, 0}}, + + {"┴", {1, 1, 1, 0, 0}}, + {"┵", {2, 1, 1, 0, 0}}, + {"┶", {1, 1, 2, 0, 0}}, + {"┷", {2, 1, 2, 0, 0}}, + {"┸", {1, 2, 1, 0, 0}}, + {"┹", {2, 2, 1, 0, 0}}, + {"┺", {1, 2, 2, 0, 0}}, + {"┻", {2, 2, 2, 0, 0}}, + + {"┼", {1, 1, 1, 1, 0}}, + {"┽", {2, 1, 1, 1, 0}}, + {"┾", {1, 1, 2, 1, 0}}, + {"┿", {2, 1, 2, 1, 0}}, + {"╀", {1, 2, 1, 1, 0}}, + {"╁", {1, 1, 1, 2, 0}}, + {"╂", {1, 2, 1, 2, 0}}, + {"╃", {2, 2, 1, 1, 0}}, + {"╄", {1, 2, 2, 1, 0}}, + {"╅", {2, 1, 1, 2, 0}}, + {"╆", {1, 1, 2, 2, 0}}, + {"╇", {2, 2, 2, 1, 0}}, + {"╈", {2, 1, 2, 2, 0}}, + {"╉", {2, 2, 1, 2, 0}}, + {"╊", {1, 2, 2, 2, 0}}, + {"╋", {2, 2, 2, 2, 0}}, + + {"═", {3, 0, 3, 0, 0}}, + {"║", {0, 3, 0, 3, 0}}, + + {"╒", {0, 0, 3, 1, 0}}, + {"╓", {0, 0, 1, 3, 0}}, + {"╔", {0, 0, 3, 3, 0}}, + + {"╕", {3, 0, 0, 1, 0}}, + {"╖", {1, 0, 0, 3, 0}}, + {"╗", {3, 0, 0, 3, 0}}, + + {"╘", {0, 1, 3, 0, 0}}, + {"╙", {0, 3, 1, 0, 0}}, + {"╚", {0, 3, 3, 0, 0}}, + + {"╛", {3, 1, 0, 0, 0}}, + {"╜", {1, 3, 0, 0, 0}}, + {"╝", {3, 3, 0, 0, 0}}, + + {"╞", {0, 1, 3, 1, 0}}, + {"╟", {0, 3, 1, 3, 0}}, + {"╠", {0, 3, 3, 3, 0}}, + + {"╡", {3, 1, 0, 1, 0}}, + {"╢", {1, 3, 0, 3, 0}}, + {"╣", {3, 3, 0, 3, 0}}, + + {"╤", {3, 0, 3, 1, 0}}, + {"╥", {1, 0, 1, 3, 0}}, + {"╦", {3, 0, 3, 3, 0}}, + + {"╧", {3, 1, 3, 0, 0}}, + {"╨", {1, 3, 1, 0, 0}}, + {"╩", {3, 3, 3, 0, 0}}, + + {"╪", {3, 1, 3, 1, 0}}, + {"╫", {1, 3, 1, 3, 0}}, + {"╬", {3, 3, 3, 3, 0}}, + + {"╭", {0, 0, 1, 1, 1}}, + {"╮", {1, 0, 0, 1, 1}}, + {"╯", {1, 1, 0, 0, 1}}, + {"╰", {0, 1, 1, 0, 1}}, + + {"╴", {1, 0, 0, 0, 0}}, + {"╵", {0, 1, 0, 0, 0}}, + {"╶", {0, 0, 1, 0, 0}}, + {"╷", {0, 0, 0, 1, 0}}, + + {"╸", {2, 0, 0, 0, 0}}, + {"╹", {0, 2, 0, 0, 0}}, + {"╺", {0, 0, 2, 0, 0}}, + {"╻", {0, 0, 0, 2, 0}}, + + {"╼", {1, 0, 2, 0, 0}}, + {"╽", {0, 1, 0, 2, 0}}, + {"╾", {2, 0, 1, 0, 0}}, + {"╿", {0, 2, 0, 1, 0}}, +}; +// clang-format on + +template <class A, class B> +std::map<B, A> InvertMap(const std::map<A, B> input) { + std::map<B, A> output; + for (const auto& it : input) { + output[it.second] = it.first; + } + return output; +} + +const std::map<TileEncoding, std::string> tile_encoding_inverse = // NOLINT + InvertMap(tile_encoding); + +void UpgradeLeftRight(std::string& left, std::string& right) { + const auto it_left = tile_encoding.find(left); + if (it_left == tile_encoding.end()) { + return; + } + const auto it_right = tile_encoding.find(right); + if (it_right == tile_encoding.end()) { + return; + } + + if (it_left->second.right == 0 && it_right->second.left != 0) { + TileEncoding encoding_left = it_left->second; + encoding_left.right = it_right->second.left; + const auto it_left_upgrade = tile_encoding_inverse.find(encoding_left); + if (it_left_upgrade != tile_encoding_inverse.end()) { + left = it_left_upgrade->second; + } + } + + if (it_right->second.left == 0 && it_left->second.right != 0) { + TileEncoding encoding_right = it_right->second; + encoding_right.left = it_left->second.right; + const auto it_right_upgrade = tile_encoding_inverse.find(encoding_right); + if (it_right_upgrade != tile_encoding_inverse.end()) { + right = it_right_upgrade->second; + } + } +} + +void UpgradeTopDown(std::string& top, std::string& down) { + const auto it_top = tile_encoding.find(top); + if (it_top == tile_encoding.end()) { + return; + } + const auto it_down = tile_encoding.find(down); + if (it_down == tile_encoding.end()) { + return; + } + + if (it_top->second.down == 0 && it_down->second.top != 0) { + TileEncoding encoding_top = it_top->second; + encoding_top.down = it_down->second.top; + const auto it_top_down = tile_encoding_inverse.find(encoding_top); + if (it_top_down != tile_encoding_inverse.end()) { + top = it_top_down->second; + } + } + + if (it_down->second.top == 0 && it_top->second.down != 0) { + TileEncoding encoding_down = it_down->second; + encoding_down.top = it_top->second.down; + const auto it_down_top = tile_encoding_inverse.find(encoding_down); + if (it_down_top != tile_encoding_inverse.end()) { + down = it_down_top->second; + } + } +} + +bool ShouldAttemptAutoMerge(Pixel& pixel) { + return pixel.automerge && pixel.character.size() == 3; +} + +} // namespace + +/// A fixed dimension. +/// @see Fit +/// @see Full +Dimensions Dimension::Fixed(int v) { + return {v, v}; +} + +/// Use the terminal dimensions. +/// @see Fixed +/// @see Fit +Dimensions Dimension::Full() { + return Terminal::Size(); +} + +// static +/// Create a screen with the given dimension along the x-axis and y-axis. +Screen Screen::Create(Dimensions width, Dimensions height) { + return {width.dimx, height.dimy}; +} + +// static +/// Create a screen with the given dimension. +Screen Screen::Create(Dimensions dimension) { + return {dimension.dimx, dimension.dimy}; +} + +Screen::Screen(int dimx, int dimy) : Image{dimx, dimy} { +#if defined(_WIN32) + // The placement of this call is a bit weird, however we can assume that + // anybody who instantiates a Screen object eventually wants to output + // something to the console. If that is not the case, use an instance of Image + // instead. As we require UTF8 for all input/output operations we will just + // switch to UTF8 encoding here + SetConsoleOutputCP(CP_UTF8); + SetConsoleCP(CP_UTF8); + WindowsEmulateVT100Terminal(); +#endif +} + +/// Produce a std::string that can be used to print the Screen on the +/// terminal. +/// @note Don't forget to flush stdout. Alternatively, you can use +/// Screen::Print(); +std::string Screen::ToString() const { + std::stringstream ss; + + const Pixel default_pixel; + const Pixel* previous_pixel_ref = &default_pixel; + + for (int y = 0; y < dimy_; ++y) { + // New line in between two lines. + if (y != 0) { + UpdatePixelStyle(this, ss, *previous_pixel_ref, default_pixel); + previous_pixel_ref = &default_pixel; + ss << "\r\n"; + } + + // After printing a fullwith character, we need to skip the next cell. + bool previous_fullwidth = false; + for (const auto& pixel : pixels_[y]) { + if (!previous_fullwidth) { + UpdatePixelStyle(this, ss, *previous_pixel_ref, pixel); + previous_pixel_ref = &pixel; + if (pixel.character.empty()) { + ss << " "; + } else { + ss << pixel.character; + } + } + previous_fullwidth = (string_width(pixel.character) == 2); + } + } + + // Reset the style to default: + UpdatePixelStyle(this, ss, *previous_pixel_ref, default_pixel); + + return ss.str(); +} + +// Print the Screen to the terminal. +void Screen::Print() const { + std::cout << ToString() << '\0' << std::flush; +} + +/// @brief Return a string to be printed in order to reset the cursor position +/// to the beginning of the screen. +/// +/// ```cpp +/// std::string reset_position; +/// while(true) { +/// auto document = render(); +/// auto screen = Screen::Create(Dimension::Full(), Dimension::Fit(document)); +/// Render(screen, document); +/// std::cout << reset_position << screen.ToString() << std::flush; +/// reset_position = screen.ResetPosition(); +/// +/// using namespace std::chrono_literals; +/// std::this_thread::sleep_for(0.01s); +/// } +/// ``` +/// +/// @return The string to print in order to reset the cursor position to the +/// beginning. +std::string Screen::ResetPosition(bool clear) const { + std::stringstream ss; + if (clear) { + ss << "\r"; // MOVE_LEFT; + ss << "\x1b[2K"; // CLEAR_SCREEN; + for (int y = 1; y < dimy_; ++y) { + ss << "\x1B[1A"; // MOVE_UP; + ss << "\x1B[2K"; // CLEAR_LINE; + } + } else { + ss << "\r"; // MOVE_LEFT; + for (int y = 1; y < dimy_; ++y) { + ss << "\x1B[1A"; // MOVE_UP; + } + } + return ss.str(); +} + +/// @brief Clear all the pixel from the screen. +void Screen::Clear() { + Image::Clear(); + + cursor_.x = dimx_ - 1; + cursor_.y = dimy_ - 1; + + hyperlinks_ = { + "", + }; +} + +// clang-format off +void Screen::ApplyShader() { + // Merge box characters togethers. + for (int y = 0; y < dimy_; ++y) { + for (int x = 0; x < dimx_; ++x) { + // Box drawing character uses exactly 3 byte. + Pixel& cur = pixels_[y][x]; + if (!ShouldAttemptAutoMerge(cur)) { + continue; + } + + if (x > 0) { + Pixel& left = pixels_[y][x-1]; + if (ShouldAttemptAutoMerge(left)) { + UpgradeLeftRight(left.character, cur.character); + } + } + if (y > 0) { + Pixel& top = pixels_[y-1][x]; + if (ShouldAttemptAutoMerge(top)) { + UpgradeTopDown(top.character, cur.character); + } + } + } + } +} +// clang-format on + +std::uint8_t Screen::RegisterHyperlink(const std::string& link) { + for (std::size_t i = 0; i < hyperlinks_.size(); ++i) { + if (hyperlinks_[i] == link) { + return i; + } + } + if (hyperlinks_.size() == std::numeric_limits<std::uint8_t>::max()) { + return 0; + } + hyperlinks_.push_back(link); + return hyperlinks_.size() - 1; +} + +const std::string& Screen::Hyperlink(std::uint8_t id) const { + if (id >= hyperlinks_.size()) { + return hyperlinks_[0]; + } + return hyperlinks_[id]; +} + +/// @brief Return the current selection style. +/// @see SetSelectionStyle +const Screen::SelectionStyle& Screen::GetSelectionStyle() const { + return selection_style_; +} + +/// @brief Set the current selection style. +/// @see GetSelectionStyle +void Screen::SetSelectionStyle(SelectionStyle decorator) { + selection_style_ = std::move(decorator); +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/screen/string.cpp b/contrib/libs/ftxui/src/ftxui/screen/string.cpp new file mode 100644 index 00000000000..f07b47a5da0 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/screen/string.cpp @@ -0,0 +1,1669 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +// +// Content of this file was created thanks to: +// - +// https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/WordBreakProperty.txt +// - Markus Kuhn -- 2007-05-26 (Unicode 5.0) +// http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c +// Thanks you! + +#include "ftxui/screen/string.hpp" + +#include <array> // for array +#include <cstddef> // for size_t +#include <cstdint> // for uint32_t, uint8_t, uint16_t, int32_t +#include <string> // for string, basic_string, wstring +#include <tuple> // for _Swallow_assign, ignore +#include <vector> + +#include "ftxui/screen/deprecated.hpp" // for wchar_width, wstring_width +#include "ftxui/screen/string_internal.hpp" // for WordBreakProperty, EatCodePoint, CodepointToWordBreakProperty, GlyphCount, GlyphIterate, GlyphNext, GlyphPrevious, IsCombining, IsControl, IsFullWidth, Utf8ToWordBreakProperty + +namespace { + +struct Interval { + uint32_t first; + uint32_t last; +}; + +// As of Unicode 13.0.0 +constexpr std::array<Interval, 116> g_full_width_characters = {{ + {0x01100, 0x0115f}, {0x0231a, 0x0231b}, {0x02329, 0x0232a}, + {0x023e9, 0x023ec}, {0x023f0, 0x023f0}, {0x023f3, 0x023f3}, + {0x025fd, 0x025fe}, {0x02614, 0x02615}, {0x02648, 0x02653}, + {0x0267f, 0x0267f}, {0x02693, 0x02693}, {0x026a1, 0x026a1}, + {0x026aa, 0x026ab}, {0x026bd, 0x026be}, {0x026c4, 0x026c5}, + {0x026ce, 0x026ce}, {0x026d4, 0x026d4}, {0x026ea, 0x026ea}, + {0x026f2, 0x026f3}, {0x026f5, 0x026f5}, {0x026fa, 0x026fa}, + {0x026fd, 0x026fd}, {0x02705, 0x02705}, {0x0270a, 0x0270b}, + {0x02728, 0x02728}, {0x0274c, 0x0274c}, {0x0274e, 0x0274e}, + {0x02753, 0x02755}, {0x02757, 0x02757}, {0x02795, 0x02797}, + {0x027b0, 0x027b0}, {0x027bf, 0x027bf}, {0x02b1b, 0x02b1c}, + {0x02b50, 0x02b50}, {0x02b55, 0x02b55}, {0x02e80, 0x02e99}, + {0x02e9b, 0x02ef3}, {0x02f00, 0x02fd5}, {0x02ff0, 0x02ffb}, + {0x03000, 0x0303e}, {0x03041, 0x03096}, {0x03099, 0x030ff}, + {0x03105, 0x0312f}, {0x03131, 0x0318e}, {0x03190, 0x031e3}, + {0x031f0, 0x0321e}, {0x03220, 0x03247}, {0x03250, 0x04dbf}, + {0x04e00, 0x0a48c}, {0x0a490, 0x0a4c6}, {0x0a960, 0x0a97c}, + {0x0ac00, 0x0d7a3}, {0x0f900, 0x0faff}, {0x0fe10, 0x0fe19}, + {0x0fe30, 0x0fe52}, {0x0fe54, 0x0fe66}, {0x0fe68, 0x0fe6b}, + {0x0ff01, 0x0ff60}, {0x0ffe0, 0x0ffe6}, {0x16fe0, 0x16fe4}, + {0x16ff0, 0x16ff1}, {0x17000, 0x187f7}, {0x18800, 0x18cd5}, + {0x18d00, 0x18d08}, {0x1b000, 0x1b11e}, {0x1b150, 0x1b152}, + {0x1b164, 0x1b167}, {0x1b170, 0x1b2fb}, {0x1f004, 0x1f004}, + {0x1f0cf, 0x1f0cf}, {0x1f18e, 0x1f18e}, {0x1f191, 0x1f19a}, + {0x1f200, 0x1f202}, {0x1f210, 0x1f23b}, {0x1f240, 0x1f248}, + {0x1f250, 0x1f251}, {0x1f260, 0x1f265}, {0x1f300, 0x1f320}, + {0x1f32d, 0x1f335}, {0x1f337, 0x1f37c}, {0x1f37e, 0x1f393}, + {0x1f3a0, 0x1f3ca}, {0x1f3cf, 0x1f3d3}, {0x1f3e0, 0x1f3f0}, + {0x1f3f4, 0x1f3f4}, {0x1f3f8, 0x1f43e}, {0x1f440, 0x1f440}, + {0x1f442, 0x1f4fc}, {0x1f4ff, 0x1f53d}, {0x1f54b, 0x1f54e}, + {0x1f550, 0x1f567}, {0x1f57a, 0x1f57a}, {0x1f595, 0x1f596}, + {0x1f5a4, 0x1f5a4}, {0x1f5fb, 0x1f64f}, {0x1f680, 0x1f6c5}, + {0x1f6cc, 0x1f6cc}, {0x1f6d0, 0x1f6d2}, {0x1f6d5, 0x1f6d7}, + {0x1f6eb, 0x1f6ec}, {0x1f6f4, 0x1f6fc}, {0x1f7e0, 0x1f7eb}, + {0x1f90c, 0x1f93a}, {0x1f93c, 0x1f945}, {0x1f947, 0x1f978}, + {0x1f97a, 0x1f9cb}, {0x1f9cd, 0x1f9ff}, {0x1fa70, 0x1fa74}, + {0x1fa78, 0x1fa7a}, {0x1fa80, 0x1fa86}, {0x1fa90, 0x1faa8}, + {0x1fab0, 0x1fab6}, {0x1fac0, 0x1fac2}, {0x1fad0, 0x1fad6}, + {0x20000, 0x2fffd}, {0x30000, 0x3fffd}, +}}; + +using WBP = ftxui::WordBreakProperty; +struct WordBreakPropertyInterval { + uint32_t first; + uint32_t last; + WBP property; +}; + +// Properties from: +// https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/WordBreakProperty.txt +constexpr std::array<WordBreakPropertyInterval, 993> g_word_break_intervals = {{ + {0x0000A, 0x0000A, WBP::LF}, + {0x0000B, 0x0000C, WBP::Newline}, + {0x0000D, 0x0000D, WBP::CR}, + {0x00020, 0x00020, WBP::WSegSpace}, + {0x00022, 0x00022, WBP::Double_Quote}, + {0x00027, 0x00027, WBP::Single_Quote}, + {0x0002C, 0x0002C, WBP::MidNum}, + {0x0002E, 0x0002E, WBP::MidNumLet}, + {0x00030, 0x00039, WBP::Numeric}, + {0x0003A, 0x0003A, WBP::MidLetter}, + {0x0003B, 0x0003B, WBP::MidNum}, + {0x00041, 0x0005A, WBP::ALetter}, + {0x0005F, 0x0005F, WBP::ExtendNumLet}, + {0x00061, 0x0007A, WBP::ALetter}, + {0x00085, 0x00085, WBP::Newline}, + {0x000AA, 0x000AA, WBP::ALetter}, + {0x000AD, 0x000AD, WBP::Format}, + {0x000B5, 0x000B5, WBP::ALetter}, + {0x000B7, 0x000B7, WBP::MidLetter}, + {0x000BA, 0x000BA, WBP::ALetter}, + {0x000C0, 0x000D6, WBP::ALetter}, + {0x000D8, 0x000F6, WBP::ALetter}, + {0x000F8, 0x002D7, WBP::ALetter}, + {0x002DE, 0x002FF, WBP::ALetter}, + {0x00300, 0x0036F, WBP::Extend}, + {0x00370, 0x00374, WBP::ALetter}, + {0x00376, 0x00377, WBP::ALetter}, + {0x0037A, 0x0037D, WBP::ALetter}, + {0x0037E, 0x0037E, WBP::MidNum}, + {0x0037F, 0x0037F, WBP::ALetter}, + {0x00386, 0x00386, WBP::ALetter}, + {0x00387, 0x00387, WBP::MidLetter}, + {0x00388, 0x0038A, WBP::ALetter}, + {0x0038C, 0x0038C, WBP::ALetter}, + {0x0038E, 0x003A1, WBP::ALetter}, + {0x003A3, 0x003F5, WBP::ALetter}, + {0x003F7, 0x00481, WBP::ALetter}, + {0x00483, 0x00489, WBP::Extend}, + {0x0048A, 0x0052F, WBP::ALetter}, + {0x00531, 0x00556, WBP::ALetter}, + {0x00559, 0x0055C, WBP::ALetter}, + {0x0055E, 0x0055E, WBP::ALetter}, + {0x0055F, 0x0055F, WBP::MidLetter}, + {0x00560, 0x00588, WBP::ALetter}, + {0x00589, 0x00589, WBP::MidNum}, + {0x0058A, 0x0058A, WBP::ALetter}, + {0x00591, 0x005BD, WBP::Extend}, + {0x005BF, 0x005BF, WBP::Extend}, + {0x005C1, 0x005C2, WBP::Extend}, + {0x005C4, 0x005C5, WBP::Extend}, + {0x005C7, 0x005C7, WBP::Extend}, + {0x005D0, 0x005EA, WBP::Hebrew_Letter}, + {0x005EF, 0x005F2, WBP::Hebrew_Letter}, + {0x005F3, 0x005F3, WBP::ALetter}, + {0x005F4, 0x005F4, WBP::MidLetter}, + {0x00600, 0x00605, WBP::Format}, + {0x0060C, 0x0060D, WBP::MidNum}, + {0x00610, 0x0061A, WBP::Extend}, + {0x0061C, 0x0061C, WBP::Format}, + {0x00620, 0x0064A, WBP::ALetter}, + {0x0064B, 0x0065F, WBP::Extend}, + {0x00660, 0x00669, WBP::Numeric}, + {0x0066B, 0x0066B, WBP::Numeric}, + {0x0066C, 0x0066C, WBP::MidNum}, + {0x0066E, 0x0066F, WBP::ALetter}, + {0x00670, 0x00670, WBP::Extend}, + {0x00671, 0x006D3, WBP::ALetter}, + {0x006D5, 0x006D5, WBP::ALetter}, + {0x006D6, 0x006DC, WBP::Extend}, + {0x006DD, 0x006DD, WBP::Format}, + {0x006DF, 0x006E4, WBP::Extend}, + {0x006E5, 0x006E6, WBP::ALetter}, + {0x006E7, 0x006E8, WBP::Extend}, + {0x006EA, 0x006ED, WBP::Extend}, + {0x006EE, 0x006EF, WBP::ALetter}, + {0x006F0, 0x006F9, WBP::Numeric}, + {0x006FA, 0x006FC, WBP::ALetter}, + {0x006FF, 0x006FF, WBP::ALetter}, + {0x0070F, 0x0070F, WBP::Format}, + {0x00710, 0x00710, WBP::ALetter}, + {0x00711, 0x00711, WBP::Extend}, + {0x00712, 0x0072F, WBP::ALetter}, + {0x00730, 0x0074A, WBP::Extend}, + {0x0074D, 0x007A5, WBP::ALetter}, + {0x007A6, 0x007B0, WBP::Extend}, + {0x007B1, 0x007B1, WBP::ALetter}, + {0x007C0, 0x007C9, WBP::Numeric}, + {0x007CA, 0x007EA, WBP::ALetter}, + {0x007EB, 0x007F3, WBP::Extend}, + {0x007F4, 0x007F5, WBP::ALetter}, + {0x007F8, 0x007F8, WBP::MidNum}, + {0x007FA, 0x007FA, WBP::ALetter}, + {0x007FD, 0x007FD, WBP::Extend}, + {0x00800, 0x00815, WBP::ALetter}, + {0x00816, 0x00819, WBP::Extend}, + {0x0081A, 0x0081A, WBP::ALetter}, + {0x0081B, 0x00823, WBP::Extend}, + {0x00824, 0x00824, WBP::ALetter}, + {0x00825, 0x00827, WBP::Extend}, + {0x00828, 0x00828, WBP::ALetter}, + {0x00829, 0x0082D, WBP::Extend}, + {0x00840, 0x00858, WBP::ALetter}, + {0x00859, 0x0085B, WBP::Extend}, + {0x00860, 0x0086A, WBP::ALetter}, + {0x008A0, 0x008B4, WBP::ALetter}, + {0x008B6, 0x008C7, WBP::ALetter}, + {0x008D3, 0x008E1, WBP::Extend}, + {0x008E2, 0x008E2, WBP::Format}, + {0x008E3, 0x00903, WBP::Extend}, + {0x00904, 0x00939, WBP::ALetter}, + {0x0093A, 0x0093C, WBP::Extend}, + {0x0093D, 0x0093D, WBP::ALetter}, + {0x0093E, 0x0094F, WBP::Extend}, + {0x00950, 0x00950, WBP::ALetter}, + {0x00951, 0x00957, WBP::Extend}, + {0x00958, 0x00961, WBP::ALetter}, + {0x00962, 0x00963, WBP::Extend}, + {0x00966, 0x0096F, WBP::Numeric}, + {0x00971, 0x00980, WBP::ALetter}, + {0x00981, 0x00983, WBP::Extend}, + {0x00985, 0x0098C, WBP::ALetter}, + {0x0098F, 0x00990, WBP::ALetter}, + {0x00993, 0x009A8, WBP::ALetter}, + {0x009AA, 0x009B0, WBP::ALetter}, + {0x009B2, 0x009B2, WBP::ALetter}, + {0x009B6, 0x009B9, WBP::ALetter}, + {0x009BC, 0x009BC, WBP::Extend}, + {0x009BD, 0x009BD, WBP::ALetter}, + {0x009BE, 0x009C4, WBP::Extend}, + {0x009C7, 0x009C8, WBP::Extend}, + {0x009CB, 0x009CD, WBP::Extend}, + {0x009CE, 0x009CE, WBP::ALetter}, + {0x009D7, 0x009D7, WBP::Extend}, + {0x009DC, 0x009DD, WBP::ALetter}, + {0x009DF, 0x009E1, WBP::ALetter}, + {0x009E2, 0x009E3, WBP::Extend}, + {0x009E6, 0x009EF, WBP::Numeric}, + {0x009F0, 0x009F1, WBP::ALetter}, + {0x009FC, 0x009FC, WBP::ALetter}, + {0x009FE, 0x009FE, WBP::Extend}, + {0x00A01, 0x00A03, WBP::Extend}, + {0x00A05, 0x00A0A, WBP::ALetter}, + {0x00A0F, 0x00A10, WBP::ALetter}, + {0x00A13, 0x00A28, WBP::ALetter}, + {0x00A2A, 0x00A30, WBP::ALetter}, + {0x00A32, 0x00A33, WBP::ALetter}, + {0x00A35, 0x00A36, WBP::ALetter}, + {0x00A38, 0x00A39, WBP::ALetter}, + {0x00A3C, 0x00A3C, WBP::Extend}, + {0x00A3E, 0x00A42, WBP::Extend}, + {0x00A47, 0x00A48, WBP::Extend}, + {0x00A4B, 0x00A4D, WBP::Extend}, + {0x00A51, 0x00A51, WBP::Extend}, + {0x00A59, 0x00A5C, WBP::ALetter}, + {0x00A5E, 0x00A5E, WBP::ALetter}, + {0x00A66, 0x00A6F, WBP::Numeric}, + {0x00A70, 0x00A71, WBP::Extend}, + {0x00A72, 0x00A74, WBP::ALetter}, + {0x00A75, 0x00A75, WBP::Extend}, + {0x00A81, 0x00A83, WBP::Extend}, + {0x00A85, 0x00A8D, WBP::ALetter}, + {0x00A8F, 0x00A91, WBP::ALetter}, + {0x00A93, 0x00AA8, WBP::ALetter}, + {0x00AAA, 0x00AB0, WBP::ALetter}, + {0x00AB2, 0x00AB3, WBP::ALetter}, + {0x00AB5, 0x00AB9, WBP::ALetter}, + {0x00ABC, 0x00ABC, WBP::Extend}, + {0x00ABD, 0x00ABD, WBP::ALetter}, + {0x00ABE, 0x00AC5, WBP::Extend}, + {0x00AC7, 0x00AC9, WBP::Extend}, + {0x00ACB, 0x00ACD, WBP::Extend}, + {0x00AD0, 0x00AD0, WBP::ALetter}, + {0x00AE0, 0x00AE1, WBP::ALetter}, + {0x00AE2, 0x00AE3, WBP::Extend}, + {0x00AE6, 0x00AEF, WBP::Numeric}, + {0x00AF9, 0x00AF9, WBP::ALetter}, + {0x00AFA, 0x00AFF, WBP::Extend}, + {0x00B01, 0x00B03, WBP::Extend}, + {0x00B05, 0x00B0C, WBP::ALetter}, + {0x00B0F, 0x00B10, WBP::ALetter}, + {0x00B13, 0x00B28, WBP::ALetter}, + {0x00B2A, 0x00B30, WBP::ALetter}, + {0x00B32, 0x00B33, WBP::ALetter}, + {0x00B35, 0x00B39, WBP::ALetter}, + {0x00B3C, 0x00B3C, WBP::Extend}, + {0x00B3D, 0x00B3D, WBP::ALetter}, + {0x00B3E, 0x00B44, WBP::Extend}, + {0x00B47, 0x00B48, WBP::Extend}, + {0x00B4B, 0x00B4D, WBP::Extend}, + {0x00B55, 0x00B57, WBP::Extend}, + {0x00B5C, 0x00B5D, WBP::ALetter}, + {0x00B5F, 0x00B61, WBP::ALetter}, + {0x00B62, 0x00B63, WBP::Extend}, + {0x00B66, 0x00B6F, WBP::Numeric}, + {0x00B71, 0x00B71, WBP::ALetter}, + {0x00B82, 0x00B82, WBP::Extend}, + {0x00B83, 0x00B83, WBP::ALetter}, + {0x00B85, 0x00B8A, WBP::ALetter}, + {0x00B8E, 0x00B90, WBP::ALetter}, + {0x00B92, 0x00B95, WBP::ALetter}, + {0x00B99, 0x00B9A, WBP::ALetter}, + {0x00B9C, 0x00B9C, WBP::ALetter}, + {0x00B9E, 0x00B9F, WBP::ALetter}, + {0x00BA3, 0x00BA4, WBP::ALetter}, + {0x00BA8, 0x00BAA, WBP::ALetter}, + {0x00BAE, 0x00BB9, WBP::ALetter}, + {0x00BBE, 0x00BC2, WBP::Extend}, + {0x00BC6, 0x00BC8, WBP::Extend}, + {0x00BCA, 0x00BCD, WBP::Extend}, + {0x00BD0, 0x00BD0, WBP::ALetter}, + {0x00BD7, 0x00BD7, WBP::Extend}, + {0x00BE6, 0x00BEF, WBP::Numeric}, + {0x00C00, 0x00C04, WBP::Extend}, + {0x00C05, 0x00C0C, WBP::ALetter}, + {0x00C0E, 0x00C10, WBP::ALetter}, + {0x00C12, 0x00C28, WBP::ALetter}, + {0x00C2A, 0x00C39, WBP::ALetter}, + {0x00C3D, 0x00C3D, WBP::ALetter}, + {0x00C3E, 0x00C44, WBP::Extend}, + {0x00C46, 0x00C48, WBP::Extend}, + {0x00C4A, 0x00C4D, WBP::Extend}, + {0x00C55, 0x00C56, WBP::Extend}, + {0x00C58, 0x00C5A, WBP::ALetter}, + {0x00C60, 0x00C61, WBP::ALetter}, + {0x00C62, 0x00C63, WBP::Extend}, + {0x00C66, 0x00C6F, WBP::Numeric}, + {0x00C80, 0x00C80, WBP::ALetter}, + {0x00C81, 0x00C83, WBP::Extend}, + {0x00C85, 0x00C8C, WBP::ALetter}, + {0x00C8E, 0x00C90, WBP::ALetter}, + {0x00C92, 0x00CA8, WBP::ALetter}, + {0x00CAA, 0x00CB3, WBP::ALetter}, + {0x00CB5, 0x00CB9, WBP::ALetter}, + {0x00CBC, 0x00CBC, WBP::Extend}, + {0x00CBD, 0x00CBD, WBP::ALetter}, + {0x00CBE, 0x00CC4, WBP::Extend}, + {0x00CC6, 0x00CC8, WBP::Extend}, + {0x00CCA, 0x00CCD, WBP::Extend}, + {0x00CD5, 0x00CD6, WBP::Extend}, + {0x00CDE, 0x00CDE, WBP::ALetter}, + {0x00CE0, 0x00CE1, WBP::ALetter}, + {0x00CE2, 0x00CE3, WBP::Extend}, + {0x00CE6, 0x00CEF, WBP::Numeric}, + {0x00CF1, 0x00CF2, WBP::ALetter}, + {0x00D00, 0x00D03, WBP::Extend}, + {0x00D04, 0x00D0C, WBP::ALetter}, + {0x00D0E, 0x00D10, WBP::ALetter}, + {0x00D12, 0x00D3A, WBP::ALetter}, + {0x00D3B, 0x00D3C, WBP::Extend}, + {0x00D3D, 0x00D3D, WBP::ALetter}, + {0x00D3E, 0x00D44, WBP::Extend}, + {0x00D46, 0x00D48, WBP::Extend}, + {0x00D4A, 0x00D4D, WBP::Extend}, + {0x00D4E, 0x00D4E, WBP::ALetter}, + {0x00D54, 0x00D56, WBP::ALetter}, + {0x00D57, 0x00D57, WBP::Extend}, + {0x00D5F, 0x00D61, WBP::ALetter}, + {0x00D62, 0x00D63, WBP::Extend}, + {0x00D66, 0x00D6F, WBP::Numeric}, + {0x00D7A, 0x00D7F, WBP::ALetter}, + {0x00D81, 0x00D83, WBP::Extend}, + {0x00D85, 0x00D96, WBP::ALetter}, + {0x00D9A, 0x00DB1, WBP::ALetter}, + {0x00DB3, 0x00DBB, WBP::ALetter}, + {0x00DBD, 0x00DBD, WBP::ALetter}, + {0x00DC0, 0x00DC6, WBP::ALetter}, + {0x00DCA, 0x00DCA, WBP::Extend}, + {0x00DCF, 0x00DD4, WBP::Extend}, + {0x00DD6, 0x00DD6, WBP::Extend}, + {0x00DD8, 0x00DDF, WBP::Extend}, + {0x00DE6, 0x00DEF, WBP::Numeric}, + {0x00DF2, 0x00DF3, WBP::Extend}, + {0x00E31, 0x00E31, WBP::Extend}, + {0x00E34, 0x00E3A, WBP::Extend}, + {0x00E47, 0x00E4E, WBP::Extend}, + {0x00E50, 0x00E59, WBP::Numeric}, + {0x00EB1, 0x00EB1, WBP::Extend}, + {0x00EB4, 0x00EBC, WBP::Extend}, + {0x00EC8, 0x00ECD, WBP::Extend}, + {0x00ED0, 0x00ED9, WBP::Numeric}, + {0x00F00, 0x00F00, WBP::ALetter}, + {0x00F18, 0x00F19, WBP::Extend}, + {0x00F20, 0x00F29, WBP::Numeric}, + {0x00F35, 0x00F35, WBP::Extend}, + {0x00F37, 0x00F37, WBP::Extend}, + {0x00F39, 0x00F39, WBP::Extend}, + {0x00F3E, 0x00F3F, WBP::Extend}, + {0x00F40, 0x00F47, WBP::ALetter}, + {0x00F49, 0x00F6C, WBP::ALetter}, + {0x00F71, 0x00F84, WBP::Extend}, + {0x00F86, 0x00F87, WBP::Extend}, + {0x00F88, 0x00F8C, WBP::ALetter}, + {0x00F8D, 0x00F97, WBP::Extend}, + {0x00F99, 0x00FBC, WBP::Extend}, + {0x00FC6, 0x00FC6, WBP::Extend}, + {0x0102B, 0x0103E, WBP::Extend}, + {0x01040, 0x01049, WBP::Numeric}, + {0x01056, 0x01059, WBP::Extend}, + {0x0105E, 0x01060, WBP::Extend}, + {0x01062, 0x01064, WBP::Extend}, + {0x01067, 0x0106D, WBP::Extend}, + {0x01071, 0x01074, WBP::Extend}, + {0x01082, 0x0108D, WBP::Extend}, + {0x0108F, 0x0108F, WBP::Extend}, + {0x01090, 0x01099, WBP::Numeric}, + {0x0109A, 0x0109D, WBP::Extend}, + {0x010A0, 0x010C5, WBP::ALetter}, + {0x010C7, 0x010C7, WBP::ALetter}, + {0x010CD, 0x010CD, WBP::ALetter}, + {0x010D0, 0x010FA, WBP::ALetter}, + {0x010FC, 0x01248, WBP::ALetter}, + {0x0124A, 0x0124D, WBP::ALetter}, + {0x01250, 0x01256, WBP::ALetter}, + {0x01258, 0x01258, WBP::ALetter}, + {0x0125A, 0x0125D, WBP::ALetter}, + {0x01260, 0x01288, WBP::ALetter}, + {0x0128A, 0x0128D, WBP::ALetter}, + {0x01290, 0x012B0, WBP::ALetter}, + {0x012B2, 0x012B5, WBP::ALetter}, + {0x012B8, 0x012BE, WBP::ALetter}, + {0x012C0, 0x012C0, WBP::ALetter}, + {0x012C2, 0x012C5, WBP::ALetter}, + {0x012C8, 0x012D6, WBP::ALetter}, + {0x012D8, 0x01310, WBP::ALetter}, + {0x01312, 0x01315, WBP::ALetter}, + {0x01318, 0x0135A, WBP::ALetter}, + {0x0135D, 0x0135F, WBP::Extend}, + {0x01380, 0x0138F, WBP::ALetter}, + {0x013A0, 0x013F5, WBP::ALetter}, + {0x013F8, 0x013FD, WBP::ALetter}, + {0x01401, 0x0166C, WBP::ALetter}, + {0x0166F, 0x0167F, WBP::ALetter}, + {0x01680, 0x01680, WBP::WSegSpace}, + {0x01681, 0x0169A, WBP::ALetter}, + {0x016A0, 0x016EA, WBP::ALetter}, + {0x016EE, 0x016F8, WBP::ALetter}, + {0x01700, 0x0170C, WBP::ALetter}, + {0x0170E, 0x01711, WBP::ALetter}, + {0x01712, 0x01714, WBP::Extend}, + {0x01720, 0x01731, WBP::ALetter}, + {0x01732, 0x01734, WBP::Extend}, + {0x01740, 0x01751, WBP::ALetter}, + {0x01752, 0x01753, WBP::Extend}, + {0x01760, 0x0176C, WBP::ALetter}, + {0x0176E, 0x01770, WBP::ALetter}, + {0x01772, 0x01773, WBP::Extend}, + {0x017B4, 0x017D3, WBP::Extend}, + {0x017DD, 0x017DD, WBP::Extend}, + {0x017E0, 0x017E9, WBP::Numeric}, + {0x0180B, 0x0180D, WBP::Extend}, + {0x0180E, 0x0180E, WBP::Format}, + {0x01810, 0x01819, WBP::Numeric}, + {0x01820, 0x01878, WBP::ALetter}, + {0x01880, 0x01884, WBP::ALetter}, + {0x01885, 0x01886, WBP::Extend}, + {0x01887, 0x018A8, WBP::ALetter}, + {0x018A9, 0x018A9, WBP::Extend}, + {0x018AA, 0x018AA, WBP::ALetter}, + {0x018B0, 0x018F5, WBP::ALetter}, + {0x01900, 0x0191E, WBP::ALetter}, + {0x01920, 0x0192B, WBP::Extend}, + {0x01930, 0x0193B, WBP::Extend}, + {0x01946, 0x0194F, WBP::Numeric}, + {0x019D0, 0x019D9, WBP::Numeric}, + {0x01A00, 0x01A16, WBP::ALetter}, + {0x01A17, 0x01A1B, WBP::Extend}, + {0x01A55, 0x01A5E, WBP::Extend}, + {0x01A60, 0x01A7C, WBP::Extend}, + {0x01A7F, 0x01A7F, WBP::Extend}, + {0x01A80, 0x01A89, WBP::Numeric}, + {0x01A90, 0x01A99, WBP::Numeric}, + {0x01AB0, 0x01AC0, WBP::Extend}, + {0x01B00, 0x01B04, WBP::Extend}, + {0x01B05, 0x01B33, WBP::ALetter}, + {0x01B34, 0x01B44, WBP::Extend}, + {0x01B45, 0x01B4B, WBP::ALetter}, + {0x01B50, 0x01B59, WBP::Numeric}, + {0x01B6B, 0x01B73, WBP::Extend}, + {0x01B80, 0x01B82, WBP::Extend}, + {0x01B83, 0x01BA0, WBP::ALetter}, + {0x01BA1, 0x01BAD, WBP::Extend}, + {0x01BAE, 0x01BAF, WBP::ALetter}, + {0x01BB0, 0x01BB9, WBP::Numeric}, + {0x01BBA, 0x01BE5, WBP::ALetter}, + {0x01BE6, 0x01BF3, WBP::Extend}, + {0x01C00, 0x01C23, WBP::ALetter}, + {0x01C24, 0x01C37, WBP::Extend}, + {0x01C40, 0x01C49, WBP::Numeric}, + {0x01C4D, 0x01C4F, WBP::ALetter}, + {0x01C50, 0x01C59, WBP::Numeric}, + {0x01C5A, 0x01C7D, WBP::ALetter}, + {0x01C80, 0x01C88, WBP::ALetter}, + {0x01C90, 0x01CBA, WBP::ALetter}, + {0x01CBD, 0x01CBF, WBP::ALetter}, + {0x01CD0, 0x01CD2, WBP::Extend}, + {0x01CD4, 0x01CE8, WBP::Extend}, + {0x01CE9, 0x01CEC, WBP::ALetter}, + {0x01CED, 0x01CED, WBP::Extend}, + {0x01CEE, 0x01CF3, WBP::ALetter}, + {0x01CF4, 0x01CF4, WBP::Extend}, + {0x01CF5, 0x01CF6, WBP::ALetter}, + {0x01CF7, 0x01CF9, WBP::Extend}, + {0x01CFA, 0x01CFA, WBP::ALetter}, + {0x01D00, 0x01DBF, WBP::ALetter}, + {0x01DC0, 0x01DF9, WBP::Extend}, + {0x01DFB, 0x01DFF, WBP::Extend}, + {0x01E00, 0x01F15, WBP::ALetter}, + {0x01F18, 0x01F1D, WBP::ALetter}, + {0x01F20, 0x01F45, WBP::ALetter}, + {0x01F48, 0x01F4D, WBP::ALetter}, + {0x01F50, 0x01F57, WBP::ALetter}, + {0x01F59, 0x01F59, WBP::ALetter}, + {0x01F5B, 0x01F5B, WBP::ALetter}, + {0x01F5D, 0x01F5D, WBP::ALetter}, + {0x01F5F, 0x01F7D, WBP::ALetter}, + {0x01F80, 0x01FB4, WBP::ALetter}, + {0x01FB6, 0x01FBC, WBP::ALetter}, + {0x01FBE, 0x01FBE, WBP::ALetter}, + {0x01FC2, 0x01FC4, WBP::ALetter}, + {0x01FC6, 0x01FCC, WBP::ALetter}, + {0x01FD0, 0x01FD3, WBP::ALetter}, + {0x01FD6, 0x01FDB, WBP::ALetter}, + {0x01FE0, 0x01FEC, WBP::ALetter}, + {0x01FF2, 0x01FF4, WBP::ALetter}, + {0x01FF6, 0x01FFC, WBP::ALetter}, + {0x02000, 0x02006, WBP::WSegSpace}, + {0x02008, 0x0200A, WBP::WSegSpace}, + {0x0200C, 0x0200C, WBP::Extend}, + {0x0200D, 0x0200D, WBP::ZWJ}, + {0x0200E, 0x0200F, WBP::Format}, + {0x02018, 0x02019, WBP::MidNumLet}, + {0x02024, 0x02024, WBP::MidNumLet}, + {0x02027, 0x02027, WBP::MidLetter}, + {0x02028, 0x02029, WBP::Newline}, + {0x0202A, 0x0202E, WBP::Format}, + {0x0202F, 0x0202F, WBP::ExtendNumLet}, + {0x0203F, 0x02040, WBP::ExtendNumLet}, + {0x02044, 0x02044, WBP::MidNum}, + {0x02054, 0x02054, WBP::ExtendNumLet}, + {0x0205F, 0x0205F, WBP::WSegSpace}, + {0x02060, 0x02064, WBP::Format}, + {0x02066, 0x0206F, WBP::Format}, + {0x02071, 0x02071, WBP::ALetter}, + {0x0207F, 0x0207F, WBP::ALetter}, + {0x02090, 0x0209C, WBP::ALetter}, + {0x020D0, 0x020F0, WBP::Extend}, + {0x02102, 0x02102, WBP::ALetter}, + {0x02107, 0x02107, WBP::ALetter}, + {0x0210A, 0x02113, WBP::ALetter}, + {0x02115, 0x02115, WBP::ALetter}, + {0x02119, 0x0211D, WBP::ALetter}, + {0x02124, 0x02124, WBP::ALetter}, + {0x02126, 0x02126, WBP::ALetter}, + {0x02128, 0x02128, WBP::ALetter}, + {0x0212A, 0x0212D, WBP::ALetter}, + {0x0212F, 0x02139, WBP::ALetter}, + {0x0213C, 0x0213F, WBP::ALetter}, + {0x02145, 0x02149, WBP::ALetter}, + {0x0214E, 0x0214E, WBP::ALetter}, + {0x02160, 0x02188, WBP::ALetter}, + {0x024B6, 0x024E9, WBP::ALetter}, + {0x02C00, 0x02C2E, WBP::ALetter}, + {0x02C30, 0x02C5E, WBP::ALetter}, + {0x02C60, 0x02CE4, WBP::ALetter}, + {0x02CEB, 0x02CEE, WBP::ALetter}, + {0x02CEF, 0x02CF1, WBP::Extend}, + {0x02CF2, 0x02CF3, WBP::ALetter}, + {0x02D00, 0x02D25, WBP::ALetter}, + {0x02D27, 0x02D27, WBP::ALetter}, + {0x02D2D, 0x02D2D, WBP::ALetter}, + {0x02D30, 0x02D67, WBP::ALetter}, + {0x02D6F, 0x02D6F, WBP::ALetter}, + {0x02D7F, 0x02D7F, WBP::Extend}, + {0x02D80, 0x02D96, WBP::ALetter}, + {0x02DA0, 0x02DA6, WBP::ALetter}, + {0x02DA8, 0x02DAE, WBP::ALetter}, + {0x02DB0, 0x02DB6, WBP::ALetter}, + {0x02DB8, 0x02DBE, WBP::ALetter}, + {0x02DC0, 0x02DC6, WBP::ALetter}, + {0x02DC8, 0x02DCE, WBP::ALetter}, + {0x02DD0, 0x02DD6, WBP::ALetter}, + {0x02DD8, 0x02DDE, WBP::ALetter}, + {0x02DE0, 0x02DFF, WBP::Extend}, + {0x02E2F, 0x02E2F, WBP::ALetter}, + {0x03000, 0x03000, WBP::WSegSpace}, + {0x03005, 0x03005, WBP::ALetter}, + {0x0302A, 0x0302F, WBP::Extend}, + {0x03031, 0x03035, WBP::Katakana}, + {0x0303B, 0x0303C, WBP::ALetter}, + {0x03099, 0x0309A, WBP::Extend}, + {0x0309B, 0x0309C, WBP::Katakana}, + {0x030A0, 0x030FA, WBP::Katakana}, + {0x030FC, 0x030FF, WBP::Katakana}, + {0x03105, 0x0312F, WBP::ALetter}, + {0x03131, 0x0318E, WBP::ALetter}, + {0x031A0, 0x031BF, WBP::ALetter}, + {0x031F0, 0x031FF, WBP::Katakana}, + {0x032D0, 0x032FE, WBP::Katakana}, + {0x03300, 0x03357, WBP::Katakana}, + {0x0A000, 0x0A48C, WBP::ALetter}, + {0x0A4D0, 0x0A4FD, WBP::ALetter}, + {0x0A500, 0x0A60C, WBP::ALetter}, + {0x0A610, 0x0A61F, WBP::ALetter}, + {0x0A620, 0x0A629, WBP::Numeric}, + {0x0A62A, 0x0A62B, WBP::ALetter}, + {0x0A640, 0x0A66E, WBP::ALetter}, + {0x0A66F, 0x0A672, WBP::Extend}, + {0x0A674, 0x0A67D, WBP::Extend}, + {0x0A67F, 0x0A69D, WBP::ALetter}, + {0x0A69E, 0x0A69F, WBP::Extend}, + {0x0A6A0, 0x0A6EF, WBP::ALetter}, + {0x0A6F0, 0x0A6F1, WBP::Extend}, + {0x0A708, 0x0A7BF, WBP::ALetter}, + {0x0A7C2, 0x0A7CA, WBP::ALetter}, + {0x0A7F5, 0x0A801, WBP::ALetter}, + {0x0A802, 0x0A802, WBP::Extend}, + {0x0A803, 0x0A805, WBP::ALetter}, + {0x0A806, 0x0A806, WBP::Extend}, + {0x0A807, 0x0A80A, WBP::ALetter}, + {0x0A80B, 0x0A80B, WBP::Extend}, + {0x0A80C, 0x0A822, WBP::ALetter}, + {0x0A823, 0x0A827, WBP::Extend}, + {0x0A82C, 0x0A82C, WBP::Extend}, + {0x0A840, 0x0A873, WBP::ALetter}, + {0x0A880, 0x0A881, WBP::Extend}, + {0x0A882, 0x0A8B3, WBP::ALetter}, + {0x0A8B4, 0x0A8C5, WBP::Extend}, + {0x0A8D0, 0x0A8D9, WBP::Numeric}, + {0x0A8E0, 0x0A8F1, WBP::Extend}, + {0x0A8F2, 0x0A8F7, WBP::ALetter}, + {0x0A8FB, 0x0A8FB, WBP::ALetter}, + {0x0A8FD, 0x0A8FE, WBP::ALetter}, + {0x0A8FF, 0x0A8FF, WBP::Extend}, + {0x0A900, 0x0A909, WBP::Numeric}, + {0x0A90A, 0x0A925, WBP::ALetter}, + {0x0A926, 0x0A92D, WBP::Extend}, + {0x0A930, 0x0A946, WBP::ALetter}, + {0x0A947, 0x0A953, WBP::Extend}, + {0x0A960, 0x0A97C, WBP::ALetter}, + {0x0A980, 0x0A983, WBP::Extend}, + {0x0A984, 0x0A9B2, WBP::ALetter}, + {0x0A9B3, 0x0A9C0, WBP::Extend}, + {0x0A9CF, 0x0A9CF, WBP::ALetter}, + {0x0A9D0, 0x0A9D9, WBP::Numeric}, + {0x0A9E5, 0x0A9E5, WBP::Extend}, + {0x0A9F0, 0x0A9F9, WBP::Numeric}, + {0x0AA00, 0x0AA28, WBP::ALetter}, + {0x0AA29, 0x0AA36, WBP::Extend}, + {0x0AA40, 0x0AA42, WBP::ALetter}, + {0x0AA43, 0x0AA43, WBP::Extend}, + {0x0AA44, 0x0AA4B, WBP::ALetter}, + {0x0AA4C, 0x0AA4D, WBP::Extend}, + {0x0AA50, 0x0AA59, WBP::Numeric}, + {0x0AA7B, 0x0AA7D, WBP::Extend}, + {0x0AAB0, 0x0AAB0, WBP::Extend}, + {0x0AAB2, 0x0AAB4, WBP::Extend}, + {0x0AAB7, 0x0AAB8, WBP::Extend}, + {0x0AABE, 0x0AABF, WBP::Extend}, + {0x0AAC1, 0x0AAC1, WBP::Extend}, + {0x0AAE0, 0x0AAEA, WBP::ALetter}, + {0x0AAEB, 0x0AAEF, WBP::Extend}, + {0x0AAF2, 0x0AAF4, WBP::ALetter}, + {0x0AAF5, 0x0AAF6, WBP::Extend}, + {0x0AB01, 0x0AB06, WBP::ALetter}, + {0x0AB09, 0x0AB0E, WBP::ALetter}, + {0x0AB11, 0x0AB16, WBP::ALetter}, + {0x0AB20, 0x0AB26, WBP::ALetter}, + {0x0AB28, 0x0AB2E, WBP::ALetter}, + {0x0AB30, 0x0AB69, WBP::ALetter}, + {0x0AB70, 0x0ABE2, WBP::ALetter}, + {0x0ABE3, 0x0ABEA, WBP::Extend}, + {0x0ABEC, 0x0ABED, WBP::Extend}, + {0x0ABF0, 0x0ABF9, WBP::Numeric}, + {0x0AC00, 0x0D7A3, WBP::ALetter}, + {0x0D7B0, 0x0D7C6, WBP::ALetter}, + {0x0D7CB, 0x0D7FB, WBP::ALetter}, + {0x0FB00, 0x0FB06, WBP::ALetter}, + {0x0FB13, 0x0FB17, WBP::ALetter}, + {0x0FB1D, 0x0FB1D, WBP::Hebrew_Letter}, + {0x0FB1E, 0x0FB1E, WBP::Extend}, + {0x0FB1F, 0x0FB28, WBP::Hebrew_Letter}, + {0x0FB2A, 0x0FB36, WBP::Hebrew_Letter}, + {0x0FB38, 0x0FB3C, WBP::Hebrew_Letter}, + {0x0FB3E, 0x0FB3E, WBP::Hebrew_Letter}, + {0x0FB40, 0x0FB41, WBP::Hebrew_Letter}, + {0x0FB43, 0x0FB44, WBP::Hebrew_Letter}, + {0x0FB46, 0x0FB4F, WBP::Hebrew_Letter}, + {0x0FB50, 0x0FBB1, WBP::ALetter}, + {0x0FBD3, 0x0FD3D, WBP::ALetter}, + {0x0FD50, 0x0FD8F, WBP::ALetter}, + {0x0FD92, 0x0FDC7, WBP::ALetter}, + {0x0FDF0, 0x0FDFB, WBP::ALetter}, + {0x0FE00, 0x0FE0F, WBP::Extend}, + {0x0FE10, 0x0FE10, WBP::MidNum}, + {0x0FE13, 0x0FE13, WBP::MidLetter}, + {0x0FE14, 0x0FE14, WBP::MidNum}, + {0x0FE20, 0x0FE2F, WBP::Extend}, + {0x0FE33, 0x0FE34, WBP::ExtendNumLet}, + {0x0FE4D, 0x0FE4F, WBP::ExtendNumLet}, + {0x0FE50, 0x0FE50, WBP::MidNum}, + {0x0FE52, 0x0FE52, WBP::MidNumLet}, + {0x0FE54, 0x0FE54, WBP::MidNum}, + {0x0FE55, 0x0FE55, WBP::MidLetter}, + {0x0FE70, 0x0FE74, WBP::ALetter}, + {0x0FE76, 0x0FEFC, WBP::ALetter}, + {0x0FEFF, 0x0FEFF, WBP::Format}, + {0x0FF07, 0x0FF07, WBP::MidNumLet}, + {0x0FF0C, 0x0FF0C, WBP::MidNum}, + {0x0FF0E, 0x0FF0E, WBP::MidNumLet}, + {0x0FF10, 0x0FF19, WBP::Numeric}, + {0x0FF1A, 0x0FF1A, WBP::MidLetter}, + {0x0FF1B, 0x0FF1B, WBP::MidNum}, + {0x0FF21, 0x0FF3A, WBP::ALetter}, + {0x0FF3F, 0x0FF3F, WBP::ExtendNumLet}, + {0x0FF41, 0x0FF5A, WBP::ALetter}, + {0x0FF66, 0x0FF9D, WBP::Katakana}, + {0x0FF9E, 0x0FF9F, WBP::Extend}, + {0x0FFA0, 0x0FFBE, WBP::ALetter}, + {0x0FFC2, 0x0FFC7, WBP::ALetter}, + {0x0FFCA, 0x0FFCF, WBP::ALetter}, + {0x0FFD2, 0x0FFD7, WBP::ALetter}, + {0x0FFDA, 0x0FFDC, WBP::ALetter}, + {0x0FFF9, 0x0FFFB, WBP::Format}, + {0x10000, 0x1000B, WBP::ALetter}, + {0x1000D, 0x10026, WBP::ALetter}, + {0x10028, 0x1003A, WBP::ALetter}, + {0x1003C, 0x1003D, WBP::ALetter}, + {0x1003F, 0x1004D, WBP::ALetter}, + {0x10050, 0x1005D, WBP::ALetter}, + {0x10080, 0x100FA, WBP::ALetter}, + {0x10140, 0x10174, WBP::ALetter}, + {0x101FD, 0x101FD, WBP::Extend}, + {0x10280, 0x1029C, WBP::ALetter}, + {0x102A0, 0x102D0, WBP::ALetter}, + {0x102E0, 0x102E0, WBP::Extend}, + {0x10300, 0x1031F, WBP::ALetter}, + {0x1032D, 0x1034A, WBP::ALetter}, + {0x10350, 0x10375, WBP::ALetter}, + {0x10376, 0x1037A, WBP::Extend}, + {0x10380, 0x1039D, WBP::ALetter}, + {0x103A0, 0x103C3, WBP::ALetter}, + {0x103C8, 0x103CF, WBP::ALetter}, + {0x103D1, 0x103D5, WBP::ALetter}, + {0x10400, 0x1049D, WBP::ALetter}, + {0x104A0, 0x104A9, WBP::Numeric}, + {0x104B0, 0x104D3, WBP::ALetter}, + {0x104D8, 0x104FB, WBP::ALetter}, + {0x10500, 0x10527, WBP::ALetter}, + {0x10530, 0x10563, WBP::ALetter}, + {0x10600, 0x10736, WBP::ALetter}, + {0x10740, 0x10755, WBP::ALetter}, + {0x10760, 0x10767, WBP::ALetter}, + {0x10800, 0x10805, WBP::ALetter}, + {0x10808, 0x10808, WBP::ALetter}, + {0x1080A, 0x10835, WBP::ALetter}, + {0x10837, 0x10838, WBP::ALetter}, + {0x1083C, 0x1083C, WBP::ALetter}, + {0x1083F, 0x10855, WBP::ALetter}, + {0x10860, 0x10876, WBP::ALetter}, + {0x10880, 0x1089E, WBP::ALetter}, + {0x108E0, 0x108F2, WBP::ALetter}, + {0x108F4, 0x108F5, WBP::ALetter}, + {0x10900, 0x10915, WBP::ALetter}, + {0x10920, 0x10939, WBP::ALetter}, + {0x10980, 0x109B7, WBP::ALetter}, + {0x109BE, 0x109BF, WBP::ALetter}, + {0x10A00, 0x10A00, WBP::ALetter}, + {0x10A01, 0x10A03, WBP::Extend}, + {0x10A05, 0x10A06, WBP::Extend}, + {0x10A0C, 0x10A0F, WBP::Extend}, + {0x10A10, 0x10A13, WBP::ALetter}, + {0x10A15, 0x10A17, WBP::ALetter}, + {0x10A19, 0x10A35, WBP::ALetter}, + {0x10A38, 0x10A3A, WBP::Extend}, + {0x10A3F, 0x10A3F, WBP::Extend}, + {0x10A60, 0x10A7C, WBP::ALetter}, + {0x10A80, 0x10A9C, WBP::ALetter}, + {0x10AC0, 0x10AC7, WBP::ALetter}, + {0x10AC9, 0x10AE4, WBP::ALetter}, + {0x10AE5, 0x10AE6, WBP::Extend}, + {0x10B00, 0x10B35, WBP::ALetter}, + {0x10B40, 0x10B55, WBP::ALetter}, + {0x10B60, 0x10B72, WBP::ALetter}, + {0x10B80, 0x10B91, WBP::ALetter}, + {0x10C00, 0x10C48, WBP::ALetter}, + {0x10C80, 0x10CB2, WBP::ALetter}, + {0x10CC0, 0x10CF2, WBP::ALetter}, + {0x10D00, 0x10D23, WBP::ALetter}, + {0x10D24, 0x10D27, WBP::Extend}, + {0x10D30, 0x10D39, WBP::Numeric}, + {0x10E80, 0x10EA9, WBP::ALetter}, + {0x10EAB, 0x10EAC, WBP::Extend}, + {0x10EB0, 0x10EB1, WBP::ALetter}, + {0x10F00, 0x10F1C, WBP::ALetter}, + {0x10F27, 0x10F27, WBP::ALetter}, + {0x10F30, 0x10F45, WBP::ALetter}, + {0x10F46, 0x10F50, WBP::Extend}, + {0x10FB0, 0x10FC4, WBP::ALetter}, + {0x10FE0, 0x10FF6, WBP::ALetter}, + {0x11000, 0x11002, WBP::Extend}, + {0x11003, 0x11037, WBP::ALetter}, + {0x11038, 0x11046, WBP::Extend}, + {0x11066, 0x1106F, WBP::Numeric}, + {0x1107F, 0x11082, WBP::Extend}, + {0x11083, 0x110AF, WBP::ALetter}, + {0x110B0, 0x110BA, WBP::Extend}, + {0x110BD, 0x110BD, WBP::Format}, + {0x110CD, 0x110CD, WBP::Format}, + {0x110D0, 0x110E8, WBP::ALetter}, + {0x110F0, 0x110F9, WBP::Numeric}, + {0x11100, 0x11102, WBP::Extend}, + {0x11103, 0x11126, WBP::ALetter}, + {0x11127, 0x11134, WBP::Extend}, + {0x11136, 0x1113F, WBP::Numeric}, + {0x11144, 0x11144, WBP::ALetter}, + {0x11145, 0x11146, WBP::Extend}, + {0x11147, 0x11147, WBP::ALetter}, + {0x11150, 0x11172, WBP::ALetter}, + {0x11173, 0x11173, WBP::Extend}, + {0x11176, 0x11176, WBP::ALetter}, + {0x11180, 0x11182, WBP::Extend}, + {0x11183, 0x111B2, WBP::ALetter}, + {0x111B3, 0x111C0, WBP::Extend}, + {0x111C1, 0x111C4, WBP::ALetter}, + {0x111C9, 0x111CC, WBP::Extend}, + {0x111CE, 0x111CF, WBP::Extend}, + {0x111D0, 0x111D9, WBP::Numeric}, + {0x111DA, 0x111DA, WBP::ALetter}, + {0x111DC, 0x111DC, WBP::ALetter}, + {0x11200, 0x11211, WBP::ALetter}, + {0x11213, 0x1122B, WBP::ALetter}, + {0x1122C, 0x11237, WBP::Extend}, + {0x1123E, 0x1123E, WBP::Extend}, + {0x11280, 0x11286, WBP::ALetter}, + {0x11288, 0x11288, WBP::ALetter}, + {0x1128A, 0x1128D, WBP::ALetter}, + {0x1128F, 0x1129D, WBP::ALetter}, + {0x1129F, 0x112A8, WBP::ALetter}, + {0x112B0, 0x112DE, WBP::ALetter}, + {0x112DF, 0x112EA, WBP::Extend}, + {0x112F0, 0x112F9, WBP::Numeric}, + {0x11300, 0x11303, WBP::Extend}, + {0x11305, 0x1130C, WBP::ALetter}, + {0x1130F, 0x11310, WBP::ALetter}, + {0x11313, 0x11328, WBP::ALetter}, + {0x1132A, 0x11330, WBP::ALetter}, + {0x11332, 0x11333, WBP::ALetter}, + {0x11335, 0x11339, WBP::ALetter}, + {0x1133B, 0x1133C, WBP::Extend}, + {0x1133D, 0x1133D, WBP::ALetter}, + {0x1133E, 0x11344, WBP::Extend}, + {0x11347, 0x11348, WBP::Extend}, + {0x1134B, 0x1134D, WBP::Extend}, + {0x11350, 0x11350, WBP::ALetter}, + {0x11357, 0x11357, WBP::Extend}, + {0x1135D, 0x11361, WBP::ALetter}, + {0x11362, 0x11363, WBP::Extend}, + {0x11366, 0x1136C, WBP::Extend}, + {0x11370, 0x11374, WBP::Extend}, + {0x11400, 0x11434, WBP::ALetter}, + {0x11435, 0x11446, WBP::Extend}, + {0x11447, 0x1144A, WBP::ALetter}, + {0x11450, 0x11459, WBP::Numeric}, + {0x1145E, 0x1145E, WBP::Extend}, + {0x1145F, 0x11461, WBP::ALetter}, + {0x11480, 0x114AF, WBP::ALetter}, + {0x114B0, 0x114C3, WBP::Extend}, + {0x114C4, 0x114C5, WBP::ALetter}, + {0x114C7, 0x114C7, WBP::ALetter}, + {0x114D0, 0x114D9, WBP::Numeric}, + {0x11580, 0x115AE, WBP::ALetter}, + {0x115AF, 0x115B5, WBP::Extend}, + {0x115B8, 0x115C0, WBP::Extend}, + {0x115D8, 0x115DB, WBP::ALetter}, + {0x115DC, 0x115DD, WBP::Extend}, + {0x11600, 0x1162F, WBP::ALetter}, + {0x11630, 0x11640, WBP::Extend}, + {0x11644, 0x11644, WBP::ALetter}, + {0x11650, 0x11659, WBP::Numeric}, + {0x11680, 0x116AA, WBP::ALetter}, + {0x116AB, 0x116B7, WBP::Extend}, + {0x116B8, 0x116B8, WBP::ALetter}, + {0x116C0, 0x116C9, WBP::Numeric}, + {0x1171D, 0x1172B, WBP::Extend}, + {0x11730, 0x11739, WBP::Numeric}, + {0x11800, 0x1182B, WBP::ALetter}, + {0x1182C, 0x1183A, WBP::Extend}, + {0x118A0, 0x118DF, WBP::ALetter}, + {0x118E0, 0x118E9, WBP::Numeric}, + {0x118FF, 0x11906, WBP::ALetter}, + {0x11909, 0x11909, WBP::ALetter}, + {0x1190C, 0x11913, WBP::ALetter}, + {0x11915, 0x11916, WBP::ALetter}, + {0x11918, 0x1192F, WBP::ALetter}, + {0x11930, 0x11935, WBP::Extend}, + {0x11937, 0x11938, WBP::Extend}, + {0x1193B, 0x1193E, WBP::Extend}, + {0x1193F, 0x1193F, WBP::ALetter}, + {0x11940, 0x11940, WBP::Extend}, + {0x11941, 0x11941, WBP::ALetter}, + {0x11942, 0x11943, WBP::Extend}, + {0x11950, 0x11959, WBP::Numeric}, + {0x119A0, 0x119A7, WBP::ALetter}, + {0x119AA, 0x119D0, WBP::ALetter}, + {0x119D1, 0x119D7, WBP::Extend}, + {0x119DA, 0x119E0, WBP::Extend}, + {0x119E1, 0x119E1, WBP::ALetter}, + {0x119E3, 0x119E3, WBP::ALetter}, + {0x119E4, 0x119E4, WBP::Extend}, + {0x11A00, 0x11A00, WBP::ALetter}, + {0x11A01, 0x11A0A, WBP::Extend}, + {0x11A0B, 0x11A32, WBP::ALetter}, + {0x11A33, 0x11A39, WBP::Extend}, + {0x11A3A, 0x11A3A, WBP::ALetter}, + {0x11A3B, 0x11A3E, WBP::Extend}, + {0x11A47, 0x11A47, WBP::Extend}, + {0x11A50, 0x11A50, WBP::ALetter}, + {0x11A51, 0x11A5B, WBP::Extend}, + {0x11A5C, 0x11A89, WBP::ALetter}, + {0x11A8A, 0x11A99, WBP::Extend}, + {0x11A9D, 0x11A9D, WBP::ALetter}, + {0x11AC0, 0x11AF8, WBP::ALetter}, + {0x11C00, 0x11C08, WBP::ALetter}, + {0x11C0A, 0x11C2E, WBP::ALetter}, + {0x11C2F, 0x11C36, WBP::Extend}, + {0x11C38, 0x11C3F, WBP::Extend}, + {0x11C40, 0x11C40, WBP::ALetter}, + {0x11C50, 0x11C59, WBP::Numeric}, + {0x11C72, 0x11C8F, WBP::ALetter}, + {0x11C92, 0x11CA7, WBP::Extend}, + {0x11CA9, 0x11CB6, WBP::Extend}, + {0x11D00, 0x11D06, WBP::ALetter}, + {0x11D08, 0x11D09, WBP::ALetter}, + {0x11D0B, 0x11D30, WBP::ALetter}, + {0x11D31, 0x11D36, WBP::Extend}, + {0x11D3A, 0x11D3A, WBP::Extend}, + {0x11D3C, 0x11D3D, WBP::Extend}, + {0x11D3F, 0x11D45, WBP::Extend}, + {0x11D46, 0x11D46, WBP::ALetter}, + {0x11D47, 0x11D47, WBP::Extend}, + {0x11D50, 0x11D59, WBP::Numeric}, + {0x11D60, 0x11D65, WBP::ALetter}, + {0x11D67, 0x11D68, WBP::ALetter}, + {0x11D6A, 0x11D89, WBP::ALetter}, + {0x11D8A, 0x11D8E, WBP::Extend}, + {0x11D90, 0x11D91, WBP::Extend}, + {0x11D93, 0x11D97, WBP::Extend}, + {0x11D98, 0x11D98, WBP::ALetter}, + {0x11DA0, 0x11DA9, WBP::Numeric}, + {0x11EE0, 0x11EF2, WBP::ALetter}, + {0x11EF3, 0x11EF6, WBP::Extend}, + {0x11FB0, 0x11FB0, WBP::ALetter}, + {0x12000, 0x12399, WBP::ALetter}, + {0x12400, 0x1246E, WBP::ALetter}, + {0x12480, 0x12543, WBP::ALetter}, + {0x13000, 0x1342E, WBP::ALetter}, + {0x13430, 0x13438, WBP::Format}, + {0x14400, 0x14646, WBP::ALetter}, + {0x16800, 0x16A38, WBP::ALetter}, + {0x16A40, 0x16A5E, WBP::ALetter}, + {0x16A60, 0x16A69, WBP::Numeric}, + {0x16AD0, 0x16AED, WBP::ALetter}, + {0x16AF0, 0x16AF4, WBP::Extend}, + {0x16B00, 0x16B2F, WBP::ALetter}, + {0x16B30, 0x16B36, WBP::Extend}, + {0x16B40, 0x16B43, WBP::ALetter}, + {0x16B50, 0x16B59, WBP::Numeric}, + {0x16B63, 0x16B77, WBP::ALetter}, + {0x16B7D, 0x16B8F, WBP::ALetter}, + {0x16E40, 0x16E7F, WBP::ALetter}, + {0x16F00, 0x16F4A, WBP::ALetter}, + {0x16F4F, 0x16F4F, WBP::Extend}, + {0x16F50, 0x16F50, WBP::ALetter}, + {0x16F51, 0x16F87, WBP::Extend}, + {0x16F8F, 0x16F92, WBP::Extend}, + {0x16F93, 0x16F9F, WBP::ALetter}, + {0x16FE0, 0x16FE1, WBP::ALetter}, + {0x16FE3, 0x16FE3, WBP::ALetter}, + {0x16FE4, 0x16FE4, WBP::Extend}, + {0x16FF0, 0x16FF1, WBP::Extend}, + {0x1B000, 0x1B000, WBP::Katakana}, + {0x1B164, 0x1B167, WBP::Katakana}, + {0x1BC00, 0x1BC6A, WBP::ALetter}, + {0x1BC70, 0x1BC7C, WBP::ALetter}, + {0x1BC80, 0x1BC88, WBP::ALetter}, + {0x1BC90, 0x1BC99, WBP::ALetter}, + {0x1BC9D, 0x1BC9E, WBP::Extend}, + {0x1BCA0, 0x1BCA3, WBP::Format}, + {0x1D165, 0x1D169, WBP::Extend}, + {0x1D16D, 0x1D172, WBP::Extend}, + {0x1D173, 0x1D17A, WBP::Format}, + {0x1D17B, 0x1D182, WBP::Extend}, + {0x1D185, 0x1D18B, WBP::Extend}, + {0x1D1AA, 0x1D1AD, WBP::Extend}, + {0x1D242, 0x1D244, WBP::Extend}, + {0x1D400, 0x1D454, WBP::ALetter}, + {0x1D456, 0x1D49C, WBP::ALetter}, + {0x1D49E, 0x1D49F, WBP::ALetter}, + {0x1D4A2, 0x1D4A2, WBP::ALetter}, + {0x1D4A5, 0x1D4A6, WBP::ALetter}, + {0x1D4A9, 0x1D4AC, WBP::ALetter}, + {0x1D4AE, 0x1D4B9, WBP::ALetter}, + {0x1D4BB, 0x1D4BB, WBP::ALetter}, + {0x1D4BD, 0x1D4C3, WBP::ALetter}, + {0x1D4C5, 0x1D505, WBP::ALetter}, + {0x1D507, 0x1D50A, WBP::ALetter}, + {0x1D50D, 0x1D514, WBP::ALetter}, + {0x1D516, 0x1D51C, WBP::ALetter}, + {0x1D51E, 0x1D539, WBP::ALetter}, + {0x1D53B, 0x1D53E, WBP::ALetter}, + {0x1D540, 0x1D544, WBP::ALetter}, + {0x1D546, 0x1D546, WBP::ALetter}, + {0x1D54A, 0x1D550, WBP::ALetter}, + {0x1D552, 0x1D6A5, WBP::ALetter}, + {0x1D6A8, 0x1D6C0, WBP::ALetter}, + {0x1D6C2, 0x1D6DA, WBP::ALetter}, + {0x1D6DC, 0x1D6FA, WBP::ALetter}, + {0x1D6FC, 0x1D714, WBP::ALetter}, + {0x1D716, 0x1D734, WBP::ALetter}, + {0x1D736, 0x1D74E, WBP::ALetter}, + {0x1D750, 0x1D76E, WBP::ALetter}, + {0x1D770, 0x1D788, WBP::ALetter}, + {0x1D78A, 0x1D7A8, WBP::ALetter}, + {0x1D7AA, 0x1D7C2, WBP::ALetter}, + {0x1D7C4, 0x1D7CB, WBP::ALetter}, + {0x1D7CE, 0x1D7FF, WBP::Numeric}, + {0x1DA00, 0x1DA36, WBP::Extend}, + {0x1DA3B, 0x1DA6C, WBP::Extend}, + {0x1DA75, 0x1DA75, WBP::Extend}, + {0x1DA84, 0x1DA84, WBP::Extend}, + {0x1DA9B, 0x1DA9F, WBP::Extend}, + {0x1DAA1, 0x1DAAF, WBP::Extend}, + {0x1E000, 0x1E006, WBP::Extend}, + {0x1E008, 0x1E018, WBP::Extend}, + {0x1E01B, 0x1E021, WBP::Extend}, + {0x1E023, 0x1E024, WBP::Extend}, + {0x1E026, 0x1E02A, WBP::Extend}, + {0x1E100, 0x1E12C, WBP::ALetter}, + {0x1E130, 0x1E136, WBP::Extend}, + {0x1E137, 0x1E13D, WBP::ALetter}, + {0x1E140, 0x1E149, WBP::Numeric}, + {0x1E14E, 0x1E14E, WBP::ALetter}, + {0x1E2C0, 0x1E2EB, WBP::ALetter}, + {0x1E2EC, 0x1E2EF, WBP::Extend}, + {0x1E2F0, 0x1E2F9, WBP::Numeric}, + {0x1E800, 0x1E8C4, WBP::ALetter}, + {0x1E8D0, 0x1E8D6, WBP::Extend}, + {0x1E900, 0x1E943, WBP::ALetter}, + {0x1E944, 0x1E94A, WBP::Extend}, + {0x1E94B, 0x1E94B, WBP::ALetter}, + {0x1E950, 0x1E959, WBP::Numeric}, + {0x1EE00, 0x1EE03, WBP::ALetter}, + {0x1EE05, 0x1EE1F, WBP::ALetter}, + {0x1EE21, 0x1EE22, WBP::ALetter}, + {0x1EE24, 0x1EE24, WBP::ALetter}, + {0x1EE27, 0x1EE27, WBP::ALetter}, + {0x1EE29, 0x1EE32, WBP::ALetter}, + {0x1EE34, 0x1EE37, WBP::ALetter}, + {0x1EE39, 0x1EE39, WBP::ALetter}, + {0x1EE3B, 0x1EE3B, WBP::ALetter}, + {0x1EE42, 0x1EE42, WBP::ALetter}, + {0x1EE47, 0x1EE47, WBP::ALetter}, + {0x1EE49, 0x1EE49, WBP::ALetter}, + {0x1EE4B, 0x1EE4B, WBP::ALetter}, + {0x1EE4D, 0x1EE4F, WBP::ALetter}, + {0x1EE51, 0x1EE52, WBP::ALetter}, + {0x1EE54, 0x1EE54, WBP::ALetter}, + {0x1EE57, 0x1EE57, WBP::ALetter}, + {0x1EE59, 0x1EE59, WBP::ALetter}, + {0x1EE5B, 0x1EE5B, WBP::ALetter}, + {0x1EE5D, 0x1EE5D, WBP::ALetter}, + {0x1EE5F, 0x1EE5F, WBP::ALetter}, + {0x1EE61, 0x1EE62, WBP::ALetter}, + {0x1EE64, 0x1EE64, WBP::ALetter}, + {0x1EE67, 0x1EE6A, WBP::ALetter}, + {0x1EE6C, 0x1EE72, WBP::ALetter}, + {0x1EE74, 0x1EE77, WBP::ALetter}, + {0x1EE79, 0x1EE7C, WBP::ALetter}, + {0x1EE7E, 0x1EE7E, WBP::ALetter}, + {0x1EE80, 0x1EE89, WBP::ALetter}, + {0x1EE8B, 0x1EE9B, WBP::ALetter}, + {0x1EEA1, 0x1EEA3, WBP::ALetter}, + {0x1EEA5, 0x1EEA9, WBP::ALetter}, + {0x1EEAB, 0x1EEBB, WBP::ALetter}, + {0x1F130, 0x1F149, WBP::ALetter}, + {0x1F150, 0x1F169, WBP::ALetter}, + {0x1F170, 0x1F189, WBP::ALetter}, + {0x1F1E6, 0x1F1FF, WBP::Regional_Indicator}, + {0x1F3FB, 0x1F3FF, WBP::Extend}, + {0x1FBF0, 0x1FBF9, WBP::Numeric}, + {0xE0001, 0xE0001, WBP::Format}, + {0xE0020, 0xE007F, WBP::Extend}, + {0xE0100, 0xE01EF, WBP::Extend}, +}}; + +// Construct table of just WBP::Extend character intervals +constexpr auto g_extend_characters{[]() constexpr { + // Compute number of extend character intervals + constexpr size_t size = []() constexpr { + size_t count = 0; + for (auto interval : g_word_break_intervals) { + if (interval.property == WBP::Extend) { + count++; + } + } + return count; + }(); + + // Create array of extend character intervals + std::array<Interval, size> result{}; + size_t index = 0; + for (auto interval : g_word_break_intervals) { + if (interval.property == WBP::Extend) { + result[index++] = {interval.first, interval.last}; // NOLINT + } + } + return result; +}()}; + +// Find a codepoint inside a sorted list of Interval. +template <size_t N> +bool Bisearch(uint32_t ucs, const std::array<Interval, N>& table) { + if (ucs < table.front().first || ucs > table.back().last) { // NOLINT + return false; + } + + int min = 0; + int max = N - 1; + while (max >= min) { + const int mid = (min + max) / 2; + if (ucs > table[mid].last) { // NOLINT + min = mid + 1; + } else if (ucs < table[mid].first) { // NOLINT + max = mid - 1; + } else { + return true; + } + } + + return false; +} + +// Find a value inside a sorted list of Interval + property. +template <class C, size_t N> +bool Bisearch(uint32_t ucs, const std::array<C, N>& table, C* out) { + if (ucs < table.front().first || ucs > table.back().last) { // NOLINT + return false; + } + + int min = 0; + int max = N - 1; + while (max >= min) { + const int mid = (min + max) / 2; + if (ucs > table[mid].last) { // NOLINT + min = mid + 1; + } else if (ucs < table[mid].first) { // NOLINT + max = mid - 1; + } else { + *out = table[mid]; // NOLINT + return true; + } + } + + return false; +} + +int codepoint_width(uint32_t ucs) { + if (ftxui::IsControl(ucs)) { + return -1; + } + + if (ftxui::IsCombining(ucs)) { + return 0; + } + + if (ftxui::IsFullWidth(ucs)) { + return 2; + } + + return 1; +} + +} // namespace + +namespace ftxui { + +// From UTF8 encoded string |input|, eat in between 1 and 4 byte representing +// one codepoint. Put the codepoint into |ucs|. Start at |start| and update +// |end| to represent the beginning of the next byte to eat for consecutive +// executions. +bool EatCodePoint(const std::string& input, + size_t start, + size_t* end, + uint32_t* ucs) { + if (start >= input.size()) { + *end = start + 1; + return false; + } + const uint8_t C0 = input[start]; + + // 1 byte string. + if ((C0 & 0b1000'0000) == 0b0000'0000) { // NOLINT + *ucs = C0 & 0b0111'1111; // NOLINT + *end = start + 1; + return true; + } + + // 2 byte string. + if ((C0 & 0b1110'0000) == 0b1100'0000 && // NOLINT + start + 1 < input.size()) { + const uint8_t C1 = input[start + 1]; + *ucs = 0; + *ucs += C0 & 0b0001'1111; // NOLINT + *ucs <<= 6; // NOLINT + *ucs += C1 & 0b0011'1111; // NOLINT + *end = start + 2; + return true; + } + + // 3 byte string. + if ((C0 & 0b1111'0000) == 0b1110'0000 && // NOLINT + start + 2 < input.size()) { + const uint8_t C1 = input[start + 1]; + const uint8_t C2 = input[start + 2]; + *ucs = 0; + *ucs += C0 & 0b0000'1111; // NOLINT + *ucs <<= 6; // NOLINT + *ucs += C1 & 0b0011'1111; // NOLINT + *ucs <<= 6; // NOLINT + *ucs += C2 & 0b0011'1111; // NOLINT + *end = start + 3; + return true; + } + + // 4 byte string. + if ((C0 & 0b1111'1000) == 0b1111'0000 && // NOLINT + start + 3 < input.size()) { + const uint8_t C1 = input[start + 1]; + const uint8_t C2 = input[start + 2]; + const uint8_t C3 = input[start + 3]; + *ucs = 0; + *ucs += C0 & 0b0000'0111; // NOLINT + *ucs <<= 6; // NOLINT + *ucs += C1 & 0b0011'1111; // NOLINT + *ucs <<= 6; // NOLINT + *ucs += C2 & 0b0011'1111; // NOLINT + *ucs <<= 6; // NOLINT + *ucs += C3 & 0b0011'1111; // NOLINT + *end = start + 4; + return true; + } + + *end = start + 1; + return false; +} + +// From UTF16 encoded string |input|, eat in between 1 and 4 byte representing +// one codepoint. Put the codepoint into |ucs|. Start at |start| and update +// |end| to represent the beginning of the next byte to eat for consecutive +// executions. +bool EatCodePoint(const std::wstring& input, + size_t start, + size_t* end, + uint32_t* ucs) { + if (start >= input.size()) { + *end = start + 1; + return false; + } + + // On linux wstring uses the UTF32 encoding: + if constexpr (sizeof(wchar_t) == 4) { + *ucs = input[start]; // NOLINT + *end = start + 1; + return true; + } + + // On windows, wstring uses the UTF16 encoding: + int32_t C0 = input[start]; // NOLINT + + // 1 word size: + if (C0 < 0xd800 || C0 >= 0xdc00) { // NOLINT + *ucs = C0; + *end = start + 1; + return true; + } + + // 2 word size: + if (start + 1 >= input.size()) { + *end = start + 2; + return false; + } + + int32_t C1 = input[start + 1]; // NOLINT + *ucs = ((C0 & 0x3ff) << 10) + (C1 & 0x3ff) + 0x10000; // NOLINT + *end = start + 2; + return true; +} + +bool IsCombining(uint32_t ucs) { + return Bisearch(ucs, g_extend_characters); +} + +bool IsFullWidth(uint32_t ucs) { + if (ucs < 0x0300) // Quick path: // NOLINT + return false; + + return Bisearch(ucs, g_full_width_characters); +} + +bool IsControl(uint32_t ucs) { + if (ucs == 0) { + return true; + } + if (ucs < 32) { // NOLINT + const uint32_t LINE_FEED = 10; + return ucs != LINE_FEED; + } + if (ucs >= 0x7f && ucs < 0xa0) { // NOLINT + return true; + } + return false; +} + +WordBreakProperty CodepointToWordBreakProperty(uint32_t codepoint) { + WordBreakPropertyInterval interval = {0, 0, WBP::ALetter}; + std::ignore = Bisearch(codepoint, g_word_break_intervals, &interval); + return interval.property; +} + +int wchar_width(wchar_t ucs) { + return codepoint_width(uint32_t(ucs)); +} + +int wstring_width(const std::wstring& text) { + int width = 0; + + for (const wchar_t& it : text) { + const int w = wchar_width(it); + if (w < 0) { + return -1; + } + width += w; + } + return width; +} + +int string_width(const std::string& input) { + int width = 0; + size_t start = 0; + while (start < input.size()) { + uint32_t codepoint = 0; + if (!EatCodePoint(input, start, &start, &codepoint)) { + continue; + } + + if (IsControl(codepoint)) { + continue; + } + + if (IsCombining(codepoint)) { + continue; + } + + if (IsFullWidth(codepoint)) { + width += 2; + continue; + } + + width += 1; + } + return width; +} + +std::vector<std::string> Utf8ToGlyphs(const std::string& input) { + std::vector<std::string> out; + out.reserve(input.size()); + size_t start = 0; + size_t end = 0; + while (start < input.size()) { + uint32_t codepoint = 0; + if (!EatCodePoint(input, start, &end, &codepoint)) { + start = end; + continue; + } + + const std::string append = input.substr(start, end - start); + start = end; + + // Ignore control characters. + if (IsControl(codepoint)) { + continue; + } + + // Combining characters are put with the previous glyph they are modifying. + if (IsCombining(codepoint)) { + if (!out.empty()) { + out.back() += append; + } + continue; + } + + // Fullwidth characters take two cells. The second is made of the empty + // string to reserve the space the first is taking. + if (IsFullWidth(codepoint)) { + out.push_back(append); + out.emplace_back(""); + continue; + } + + // Normal characters: + out.push_back(append); + } + return out; +} + +size_t GlyphPrevious(const std::string& input, size_t start) { + while (true) { + if (start == 0) { + return 0; + } + start--; + + // Skip the UTF8 continuation bytes. + if ((input[start] & 0b1100'0000) == 0b1000'0000) { + continue; + } + + uint32_t codepoint = 0; + size_t end = 0; + const bool eaten = EatCodePoint(input, start, &end, &codepoint); + + // Ignore invalid, control characters and combining characters. + if (!eaten || IsControl(codepoint) || IsCombining(codepoint)) { + continue; + } + + return start; + } +} + +size_t GlyphNext(const std::string& input, size_t start) { + bool glyph_found = false; + while (start < input.size()) { + size_t end = 0; + uint32_t codepoint = 0; + const bool eaten = EatCodePoint(input, start, &end, &codepoint); + + // Ignore invalid, control characters and combining characters. + if (!eaten || IsControl(codepoint) || IsCombining(codepoint)) { + start = end; + continue; + } + + // We eat the beginning of the next glyph. If we are eating the one + // requested, return its start position immediately. + if (glyph_found) { + return static_cast<int>(start); + } + + // Otherwise, skip this glyph and iterate: + glyph_found = true; + start = end; + } + return static_cast<int>(input.size()); +} + +size_t GlyphIterate(const std::string& input, int glyph_offset, size_t start) { + if (glyph_offset >= 0) { + for (int i = 0; i < glyph_offset; ++i) { + start = GlyphNext(input, start); + } + return start; + } else { + for (int i = 0; i < -glyph_offset; ++i) { + start = GlyphPrevious(input, start); + } + return start; + } +} + +std::vector<int> CellToGlyphIndex(const std::string& input) { + int x = -1; + std::vector<int> out; + out.reserve(input.size()); + size_t start = 0; + size_t end = 0; + while (start < input.size()) { + uint32_t codepoint = 0; + const bool eaten = EatCodePoint(input, start, &end, &codepoint); + start = end; + + // Ignore invalid / control characters. + if (!eaten || IsControl(codepoint)) { + continue; + } + + // Combining characters are put with the previous glyph they are modifying. + if (IsCombining(codepoint)) { + if (x == -1) { + ++x; + out.push_back(x); + } + continue; + } + + // Fullwidth characters take two cells. The second is made of the empty + // string to reserve the space the first is taking. + if (IsFullWidth(codepoint)) { + ++x; + out.push_back(x); + out.push_back(x); + continue; + } + + // Normal characters: + ++x; + out.push_back(x); + } + return out; +} + +int GlyphCount(const std::string& input) { + int size = 0; + size_t start = 0; + size_t end = 0; + while (start < input.size()) { + uint32_t codepoint = 0; + const bool eaten = EatCodePoint(input, start, &end, &codepoint); + start = end; + + // Ignore invalid characters: + if (!eaten || IsControl(codepoint)) { + continue; + } + + // Ignore combining characters, except when they don't have a preceding to + // combine with. + if (IsCombining(codepoint)) { + if (size == 0) { + size++; + } + continue; + } + + size++; + } + return size; +} + +std::vector<WordBreakProperty> Utf8ToWordBreakProperty( + const std::string& input) { + std::vector<WordBreakProperty> out; + out.reserve(input.size()); + size_t start = 0; + size_t end = 0; + while (start < input.size()) { + uint32_t codepoint = 0; + if (!EatCodePoint(input, start, &end, &codepoint)) { + start = end; + continue; + } + start = end; + + // Ignore control characters. + if (IsControl(codepoint)) { + continue; + } + + // Ignore combining characters. + if (IsCombining(codepoint)) { + continue; + } + + WordBreakPropertyInterval interval = {0, 0, WBP::ALetter}; + std::ignore = Bisearch(codepoint, g_word_break_intervals, &interval); + out.push_back(interval.property); + } + return out; +} + +/// Convert a UTF8 std::string into a std::wstring. +std::string to_string(const std::wstring& s) { + std::string out; + + size_t i = 0; + uint32_t codepoint = 0; + while (EatCodePoint(s, i, &i, &codepoint)) { + // Code point <-> UTF-8 conversion + // + // ┏━━━━━━━━┳━━━━━━━━┳━━━━━━━━┳━━━━━━━━┓ + // ┃Byte 1 ┃Byte 2 ┃Byte 3 ┃Byte 4 ┃ + // ┡━━━━━━━━╇━━━━━━━━╇━━━━━━━━╇━━━━━━━━┩ + // │0xxxxxxx│ │ │ │ + // ├────────┼────────┼────────┼────────┤ + // │110xxxxx│10xxxxxx│ │ │ + // ├────────┼────────┼────────┼────────┤ + // │1110xxxx│10xxxxxx│10xxxxxx│ │ + // ├────────┼────────┼────────┼────────┤ + // │11110xxx│10xxxxxx│10xxxxxx│10xxxxxx│ + // └────────┴────────┴────────┴────────┘ + + // 1 byte UTF8 + if (codepoint <= 0b000'0000'0111'1111) { // NOLINT + const uint8_t p1 = codepoint; + out.push_back(p1); // NOLINT + continue; + } + + // 2 bytes UTF8 + if (codepoint <= 0b000'0111'1111'1111) { // NOLINT + uint8_t p2 = codepoint & 0b111111; // NOLINT + codepoint >>= 6; // NOLINT + uint8_t p1 = codepoint; // NOLINT + out.push_back(0b11000000 + p1); // NOLINT + out.push_back(0b10000000 + p2); // NOLINT + continue; + } + + // 3 bytes UTF8 + if (codepoint <= 0b1111'1111'1111'1111) { // NOLINT + uint8_t p3 = codepoint & 0b111111; // NOLINT + codepoint >>= 6; // NOLINT + uint8_t p2 = codepoint & 0b111111; // NOLINT + codepoint >>= 6; // NOLINT + uint8_t p1 = codepoint; // NOLINT + out.push_back(0b11100000 + p1); // NOLINT + out.push_back(0b10000000 + p2); // NOLINT + out.push_back(0b10000000 + p3); // NOLINT + continue; + } + + // 4 bytes UTF8 + if (codepoint <= 0b1'0000'1111'1111'1111'1111) { // NOLINT + uint8_t p4 = codepoint & 0b111111; // NOLINT + codepoint >>= 6; // NOLINT + uint8_t p3 = codepoint & 0b111111; // NOLINT + codepoint >>= 6; // NOLINT + uint8_t p2 = codepoint & 0b111111; // NOLINT + codepoint >>= 6; // NOLINT + uint8_t p1 = codepoint; // NOLINT + out.push_back(0b11110000 + p1); // NOLINT + out.push_back(0b10000000 + p2); // NOLINT + out.push_back(0b10000000 + p3); // NOLINT + out.push_back(0b10000000 + p4); // NOLINT + continue; + } + + // Something else? + } + return out; +} + +/// Convert a std::wstring into a UTF8 std::string. +std::wstring to_wstring(const std::string& s) { + std::wstring out; + + size_t i = 0; + uint32_t codepoint = 0; + while (EatCodePoint(s, i, &i, &codepoint)) { + // On linux wstring are UTF32 encoded: + if constexpr (sizeof(wchar_t) == 4) { + out.push_back(codepoint); // NOLINT + continue; + } + + // On Windows, wstring are UTF16 encoded: + + // Codepoint encoded using 1 word: + // NOLINTNEXTLINE + if (codepoint < 0xD800 || (codepoint > 0xDFFF && codepoint < 0x10000)) { + uint16_t p0 = codepoint; // NOLINT + out.push_back(p0); // NOLINT + continue; + } + + // Codepoint encoded using 2 words: + codepoint -= 0x010000; // NOLINT + uint16_t p0 = (((codepoint << 12) >> 22) + 0xD800); // NOLINT + uint16_t p1 = (((codepoint << 22) >> 22) + 0xDC00); // NOLINT + out.push_back(p0); // NOLINT + out.push_back(p1); // NOLINT + } + return out; +} + +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/screen/string_internal.hpp b/contrib/libs/ftxui/src/ftxui/screen/string_internal.hpp new file mode 100644 index 00000000000..048f3278105 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/screen/string_internal.hpp @@ -0,0 +1,67 @@ +// Copyright 2023 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_SCREEN_STRING_INTERNAL_HPP +#define FTXUI_SCREEN_STRING_INTERNAL_HPP + +#include <cstdint> +#include <string> +#include <vector> + +namespace ftxui { + +bool EatCodePoint(const std::string& input, + size_t start, + size_t* end, + uint32_t* ucs); +bool EatCodePoint(const std::wstring& input, + size_t start, + size_t* end, + uint32_t* ucs); + +bool IsCombining(uint32_t ucs); +bool IsFullWidth(uint32_t ucs); +bool IsControl(uint32_t ucs); + +size_t GlyphPrevious(const std::string& input, size_t start); +size_t GlyphNext(const std::string& input, size_t start); + +// Return the index in the |input| string of the glyph at |glyph_offset|, +// starting at |start| +size_t GlyphIterate(const std::string& input, + int glyph_offset, + size_t start = 0); + +// Returns the number of glyphs in |input|. +int GlyphCount(const std::string& input); + +// Properties from: +// https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/WordBreakProperty.txt +enum class WordBreakProperty : int8_t { + ALetter, + CR, + Double_Quote, + Extend, + ExtendNumLet, + Format, + Hebrew_Letter, + Katakana, + LF, + MidLetter, + MidNum, + MidNumLet, + Newline, + Numeric, + Regional_Indicator, + Single_Quote, + WSegSpace, + ZWJ, +}; +WordBreakProperty CodepointToWordBreakProperty(uint32_t codepoint); +std::vector<WordBreakProperty> Utf8ToWordBreakProperty( + const std::string& input); + +bool IsWordBreakingCharacter(const std::string& input, size_t glyph_index); +} // namespace ftxui + +#endif /* end of include guard: FTXUI_SCREEN_STRING_INTERNAL_HPP */ diff --git a/contrib/libs/ftxui/src/ftxui/screen/terminal.cpp b/contrib/libs/ftxui/src/ftxui/screen/terminal.cpp new file mode 100644 index 00000000000..8878a1f99a7 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/screen/terminal.cpp @@ -0,0 +1,146 @@ +// Copyright 2020 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#include <cstdlib> // for getenv +#include <string> // for string, allocator + +#include "ftxui/screen/terminal.hpp" + +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN + +#ifndef NOMINMAX +#define NOMINMAX +#endif + +#include <windows.h> +#else +#include <sys/ioctl.h> // for winsize, ioctl, TIOCGWINSZ +#include <unistd.h> // for STDOUT_FILENO +#endif + +namespace ftxui { + +namespace { + +bool g_cached = false; // NOLINT +Terminal::Color g_cached_supported_color; // NOLINT + +Dimensions& FallbackSize() { +#if defined(__EMSCRIPTEN__) + // This dimension was chosen arbitrarily to be able to display: + // https://arthursonzogni.com/FTXUI/examples + // This will have to be improved when someone has time to implement and need + // it. + constexpr int fallback_width = 140; + constexpr int fallback_height = 43; +#else + // The terminal size in VT100 was 80x24. It is still used nowadays by + // default in many terminal emulator. That's a good choice for a fallback + // value. + constexpr int fallback_width = 80; + constexpr int fallback_height = 24; +#endif + static Dimensions g_fallback_size{ + fallback_width, + fallback_height, + }; + return g_fallback_size; +} + +const char* Safe(const char* c) { + return (c != nullptr) ? c : ""; +} + +bool Contains(const std::string& s, const char* key) { + return s.find(key) != std::string::npos; +} + +Terminal::Color ComputeColorSupport() { +#if defined(__EMSCRIPTEN__) + return Terminal::Color::TrueColor; +#endif + + std::string COLORTERM = Safe(std::getenv("COLORTERM")); // NOLINT + if (Contains(COLORTERM, "24bit") || Contains(COLORTERM, "truecolor")) { + return Terminal::Color::TrueColor; + } + + std::string TERM = Safe(std::getenv("TERM")); // NOLINT + if (Contains(COLORTERM, "256") || Contains(TERM, "256")) { + return Terminal::Color::Palette256; + } + +#if defined(FTXUI_MICROSOFT_TERMINAL_FALLBACK) + // Microsoft terminals do not properly declare themselve supporting true + // colors: https://github.com/microsoft/terminal/issues/1040 + // As a fallback, assume microsoft terminal are the ones not setting those + // variables, and enable true colors. + if (TERM.empty() && COLORTERM.empty()) { + return Terminal::Color::TrueColor; + } +#endif + + return Terminal::Color::Palette16; +} + +} // namespace + +namespace Terminal { + +/// @brief Get the terminal size. +/// @return The terminal size. +/// @ingroup screen +Dimensions Size() { +#if defined(__EMSCRIPTEN__) + // This dimension was chosen arbitrarily to be able to display: + // https://arthursonzogni.com/FTXUI/examples + // This will have to be improved when someone has time to implement and need + // it. + return FallbackSize(); +#elif defined(_WIN32) + CONSOLE_SCREEN_BUFFER_INFO csbi; + + if (GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &csbi)) { + return Dimensions{csbi.srWindow.Right - csbi.srWindow.Left + 1, + csbi.srWindow.Bottom - csbi.srWindow.Top + 1}; + } + + return FallbackSize(); +#else + winsize w{}; + const int status = ioctl(STDOUT_FILENO, TIOCGWINSZ, &w); // NOLINT + // The ioctl return value result should be checked. Some operating systems + // don't support TIOCGWINSZ. + if (w.ws_col == 0 || w.ws_row == 0 || status < 0) { + return FallbackSize(); + } + return Dimensions{w.ws_col, w.ws_row}; +#endif +} + +/// @brief Override terminal size in case auto-detection fails +/// @param fallbackSize Terminal dimensions to fallback to +void SetFallbackSize(const Dimensions& fallbackSize) { + FallbackSize() = fallbackSize; +} + +/// @brief Get the color support of the terminal. +/// @ingroup screen +Color ColorSupport() { + if (!g_cached) { + g_cached = true; + g_cached_supported_color = ComputeColorSupport(); + } + return g_cached_supported_color; +} + +/// @brief Override terminal color support in case auto-detection fails +/// @ingroup dom +void SetColorSupport(Color color) { + g_cached = true; + g_cached_supported_color = color; +} + +} // namespace Terminal +} // namespace ftxui diff --git a/contrib/libs/ftxui/src/ftxui/screen/util.hpp b/contrib/libs/ftxui/src/ftxui/screen/util.hpp new file mode 100644 index 00000000000..0f2a5f27db4 --- /dev/null +++ b/contrib/libs/ftxui/src/ftxui/screen/util.hpp @@ -0,0 +1,17 @@ +// Copyright 2022 Arthur Sonzogni. All rights reserved. +// Use of this source code is governed by the MIT license that can be found in +// the LICENSE file. +#ifndef FTXUI_SCREEN_UTIL_HPP +#define FTXUI_SCREEN_UTIL_HPP + +namespace ftxui::util { + +// Similar to std::clamp, but allow hi to be lower than lo. +template <class T> +constexpr const T& clamp(const T& v, const T& lo, const T& hi) { + return v < lo ? lo : hi < v ? hi : v; +} + +} // namespace ftxui::util + +#endif /* end of include guard: FTXUI_SCREEN_UTIL_HPP */ diff --git a/contrib/libs/ftxui/ya.make b/contrib/libs/ftxui/ya.make new file mode 100644 index 00000000000..98fdc10edfb --- /dev/null +++ b/contrib/libs/ftxui/ya.make @@ -0,0 +1,101 @@ +# Generated by devtools/yamaker from nixpkgs 22.11. + +LIBRARY() + +LICENSE( + MIT AND + WTFPL +) + +LICENSE_TEXTS(.yandex_meta/licenses.list.txt) + +VERSION(6.0.2) + +ORIGINAL_SOURCE(https://github.com/ArthurSonzogni/FTXUI/archive/v6.0.2.tar.gz) + +ADDINCL( + GLOBAL contrib/libs/ftxui/include + contrib/libs/ftxui/src +) + +NO_COMPILER_WARNINGS() + +NO_UTIL() + +SRCS( + src/ftxui/component/animation.cpp + src/ftxui/component/button.cpp + src/ftxui/component/catch_event.cpp + src/ftxui/component/checkbox.cpp + src/ftxui/component/collapsible.cpp + src/ftxui/component/component.cpp + src/ftxui/component/component_options.cpp + src/ftxui/component/container.cpp + src/ftxui/component/dropdown.cpp + src/ftxui/component/event.cpp + src/ftxui/component/hoverable.cpp + src/ftxui/component/input.cpp + src/ftxui/component/loop.cpp + src/ftxui/component/maybe.cpp + src/ftxui/component/menu.cpp + src/ftxui/component/modal.cpp + src/ftxui/component/radiobox.cpp + src/ftxui/component/renderer.cpp + src/ftxui/component/resizable_split.cpp + src/ftxui/component/screen_interactive.cpp + src/ftxui/component/slider.cpp + src/ftxui/component/terminal_input_parser.cpp + src/ftxui/component/util.cpp + src/ftxui/component/window.cpp + src/ftxui/dom/automerge.cpp + src/ftxui/dom/blink.cpp + src/ftxui/dom/bold.cpp + src/ftxui/dom/border.cpp + src/ftxui/dom/box_helper.cpp + src/ftxui/dom/canvas.cpp + src/ftxui/dom/clear_under.cpp + src/ftxui/dom/color.cpp + src/ftxui/dom/composite_decorator.cpp + src/ftxui/dom/dbox.cpp + src/ftxui/dom/dim.cpp + src/ftxui/dom/flex.cpp + src/ftxui/dom/flexbox.cpp + src/ftxui/dom/flexbox_config.cpp + src/ftxui/dom/flexbox_helper.cpp + src/ftxui/dom/focus.cpp + src/ftxui/dom/frame.cpp + src/ftxui/dom/gauge.cpp + src/ftxui/dom/graph.cpp + src/ftxui/dom/gridbox.cpp + src/ftxui/dom/hbox.cpp + src/ftxui/dom/hyperlink.cpp + src/ftxui/dom/inverted.cpp + src/ftxui/dom/italic.cpp + src/ftxui/dom/linear_gradient.cpp + src/ftxui/dom/node.cpp + src/ftxui/dom/node_decorator.cpp + src/ftxui/dom/paragraph.cpp + src/ftxui/dom/reflect.cpp + src/ftxui/dom/scroll_indicator.cpp + src/ftxui/dom/selection.cpp + src/ftxui/dom/selection_style.cpp + src/ftxui/dom/separator.cpp + src/ftxui/dom/size.cpp + src/ftxui/dom/spinner.cpp + src/ftxui/dom/strikethrough.cpp + src/ftxui/dom/table.cpp + src/ftxui/dom/text.cpp + src/ftxui/dom/underlined.cpp + src/ftxui/dom/underlined_double.cpp + src/ftxui/dom/util.cpp + src/ftxui/dom/vbox.cpp + src/ftxui/screen/box.cpp + src/ftxui/screen/color.cpp + src/ftxui/screen/color_info.cpp + src/ftxui/screen/image.cpp + src/ftxui/screen/screen.cpp + src/ftxui/screen/string.cpp + src/ftxui/screen/terminal.cpp +) + +END() diff --git a/contrib/libs/grpc/patches/pr37768-openssl-deinit.patch b/contrib/libs/grpc/patches/pr37768-openssl-deinit.patch deleted file mode 100644 index d25735d163d..00000000000 --- a/contrib/libs/grpc/patches/pr37768-openssl-deinit.patch +++ /dev/null @@ -1,111 +0,0 @@ -From cbccf975d07da4b6ddb716518de58917348c3e14 Mon Sep 17 00:00:00 2001 -From: Yousuk Seung <ysseung@google.com> -Date: Sat, 21 Sep 2024 15:22:10 -0700 -Subject: [PATCH] [ssl] Ensure OPENSSL global cleanup happens after gRPC - shutdowns (#37768) - -Ensure OPENSSL global clean up happens after gRPC shutdown completes. OPENSSL registers an exit handler to clean up global objects, which may happen before gRPC removes all references to OPENSSL. - -<!-- - -If you know who should review your pull request, please assign it to that -person, otherwise the pull request would get assigned randomly. - -If your pull request is for a specific language, please add the appropriate -lang label. - ---> - -Closes #37768 - -COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/37768 from yousukseung:openssl-atexit-wait d3d1c964a837179050acebd5e82b1d1e20338611 -PiperOrigin-RevId: 677284514 ---- - BUILD | 2 ++ - src/core/lib/surface/init.cc | 19 ++++++++++++++ - src/core/lib/surface/init.h | 5 ++++ - src/core/tsi/ssl_transport_security.cc | 8 ++++++ - test/core/surface/init_test.cc | 35 ++++++++++++++++++++++++++ - 5 files changed, 69 insertions(+) - -diff --git a/src/core/lib/surface/init.cc b/src/core/lib/surface/init.cc -index 637941ccbd3d0..233f6f97d1677 100644 ---- a/src/core/lib/surface/init.cc -+++ b/src/core/lib/surface/init.cc -@@ -20,6 +20,8 @@ - #include <limits.h> - - #include "y_absl/base/thread_annotations.h" -+#include "y_absl/time/clock.h" -+#include "y_absl/time/time.h" - - #include <grpc/fork.h> - #include <grpc/grpc.h> -@@ -216,3 +218,16 @@ void grpc_maybe_wait_for_async_shutdown(void) { - g_shutting_down_cv->Wait(g_init_mu); - } - } -+ -+bool grpc_wait_for_shutdown_with_timeout(y_absl::Duration timeout) { -+ const auto started = y_absl::Now(); -+ const auto deadline = started + timeout; -+ gpr_once_init(&g_basic_init, do_basic_init); -+ grpc_core::MutexLock lock(g_init_mu); -+ while (g_initializations != 0) { -+ if (g_shutting_down_cv->WaitWithDeadline(g_init_mu, deadline)) { -+ return false; -+ } -+ } -+ return true; -+} -diff --git a/src/core/lib/surface/init.h b/src/core/lib/surface/init.h -index 0d0035ae30474..804b79c5d5b31 100644 ---- a/src/core/lib/surface/init.h -+++ b/src/core/lib/surface/init.h -@@ -18,8 +18,13 @@ - #ifndef GRPC_SRC_CORE_LIB_SURFACE_INIT_H - #define GRPC_SRC_CORE_LIB_SURFACE_INIT_H - -+#include "y_absl/time/time.h" -+ - #include <grpc/support/port_platform.h> - - void grpc_maybe_wait_for_async_shutdown(void); - -+// Returns false if the timeout expired before fully shut down. -+bool grpc_wait_for_shutdown_with_timeout(y_absl::Duration timeout); -+ - #endif // GRPC_SRC_CORE_LIB_SURFACE_INIT_H -diff --git a/src/core/tsi/ssl_transport_security.cc b/src/core/tsi/ssl_transport_security.cc -index 8012dc8459cac..4718417435154 100644 ---- a/src/core/tsi/ssl_transport_security.cc -+++ b/src/core/tsi/ssl_transport_security.cc -@@ -23,6 +23,8 @@ - #include <limits.h> - #include <string.h> - -+# include <cstdlib> -+ - // TODO(jboeuf): refactor inet_ntop into a portability header. - // Note: for whomever reads this and tries to refactor this, this - // can't be in grpc, it has to be in gpr. -@@ -58,6 +60,7 @@ - - #include "src/core/lib/gpr/useful.h" - #include "src/core/lib/gprpp/crash.h" -+#include "src/core/lib/surface/init.h" - #include "src/core/tsi/ssl/key_logging/ssl_key_logging.h" - #include "src/core/tsi/ssl/session_cache/ssl_session_cache.h" - #include "src/core/tsi/ssl_transport_security_utils.h" -@@ -189,6 +192,11 @@ static void verified_root_cert_free(void* /*parent*/, void* ptr, - static void init_openssl(void) { - #if OPENSSL_VERSION_NUMBER >= 0x10100000 - OPENSSL_init_ssl(0, nullptr); -+ // Ensure OPENSSL global clean up happens after gRPC shutdown completes. -+ // OPENSSL registers an exit handler to clean up global objects, which -+ // otherwise may happen before gRPC removes all references to OPENSSL. Below -+ // exit handler is guaranteed to run after OPENSSL's. -+ std::atexit([]() { grpc_wait_for_shutdown_with_timeout(y_absl::Seconds(2)); }); - #else - SSL_library_init(); - SSL_load_error_strings(); diff --git a/contrib/libs/grpc/patches/pr38101_inverted_length_check.patch b/contrib/libs/grpc/patches/pr38101_inverted_length_check.patch new file mode 100644 index 00000000000..6afdb4e9ecc --- /dev/null +++ b/contrib/libs/grpc/patches/pr38101_inverted_length_check.patch @@ -0,0 +1,17 @@ +commit 35597cf0f897a4e3a41ded9a25955ce4103c156e (HEAD -> fix-grpc) +author: reshilkin +date: 2025-05-22T17:04:11+03:00 + + DEVTOOLSSUPPORT-63262 Fix assert in grpc + +--- contrib/libs/grpc/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc (ce30871a97c577260d19661a2c9dca4fd0ec6095) ++++ contrib/libs/grpc/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc (35597cf0f897a4e3a41ded9a25955ce4103c156e) +@@ -286,7 +286,7 @@ class GrpcPolledFdWindows { + // c-ares overloads this recv_from virtual socket function to receive + // data on both UDP and TCP sockets, and from is nullptr for TCP. + if (from != nullptr) { +- GPR_ASSERT(*from_len <= recv_from_source_addr_len_); ++ GPR_ASSERT(*from_len >= recv_from_source_addr_len_); + memcpy(from, &recv_from_source_addr_, recv_from_source_addr_len_); + *from_len = recv_from_source_addr_len_; + } diff --git a/contrib/libs/grpc/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc b/contrib/libs/grpc/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc index 594876b2801..fc27dc06289 100644 --- a/contrib/libs/grpc/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +++ b/contrib/libs/grpc/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc @@ -286,7 +286,7 @@ class GrpcPolledFdWindows { // c-ares overloads this recv_from virtual socket function to receive // data on both UDP and TCP sockets, and from is nullptr for TCP. if (from != nullptr) { - GPR_ASSERT(*from_len <= recv_from_source_addr_len_); + GPR_ASSERT(*from_len >= recv_from_source_addr_len_); memcpy(from, &recv_from_source_addr_, recv_from_source_addr_len_); *from_len = recv_from_source_addr_len_; } diff --git a/contrib/libs/grpc/src/core/lib/surface/init.cc b/contrib/libs/grpc/src/core/lib/surface/init.cc index 59521f8348e..4527f273bf6 100644 --- a/contrib/libs/grpc/src/core/lib/surface/init.cc +++ b/contrib/libs/grpc/src/core/lib/surface/init.cc @@ -23,8 +23,6 @@ #include <limits.h> #include "y_absl/base/thread_annotations.h" -#include "y_absl/time/clock.h" -#include "y_absl/time/time.h" #include <grpc/fork.h> #include <grpc/grpc.h> @@ -231,16 +229,3 @@ void grpc_maybe_wait_for_async_shutdown(void) { g_shutting_down_cv->Wait(g_init_mu); } } - -bool grpc_wait_for_shutdown_with_timeout(y_absl::Duration timeout) { - const auto started = y_absl::Now(); - const auto deadline = started + timeout; - gpr_once_init(&g_basic_init, do_basic_init); - grpc_core::MutexLock lock(g_init_mu); - while (g_initializations != 0) { - if (g_shutting_down_cv->WaitWithDeadline(g_init_mu, deadline)) { - return false; - } - } - return true; -} diff --git a/contrib/libs/grpc/src/core/lib/surface/init.h b/contrib/libs/grpc/src/core/lib/surface/init.h index 20a2fdf3256..0d0035ae304 100644 --- a/contrib/libs/grpc/src/core/lib/surface/init.h +++ b/contrib/libs/grpc/src/core/lib/surface/init.h @@ -18,13 +18,8 @@ #ifndef GRPC_SRC_CORE_LIB_SURFACE_INIT_H #define GRPC_SRC_CORE_LIB_SURFACE_INIT_H -#include "y_absl/time/time.h" - #include <grpc/support/port_platform.h> void grpc_maybe_wait_for_async_shutdown(void); -// Returns false if the timeout expired before fully shut down. -bool grpc_wait_for_shutdown_with_timeout(y_absl::Duration timeout); - #endif // GRPC_SRC_CORE_LIB_SURFACE_INIT_H diff --git a/contrib/libs/grpc/src/core/tsi/ssl_transport_security.cc b/contrib/libs/grpc/src/core/tsi/ssl_transport_security.cc index fcccd0c5dd5..deb3b952d4b 100644 --- a/contrib/libs/grpc/src/core/tsi/ssl_transport_security.cc +++ b/contrib/libs/grpc/src/core/tsi/ssl_transport_security.cc @@ -23,8 +23,6 @@ #include <limits.h> #include <string.h> -# include <cstdlib> - // TODO(jboeuf): refactor inet_ntop into a portability header. // Note: for whomever reads this and tries to refactor this, this // can't be in grpc, it has to be in gpr. @@ -60,7 +58,6 @@ #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/crash.h" -#include "src/core/lib/surface/init.h" #include "src/core/tsi/ssl/key_logging/ssl_key_logging.h" #include "src/core/tsi/ssl/session_cache/ssl_session_cache.h" #include "src/core/tsi/ssl_transport_security_utils.h" @@ -175,11 +172,6 @@ static unsigned long openssl_thread_id_cb(void) { static void init_openssl(void) { #if OPENSSL_VERSION_NUMBER >= 0x10100000 OPENSSL_init_ssl(0, nullptr); - // Ensure OPENSSL global clean up happens after gRPC shutdown completes. - // OPENSSL registers an exit handler to clean up global objects, which - // otherwise may happen before gRPC removes all references to OPENSSL. Below - // exit handler is guaranteed to run after OPENSSL's. - std::atexit([]() { grpc_wait_for_shutdown_with_timeout(y_absl::Seconds(2)); }); #else SSL_library_init(); SSL_load_error_strings(); diff --git a/contrib/libs/tcmalloc/.yandex_meta/build.ym b/contrib/libs/tcmalloc/.yandex_meta/build.ym index b6bf9d0f75a..1b187525711 100644 --- a/contrib/libs/tcmalloc/.yandex_meta/build.ym +++ b/contrib/libs/tcmalloc/.yandex_meta/build.ym @@ -17,6 +17,7 @@ no_percpu_cache/ya.make no_percpu_cache/aligned_alloc.c numa_256k/ya.make numa_large_pages/ya.make +profile_marshaller/ya.make small_but_slow/ya.make tcmalloc/internal/ya.make {% endblock %} @@ -49,6 +50,7 @@ IF (NOT DLL_FOR) no_percpu_cache numa_256k numa_large_pages + profile_marshaller small_but_slow tcmalloc/internal ) diff --git a/contrib/libs/tcmalloc/common.inc b/contrib/libs/tcmalloc/common.inc index 5acb463797d..c3609da7564 100644 --- a/contrib/libs/tcmalloc/common.inc +++ b/contrib/libs/tcmalloc/common.inc @@ -24,7 +24,6 @@ GLOBAL_SRCS( tcmalloc/pagemap.cc tcmalloc/parameters.cc tcmalloc/peak_heap_tracker.cc - # tcmalloc/profile_marshaler.cc tcmalloc/reuse_size_classes.cc tcmalloc/sampler.cc tcmalloc/segv_handler.cc @@ -57,7 +56,6 @@ GLOBAL_SRCS( tcmalloc/internal/percpu_rseq_unsupported.cc tcmalloc/internal/percpu_rseq_asm.S tcmalloc/internal/proc_maps.cc - # tcmalloc/internal/profile_builder.cc tcmalloc/internal/residency.cc tcmalloc/internal/sysinfo.cc tcmalloc/internal/util.cc diff --git a/contrib/libs/tcmalloc/patches/050-avoid-cycle.patch b/contrib/libs/tcmalloc/patches/050-avoid-cycle.patch deleted file mode 100644 index aa7d649be6a..00000000000 --- a/contrib/libs/tcmalloc/patches/050-avoid-cycle.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/tcmalloc/internal/profile_builder.h b/tcmalloc/internal/profile_builder.h -index 06f2abe..d50992e 100644 ---- a/tcmalloc/internal/profile_builder.h -+++ b/tcmalloc/internal/profile_builder.h -@@ -24,7 +24,7 @@ - #include <memory> - #include <string> - --#include "tcmalloc/internal/profile.pb.h" -+// #include "tcmalloc/internal/profile.pb.h" - #include "absl/container/btree_map.h" - #include "absl/container/flat_hash_map.h" - #include "absl/status/status.h" diff --git a/contrib/libs/tcmalloc/patches/050-proto-string-compat.patch b/contrib/libs/tcmalloc/patches/050-proto-string-compat.patch new file mode 100644 index 00000000000..b6508b6d272 --- /dev/null +++ b/contrib/libs/tcmalloc/patches/050-proto-string-compat.patch @@ -0,0 +1,21 @@ +diff --git a/tcmalloc/profile_marshaler.cc b/tcmalloc/profile_marshaler.cc +index dad14cc..3e008e1 100644 +--- a/tcmalloc/profile_marshaler.cc ++++ b/tcmalloc/profile_marshaler.cc +@@ -15,6 +15,7 @@ + #include "tcmalloc/profile_marshaler.h" + + #include <string> ++#include <util/generic/string.h> + + #include "google/protobuf/io/gzip_stream.h" + #include "google/protobuf/io/zero_copy_stream_impl.h" +@@ -31,7 +32,7 @@ absl::StatusOr<std::string> Marshal(const tcmalloc::Profile& profile) { + return converted_or.status(); + } + +- std::string output; ++ TString output; + google::protobuf::io::StringOutputStream stream(&output); + google::protobuf::io::GzipOutputStream gzip_stream(&stream); + if (!(*converted_or)->SerializeToZeroCopyStream(&gzip_stream)) { diff --git a/contrib/libs/tcmalloc/profile_marshaller/ya.make b/contrib/libs/tcmalloc/profile_marshaller/ya.make new file mode 100644 index 00000000000..e2aea48bfdc --- /dev/null +++ b/contrib/libs/tcmalloc/profile_marshaller/ya.make @@ -0,0 +1,32 @@ +LIBRARY() + +LICENSE(Apache-2.0) + +WITHOUT_LICENSE_TEXTS() + +VERSION(2025-01-30) + +NO_COMPILER_WARNINGS() + +SRCDIR(contrib/libs/tcmalloc/tcmalloc) + +PEERDIR( + contrib/libs/tcmalloc/tcmalloc/internal + contrib/restricted/abseil-cpp + contrib/libs/protobuf +) + +SRCS( + internal/profile_builder.cc + profile_marshaler.cc +) + +ADDINCL( + GLOBAL contrib/libs/tcmalloc +) + +ADDINCL( + ${ARCADIA_BUILD_ROOT}/contrib/libs/tcmalloc +) + +END() diff --git a/contrib/libs/tcmalloc/tcmalloc/internal/profile_builder.h b/contrib/libs/tcmalloc/tcmalloc/internal/profile_builder.h index c52843361d1..af4184d2b88 100644 --- a/contrib/libs/tcmalloc/tcmalloc/internal/profile_builder.h +++ b/contrib/libs/tcmalloc/tcmalloc/internal/profile_builder.h @@ -25,7 +25,7 @@ #include <memory> #include <string> -// #include "tcmalloc/internal/profile.pb.h" +#include "tcmalloc/internal/profile.pb.h" #include "absl/container/btree_map.h" #include "absl/container/flat_hash_map.h" #include "absl/status/status.h" diff --git a/contrib/libs/tcmalloc/tcmalloc/internal/ya.make b/contrib/libs/tcmalloc/tcmalloc/internal/ya.make index 36fd07fe348..786d44fc25e 100644 --- a/contrib/libs/tcmalloc/tcmalloc/internal/ya.make +++ b/contrib/libs/tcmalloc/tcmalloc/internal/ya.make @@ -14,8 +14,4 @@ EXCLUDE_TAGS( SRC(profile.proto) -ADDINCL( - GLOBAL contrib/libs/tcmalloc -) - END() diff --git a/contrib/libs/tcmalloc/tcmalloc/profile_marshaler.cc b/contrib/libs/tcmalloc/tcmalloc/profile_marshaler.cc index dad14cc93bc..3e008e14cd3 100644 --- a/contrib/libs/tcmalloc/tcmalloc/profile_marshaler.cc +++ b/contrib/libs/tcmalloc/tcmalloc/profile_marshaler.cc @@ -15,6 +15,7 @@ #include "tcmalloc/profile_marshaler.h" #include <string> +#include <util/generic/string.h> #include "google/protobuf/io/gzip_stream.h" #include "google/protobuf/io/zero_copy_stream_impl.h" @@ -31,7 +32,7 @@ absl::StatusOr<std::string> Marshal(const tcmalloc::Profile& profile) { return converted_or.status(); } - std::string output; + TString output; google::protobuf::io::StringOutputStream stream(&output); google::protobuf::io::GzipOutputStream gzip_stream(&stream); if (!(*converted_or)->SerializeToZeroCopyStream(&gzip_stream)) { diff --git a/contrib/libs/tcmalloc/ya.make b/contrib/libs/tcmalloc/ya.make index 4da2d7a78a3..fc6f3166a98 100644 --- a/contrib/libs/tcmalloc/ya.make +++ b/contrib/libs/tcmalloc/ya.make @@ -33,6 +33,7 @@ IF (NOT DLL_FOR) no_percpu_cache numa_256k numa_large_pages + profile_marshaller small_but_slow tcmalloc/internal ) diff --git a/contrib/libs/yajl/src/yajl_encode.c b/contrib/libs/yajl/src/yajl_encode.c index fd082581885..0d97cc5290c 100644 --- a/contrib/libs/yajl/src/yajl_encode.c +++ b/contrib/libs/yajl/src/yajl_encode.c @@ -139,8 +139,8 @@ void yajl_string_decode(yajl_buf buf, const unsigned char * str, end+=3; /* check if this is a surrogate */ if ((codepoint & 0xFC00) == 0xD800) { - end++; - if (str[end] == '\\' && str[end + 1] == 'u') { + if (end + 2 < len && str[end + 1] == '\\' && str[end + 2] == 'u') { + end++; unsigned int surrogate = 0; hexToDigit(&surrogate, str + end + 2); codepoint = diff --git a/contrib/python/fonttools/.dist-info/METADATA b/contrib/python/fonttools/.dist-info/METADATA index 98d95f11584..51b093cabdc 100644 --- a/contrib/python/fonttools/.dist-info/METADATA +++ b/contrib/python/fonttools/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.4 Name: fonttools -Version: 4.57.0 +Version: 4.58.0 Summary: Tools to manipulate font files Home-page: http://github.com/fonttools/fonttools Author: Just van Rossum @@ -14,11 +14,9 @@ Classifier: Environment :: Console Classifier: Environment :: Other Environment Classifier: Intended Audience :: Developers Classifier: Intended Audience :: End Users/Desktop -Classifier: License :: OSI Approved :: MIT License Classifier: Natural Language :: English Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 @@ -28,9 +26,10 @@ Classifier: Programming Language :: Python :: 3 Classifier: Topic :: Text Processing :: Fonts Classifier: Topic :: Multimedia :: Graphics Classifier: Topic :: Multimedia :: Graphics :: Graphics Conversion -Requires-Python: >=3.8 +Requires-Python: >=3.9 Description-Content-Type: text/x-rst License-File: LICENSE +License-File: LICENSE.external Provides-Extra: ufo Requires-Dist: fs<3,>=2.2.0; extra == "ufo" Provides-Extra: lxml @@ -98,7 +97,7 @@ What is this? fonts to and from an XML text format, which is also called TTX. It supports TrueType, OpenType, AFM and to an extent Type 1 and some Mac-specific formats. The project has an `MIT open-source - licence <LICENSE>`__. + license <LICENSE>`__. | Among other things this means you can use it free of charge. `User documentation <https://fonttools.readthedocs.io/en/latest/>`_ and @@ -108,7 +107,7 @@ are available at `Read the Docs <https://fonttools.readthedocs.io/>`_. Installation ~~~~~~~~~~~~ -FontTools requires `Python <http://www.python.org/download/>`__ 3.8 +FontTools requires `Python <http://www.python.org/download/>`__ 3.9 or later. We try to follow the same schedule of minimum Python version support as NumPy (see `NEP 29 <https://numpy.org/neps/nep-0029-deprecation_policy.html>`__). @@ -178,9 +177,6 @@ are required to unlock the extra features named "ufo", etc. * `fs <https://pypi.org/pypi/fs>`__: (aka ``pyfilesystem2``) filesystem abstraction layer. - * `enum34 <https://pypi.org/pypi/enum34>`__: backport for the built-in ``enum`` - module (only required on Python < 3.4). - *Extra:* ``ufo`` - ``Lib/fontTools/ttLib/woff2.py`` @@ -345,7 +341,7 @@ How to make a new release automate that too. -Acknowledgements +Acknowledgments ~~~~~~~~~~~~~~~~ In alphabetical order: @@ -356,7 +352,7 @@ Vincent Connare, David Corbett, Simon Cozens, Dave Crossland, Simon Daniels, Peter Dekkers, Behdad Esfahbod, Behnam Esfahbod, Hannes Famira, Sam Fishman, Matt Fontaine, Takaaki Fuji, Rob Hagemans, Yannis Haralambous, Greg Hitchcock, Jeremie Hornus, Khaled Hosny, John Hudson, Denis Moyogo Jacquerye, Jack Jansen, -Tom Kacvinsky, Jens Kutilek, Antoine Leca, Werner Lemberg, Tal Leming, Peter +Tom Kacvinsky, Jens Kutilek, Antoine Leca, Werner Lemberg, Tal Leming, Liang Hai, Peter Lofting, Cosimo Lupo, Olli Meier, Masaya Nakamura, Dave Opstad, Laurence Penney, Roozbeh Pournader, Garret Rieger, Read Roberts, Colin Rofls, Guido van Rossum, Just van Rossum, Andreas Seidel, Georg Seifert, Chris Simpkins, Miguel Sousa, @@ -392,6 +388,44 @@ Have fun! Changelog ~~~~~~~~~ +4.58.0 (released 2025-05-10) +---------------------------- + +- Drop Python 3.8, require 3.9+ (#3819) +- [HVAR, VVAR] Prune unused regions when using a direct mapping (#3797) +- [Docs] Improvements to ufoLib documentation (#3721) +- [Docs] Improvements to varLib documentation (#3727) +- [Docs] Improvements to Pens and pen-module documentation (#3724) +- [Docs] Miscellany updates to docs (misc modules and smaller modules) (#3730) +- [subset] Close codepoints over BiDi mirror variants. (#3801) +- [feaLib] Fix serializing ChainContextPosStatement and + ChainContextSubstStatement in some rare cases (#3788) +- [designspaceLib] Clarify user expectations for getStatNames (#2892) +- [GVAR] Add support for new `GVAR` table (#3728) +- [TSI0, TSI5] Derive number of entries to decompile from data length (#2477) +- [ttLib] Fix `AttributeError` when reporting table overflow (#3808) +- [ttLib] Apply rounding more often in getCoordinates (#3798) +- [ttLib] Ignore component bounds if empty (#3799) +- [ttLib] Change the separator for duplicate glyph names from "#" to "." (#3809) +- [feaLib] Support subtable breaks in CursivePos, MarkBasePos, MarkToLigPos and + MarkToMarkPos lookups (#3800, #3807) +- [feaLib] If the same lookup has single substitutions and ligature + substitutions, upgrade single substitutions to ligature substitutions with + one input glyph (#3805) +- [feaLib] Correctly handle <NULL> in single pos lookups (#3803) +- [feaLib] Remove duplicates from class pair pos classes instead of raising an + error (#3804) +- [feaLib] Support creating extension lookups using useExtenion lookup flag + instead of silently ignoring it (#3811) +- [STAT] Add typing for the simpler STAT arguments (#3812) +- [otlLib.builder] Add future import for annotations (#3814) +- [cffLib] Fix reading supplement encoding (#3813) +- [voltLib] Add some missing functionality and fixes to voltLib and VoltToFea, + making the conversion to feature files more robust. Add also `fonttools + voltLib` command line tool to compile VOLT sources directly (doing an + intermediate fea conversion internally) (#3818) +- [pens] Add some PointPen annotations (#3820) + 4.57.0 (released 2025-04-03) ---------------------------- diff --git a/contrib/python/fonttools/README.rst b/contrib/python/fonttools/README.rst index b604ea7ca59..e40554dae8f 100644 --- a/contrib/python/fonttools/README.rst +++ b/contrib/python/fonttools/README.rst @@ -8,7 +8,7 @@ What is this? fonts to and from an XML text format, which is also called TTX. It supports TrueType, OpenType, AFM and to an extent Type 1 and some Mac-specific formats. The project has an `MIT open-source - licence <LICENSE>`__. + license <LICENSE>`__. | Among other things this means you can use it free of charge. `User documentation <https://fonttools.readthedocs.io/en/latest/>`_ and @@ -18,7 +18,7 @@ are available at `Read the Docs <https://fonttools.readthedocs.io/>`_. Installation ~~~~~~~~~~~~ -FontTools requires `Python <http://www.python.org/download/>`__ 3.8 +FontTools requires `Python <http://www.python.org/download/>`__ 3.9 or later. We try to follow the same schedule of minimum Python version support as NumPy (see `NEP 29 <https://numpy.org/neps/nep-0029-deprecation_policy.html>`__). @@ -88,9 +88,6 @@ are required to unlock the extra features named "ufo", etc. * `fs <https://pypi.org/pypi/fs>`__: (aka ``pyfilesystem2``) filesystem abstraction layer. - * `enum34 <https://pypi.org/pypi/enum34>`__: backport for the built-in ``enum`` - module (only required on Python < 3.4). - *Extra:* ``ufo`` - ``Lib/fontTools/ttLib/woff2.py`` @@ -255,7 +252,7 @@ How to make a new release automate that too. -Acknowledgements +Acknowledgments ~~~~~~~~~~~~~~~~ In alphabetical order: @@ -266,7 +263,7 @@ Vincent Connare, David Corbett, Simon Cozens, Dave Crossland, Simon Daniels, Peter Dekkers, Behdad Esfahbod, Behnam Esfahbod, Hannes Famira, Sam Fishman, Matt Fontaine, Takaaki Fuji, Rob Hagemans, Yannis Haralambous, Greg Hitchcock, Jeremie Hornus, Khaled Hosny, John Hudson, Denis Moyogo Jacquerye, Jack Jansen, -Tom Kacvinsky, Jens Kutilek, Antoine Leca, Werner Lemberg, Tal Leming, Peter +Tom Kacvinsky, Jens Kutilek, Antoine Leca, Werner Lemberg, Tal Leming, Liang Hai, Peter Lofting, Cosimo Lupo, Olli Meier, Masaya Nakamura, Dave Opstad, Laurence Penney, Roozbeh Pournader, Garret Rieger, Read Roberts, Colin Rofls, Guido van Rossum, Just van Rossum, Andreas Seidel, Georg Seifert, Chris Simpkins, Miguel Sousa, diff --git a/contrib/python/fonttools/fontTools/__init__.py b/contrib/python/fonttools/fontTools/__init__.py index bc9f0e90559..6e41eb43a8b 100644 --- a/contrib/python/fonttools/fontTools/__init__.py +++ b/contrib/python/fonttools/fontTools/__init__.py @@ -3,6 +3,6 @@ from fontTools.misc.loggingTools import configLogger log = logging.getLogger(__name__) -version = __version__ = "4.57.0" +version = __version__ = "4.58.0" __all__ = ["version", "log", "configLogger"] diff --git a/contrib/python/fonttools/fontTools/cffLib/__init__.py b/contrib/python/fonttools/fontTools/cffLib/__init__.py index d75e23b750e..4ad724a27a8 100644 --- a/contrib/python/fonttools/fontTools/cffLib/__init__.py +++ b/contrib/python/fonttools/fontTools/cffLib/__init__.py @@ -1464,10 +1464,11 @@ class CharsetConverter(SimpleConverter): if glyphName in allNames: # make up a new glyphName that's unique n = allNames[glyphName] - while (glyphName + "#" + str(n)) in allNames: + names = set(allNames) | set(charset) + while (glyphName + "." + str(n)) in names: n += 1 allNames[glyphName] = n + 1 - glyphName = glyphName + "#" + str(n) + glyphName = glyphName + "." + str(n) allNames[glyphName] = 1 newCharset.append(glyphName) charset = newCharset @@ -1663,25 +1664,26 @@ class EncodingConverter(SimpleConverter): return "StandardEncoding" elif value == 1: return "ExpertEncoding" + # custom encoding at offset `value` + assert value > 1 + file = parent.file + file.seek(value) + log.log(DEBUG, "loading Encoding at %s", value) + fmt = readCard8(file) + haveSupplement = bool(fmt & 0x80) + fmt = fmt & 0x7F + + if fmt == 0: + encoding = parseEncoding0(parent.charset, file) + elif fmt == 1: + encoding = parseEncoding1(parent.charset, file) else: - assert value > 1 - file = parent.file - file.seek(value) - log.log(DEBUG, "loading Encoding at %s", value) - fmt = readCard8(file) - haveSupplement = fmt & 0x80 - if haveSupplement: - raise NotImplementedError("Encoding supplements are not yet supported") - fmt = fmt & 0x7F - if fmt == 0: - encoding = parseEncoding0( - parent.charset, file, haveSupplement, parent.strings - ) - elif fmt == 1: - encoding = parseEncoding1( - parent.charset, file, haveSupplement, parent.strings - ) - return encoding + raise ValueError(f"Unknown Encoding format: {fmt}") + + if haveSupplement: + parseEncodingSupplement(file, encoding, parent.strings) + + return encoding def write(self, parent, value): if value == "StandardEncoding": @@ -1719,27 +1721,60 @@ class EncodingConverter(SimpleConverter): return encoding -def parseEncoding0(charset, file, haveSupplement, strings): +def readSID(file): + """Read a String ID (SID) — 2-byte unsigned integer.""" + data = file.read(2) + if len(data) != 2: + raise EOFError("Unexpected end of file while reading SID") + return struct.unpack(">H", data)[0] # big-endian uint16 + + +def parseEncodingSupplement(file, encoding, strings): + """ + Parse the CFF Encoding supplement data: + - nSups: number of supplementary mappings + - each mapping: (code, SID) pair + and apply them to the `encoding` list in place. + """ + nSups = readCard8(file) + for _ in range(nSups): + code = readCard8(file) + sid = readSID(file) + name = strings[sid] + encoding[code] = name + + +def parseEncoding0(charset, file): + """ + Format 0: simple list of codes. + After reading the base table, optionally parse the supplement. + """ nCodes = readCard8(file) encoding = [".notdef"] * 256 for glyphID in range(1, nCodes + 1): code = readCard8(file) if code != 0: encoding[code] = charset[glyphID] + return encoding -def parseEncoding1(charset, file, haveSupplement, strings): +def parseEncoding1(charset, file): + """ + Format 1: range-based encoding. + After reading the base ranges, optionally parse the supplement. + """ nRanges = readCard8(file) encoding = [".notdef"] * 256 glyphID = 1 - for i in range(nRanges): + for _ in range(nRanges): code = readCard8(file) nLeft = readCard8(file) - for glyphID in range(glyphID, glyphID + nLeft + 1): + for _ in range(nLeft + 1): encoding[code] = charset[glyphID] - code = code + 1 - glyphID = glyphID + 1 + code += 1 + glyphID += 1 + return encoding diff --git a/contrib/python/fonttools/fontTools/designspaceLib/statNames.py b/contrib/python/fonttools/fontTools/designspaceLib/statNames.py index 1474e5fcf56..4e4f73470a2 100644 --- a/contrib/python/fonttools/fontTools/designspaceLib/statNames.py +++ b/contrib/python/fonttools/fontTools/designspaceLib/statNames.py @@ -12,14 +12,13 @@ instance: from __future__ import annotations from dataclasses import dataclass -from typing import Dict, Optional, Tuple, Union +from typing import Dict, Literal, Optional, Tuple, Union import logging from fontTools.designspaceLib import ( AxisDescriptor, AxisLabelDescriptor, DesignSpaceDocument, - DesignSpaceDocumentError, DiscreteAxisDescriptor, SimpleLocationDict, SourceDescriptor, @@ -27,9 +26,13 @@ from fontTools.designspaceLib import ( LOGGER = logging.getLogger(__name__) -# TODO(Python 3.8): use Literal -# RibbiStyleName = Union[Literal["regular"], Literal["bold"], Literal["italic"], Literal["bold italic"]] -RibbiStyle = str +RibbiStyleName = Union[ + Literal["regular"], + Literal["bold"], + Literal["italic"], + Literal["bold italic"], +] + BOLD_ITALIC_TO_RIBBI_STYLE = { (False, False): "regular", (False, True): "italic", @@ -46,7 +49,7 @@ class StatNames: styleNames: Dict[str, str] postScriptFontName: Optional[str] styleMapFamilyNames: Dict[str, str] - styleMapStyleName: Optional[RibbiStyle] + styleMapStyleName: Optional[RibbiStyleName] def getStatNames( @@ -61,6 +64,10 @@ def getStatNames( localized names will be empty (family and style names), or the name will be None (PostScript name). + Note: this method does not consider info attached to the instance, like + family name. The user needs to override all names on an instance that STAT + information would compute differently than desired. + .. versionadded:: 5.0 """ familyNames: Dict[str, str] = {} @@ -201,7 +208,7 @@ def _getAxisLabelsForUserLocation( def _getRibbiStyle( self: DesignSpaceDocument, userLocation: SimpleLocationDict -) -> Tuple[RibbiStyle, SimpleLocationDict]: +) -> Tuple[RibbiStyleName, SimpleLocationDict]: """Compute the RIBBI style name of the given user location, return the location of the matching Regular in the RIBBI group. diff --git a/contrib/python/fonttools/fontTools/feaLib/ast.py b/contrib/python/fonttools/fontTools/feaLib/ast.py index 10c49058c45..8479d7300d6 100644 --- a/contrib/python/fonttools/fontTools/feaLib/ast.py +++ b/contrib/python/fonttools/fontTools/feaLib/ast.py @@ -337,6 +337,76 @@ class AnonymousBlock(Statement): return res +def _upgrade_mixed_subst_statements(statements): + # https://github.com/fonttools/fonttools/issues/612 + # A multiple substitution may have a single destination, in which case + # it will look just like a single substitution. So if there are both + # multiple and single substitutions, upgrade all the single ones to + # multiple substitutions. Similarly, a ligature substitution may have a + # single source glyph, so if there are both ligature and single + # substitutions, upgrade all the single ones to ligature substitutions. + + has_single = False + has_multiple = False + has_ligature = False + for s in statements: + if isinstance(s, SingleSubstStatement): + has_single = not any([s.prefix, s.suffix, s.forceChain]) + elif isinstance(s, MultipleSubstStatement): + has_multiple = not any([s.prefix, s.suffix, s.forceChain]) + elif isinstance(s, LigatureSubstStatement): + has_ligature = not any([s.prefix, s.suffix, s.forceChain]) + + to_multiple = False + to_ligature = False + + # If we have mixed single and multiple substitutions, + # upgrade all single substitutions to multiple substitutions. + if has_single and has_multiple and not has_ligature: + to_multiple = True + + # If we have mixed single and ligature substitutions, + # upgrade all single substitutions to ligature substitutions. + elif has_single and has_ligature and not has_multiple: + to_ligature = True + + if to_multiple or to_ligature: + ret = [] + for s in statements: + if isinstance(s, SingleSubstStatement): + glyphs = s.glyphs[0].glyphSet() + replacements = s.replacements[0].glyphSet() + if len(replacements) == 1: + replacements *= len(glyphs) + for glyph, replacement in zip(glyphs, replacements): + if to_multiple: + ret.append( + MultipleSubstStatement( + s.prefix, + glyph, + s.suffix, + [replacement], + s.forceChain, + location=s.location, + ) + ) + elif to_ligature: + ret.append( + LigatureSubstStatement( + s.prefix, + [GlyphName(glyph)], + s.suffix, + replacement, + s.forceChain, + location=s.location, + ) + ) + else: + ret.append(s) + return ret + return statements + + class Block(Statement): """A block of statements: feature, lookup, etc.""" @@ -348,7 +418,8 @@ class Block(Statement): """When handed a 'builder' object of comparable interface to :class:`fontTools.feaLib.builder`, walks the statements in this block, calling the builder callbacks.""" - for s in self.statements: + statements = _upgrade_mixed_subst_statements(self.statements) + for s in statements: s.build(builder) def asFea(self, indent=""): @@ -382,8 +453,7 @@ class FeatureBlock(Block): def build(self, builder): """Call the ``start_feature`` callback on the builder object, visit all the statements in this feature, and then call ``end_feature``.""" - # TODO(sascha): Handle use_extension. - builder.start_feature(self.location, self.name) + builder.start_feature(self.location, self.name, self.use_extension) # language exclude_dflt statements modify builder.features_ # limit them to this block with temporary builder.features_ features = builder.features_ @@ -433,8 +503,7 @@ class LookupBlock(Block): self.name, self.use_extension = name, use_extension def build(self, builder): - # TODO(sascha): Handle use_extension. - builder.start_lookup_block(self.location, self.name) + builder.start_lookup_block(self.location, self.name, self.use_extension) Block.build(self, builder) builder.end_lookup_block() @@ -753,7 +822,7 @@ class ChainContextPosStatement(Statement): if len(self.suffix): res += " " + " ".join(map(asFea, self.suffix)) else: - res += " ".join(map(asFea, self.glyph)) + res += " ".join(map(asFea, self.glyphs)) res += ";" return res @@ -811,7 +880,7 @@ class ChainContextSubstStatement(Statement): if len(self.suffix): res += " " + " ".join(map(asFea, self.suffix)) else: - res += " ".join(map(asFea, self.glyph)) + res += " ".join(map(asFea, self.glyphs)) res += ";" return res @@ -1512,7 +1581,9 @@ class SinglePosStatement(Statement): res += " ".join(map(asFea, self.prefix)) + " " res += " ".join( [ - asFea(x[0]) + "'" + ((" " + x[1].asFea()) if x[1] else "") + asFea(x[0]) + + "'" + + ((" " + x[1].asFea()) if x[1] is not None else "") for x in self.pos ] ) @@ -1520,7 +1591,10 @@ class SinglePosStatement(Statement): res += " " + " ".join(map(asFea, self.suffix)) else: res += " ".join( - [asFea(x[0]) + " " + (x[1].asFea() if x[1] else "") for x in self.pos] + [ + asFea(x[0]) + " " + (x[1].asFea() if x[1] is not None else "") + for x in self.pos + ] ) res += ";" return res @@ -2103,7 +2177,7 @@ class VariationBlock(Block): def build(self, builder): """Call the ``start_feature`` callback on the builder object, visit all the statements in this feature, and then call ``end_feature``.""" - builder.start_feature(self.location, self.name) + builder.start_feature(self.location, self.name, self.use_extension) if ( self.conditionset != "NULL" and self.conditionset not in builder.conditionsets_ diff --git a/contrib/python/fonttools/fontTools/feaLib/builder.py b/contrib/python/fonttools/fontTools/feaLib/builder.py index 8b2c7208b29..1583f06d9ec 100644 --- a/contrib/python/fonttools/fontTools/feaLib/builder.py +++ b/contrib/python/fonttools/fontTools/feaLib/builder.py @@ -126,6 +126,7 @@ class Builder(object): self.script_ = None self.lookupflag_ = 0 self.lookupflag_markFilterSet_ = None + self.use_extension_ = False self.language_systems = set() self.seen_non_DFLT_script_ = False self.named_lookups_ = {} @@ -141,6 +142,7 @@ class Builder(object): self.aalt_features_ = [] # [(location, featureName)*], for 'aalt' self.aalt_location_ = None self.aalt_alternates_ = {} + self.aalt_use_extension_ = False # for 'featureNames' self.featureNames_ = set() self.featureNames_ids_ = {} @@ -247,6 +249,7 @@ class Builder(object): result = builder_class(self.font, location) result.lookupflag = self.lookupflag_ result.markFilterSet = self.lookupflag_markFilterSet_ + result.extension = self.use_extension_ self.lookups_.append(result) return result @@ -272,6 +275,7 @@ class Builder(object): self.cur_lookup_ = builder_class(self.font, location) self.cur_lookup_.lookupflag = self.lookupflag_ self.cur_lookup_.markFilterSet = self.lookupflag_markFilterSet_ + self.cur_lookup_.extension = self.use_extension_ self.lookups_.append(self.cur_lookup_) if self.cur_lookup_name_: # We are starting a lookup rule inside a named lookup block. @@ -323,7 +327,7 @@ class Builder(object): } old_lookups = self.lookups_ self.lookups_ = [] - self.start_feature(self.aalt_location_, "aalt") + self.start_feature(self.aalt_location_, "aalt", self.aalt_use_extension_) if single: single_lookup = self.get_lookup_(location, SingleSubstBuilder) single_lookup.mapping = single @@ -1054,15 +1058,22 @@ class Builder(object): else: return frozenset({("DFLT", "dflt")}) - def start_feature(self, location, name): + def start_feature(self, location, name, use_extension=False): + if use_extension and name != "aalt": + raise FeatureLibError( + "'useExtension' keyword for feature blocks is allowed only for 'aalt' feature", + location, + ) self.language_systems = self.get_default_language_systems_() self.script_ = "DFLT" self.cur_lookup_ = None self.cur_feature_name_ = name self.lookupflag_ = 0 self.lookupflag_markFilterSet_ = None + self.use_extension_ = use_extension if name == "aalt": self.aalt_location_ = location + self.aalt_use_extension_ = use_extension def end_feature(self): assert self.cur_feature_name_ is not None @@ -1071,8 +1082,9 @@ class Builder(object): self.cur_lookup_ = None self.lookupflag_ = 0 self.lookupflag_markFilterSet_ = None + self.use_extension_ = False - def start_lookup_block(self, location, name): + def start_lookup_block(self, location, name, use_extension=False): if name in self.named_lookups_: raise FeatureLibError( 'Lookup "%s" has already been defined' % name, location @@ -1086,6 +1098,7 @@ class Builder(object): self.cur_lookup_name_ = name self.named_lookups_[name] = None self.cur_lookup_ = None + self.use_extension_ = use_extension if self.cur_feature_name_ is None: self.lookupflag_ = 0 self.lookupflag_markFilterSet_ = None @@ -1094,6 +1107,7 @@ class Builder(object): assert self.cur_lookup_name_ is not None self.cur_lookup_name_ = None self.cur_lookup_ = None + self.use_extension_ = False if self.cur_feature_name_ is None: self.lookupflag_ = 0 self.lookupflag_markFilterSet_ = None @@ -1471,7 +1485,9 @@ class Builder(object): lookup = self.get_lookup_(location, PairPosBuilder) v1 = self.makeOpenTypeValueRecord(location, value1, pairPosContext=True) v2 = self.makeOpenTypeValueRecord(location, value2, pairPosContext=True) - lookup.addClassPair(location, glyphclass1, v1, glyphclass2, v2) + cls1 = tuple(sorted(set(glyphclass1))) + cls2 = tuple(sorted(set(glyphclass2))) + lookup.addClassPair(location, cls1, v1, cls2, v2) def add_specific_pair_pos(self, location, glyph1, value1, glyph2, value2): if not glyph1 or not glyph2: diff --git a/contrib/python/fonttools/fontTools/feaLib/parser.py b/contrib/python/fonttools/fontTools/feaLib/parser.py index 5f647ca0acd..451dd624119 100644 --- a/contrib/python/fonttools/fontTools/feaLib/parser.py +++ b/contrib/python/fonttools/fontTools/feaLib/parser.py @@ -1613,7 +1613,7 @@ class Parser(object): "HorizAxis.BaseScriptList", "VertAxis.BaseScriptList", ), self.cur_token_ - scripts = [(self.parse_base_script_record_(count))] + scripts = [self.parse_base_script_record_(count)] while self.next_token_ == ",": self.expect_symbol_(",") scripts.append(self.parse_base_script_record_(count)) @@ -2062,44 +2062,6 @@ class Parser(object): ) self.expect_symbol_(";") - # A multiple substitution may have a single destination, in which case - # it will look just like a single substitution. So if there are both - # multiple and single substitutions, upgrade all the single ones to - # multiple substitutions. - - # Check if we have a mix of non-contextual singles and multiples. - has_single = False - has_multiple = False - for s in statements: - if isinstance(s, self.ast.SingleSubstStatement): - has_single = not any([s.prefix, s.suffix, s.forceChain]) - elif isinstance(s, self.ast.MultipleSubstStatement): - has_multiple = not any([s.prefix, s.suffix, s.forceChain]) - - # Upgrade all single substitutions to multiple substitutions. - if has_single and has_multiple: - statements = [] - for s in block.statements: - if isinstance(s, self.ast.SingleSubstStatement): - glyphs = s.glyphs[0].glyphSet() - replacements = s.replacements[0].glyphSet() - if len(replacements) == 1: - replacements *= len(glyphs) - for i, glyph in enumerate(glyphs): - statements.append( - self.ast.MultipleSubstStatement( - s.prefix, - glyph, - s.suffix, - [replacements[i]], - s.forceChain, - location=s.location, - ) - ) - else: - statements.append(s) - block.statements = statements - def is_cur_keyword_(self, k): if self.cur_token_type_ is Lexer.NAME: if isinstance(k, type("")): # basestring is gone in Python3 diff --git a/contrib/python/fonttools/fontTools/fontBuilder.py b/contrib/python/fonttools/fontTools/fontBuilder.py index d4af38fba48..f8da717babb 100644 --- a/contrib/python/fonttools/fontTools/fontBuilder.py +++ b/contrib/python/fonttools/fontTools/fontBuilder.py @@ -714,6 +714,12 @@ class FontBuilder(object): gvar.reserved = 0 gvar.variations = variations + def setupGVAR(self, variations): + gvar = self.font["GVAR"] = newTable("GVAR") + gvar.version = 1 + gvar.reserved = 0 + gvar.variations = variations + def calcGlyphBounds(self): """Calculate the bounding boxes of all glyphs in the `glyf` table. This is usually not called explicitly by client code. diff --git a/contrib/python/fonttools/fontTools/misc/etree.py b/contrib/python/fonttools/fontTools/misc/etree.py index d0967b5f52f..743546061c4 100644 --- a/contrib/python/fonttools/fontTools/misc/etree.py +++ b/contrib/python/fonttools/fontTools/misc/etree.py @@ -56,21 +56,7 @@ except ImportError: from xml.etree.ElementTree import * _have_lxml = False - import sys - - # dict is always ordered in python >= 3.6 and on pypy - PY36 = sys.version_info >= (3, 6) - try: - import __pypy__ - except ImportError: - __pypy__ = None - _dict_is_ordered = bool(PY36 or __pypy__) - del PY36, __pypy__ - - if _dict_is_ordered: - _Attrib = dict - else: - from collections import OrderedDict as _Attrib + _Attrib = dict if isinstance(Element, type): _Element = Element @@ -221,18 +207,9 @@ except ImportError: # characters, the surrogate blocks, FFFE, and FFFF: # Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] # Here we reversed the pattern to match only the invalid characters. - # For the 'narrow' python builds supporting only UCS-2, which represent - # characters beyond BMP as UTF-16 surrogate pairs, we need to pass through - # the surrogate block. I haven't found a more elegant solution... - UCS2 = sys.maxunicode < 0x10FFFF - if UCS2: - _invalid_xml_string = re.compile( - "[\u0000-\u0008\u000B-\u000C\u000E-\u001F\uFFFE-\uFFFF]" - ) - else: - _invalid_xml_string = re.compile( - "[\u0000-\u0008\u000B-\u000C\u000E-\u001F\uD800-\uDFFF\uFFFE-\uFFFF]" - ) + _invalid_xml_string = re.compile( + "[\u0000-\u0008\u000B-\u000C\u000E-\u001F\uD800-\uDFFF\uFFFE-\uFFFF]" + ) def _tounicode(s): """Test if a string is valid user input and decode it to unicode string diff --git a/contrib/python/fonttools/fontTools/mtiLib/__init__.py b/contrib/python/fonttools/fontTools/mtiLib/__init__.py index dbedf275e3d..e797be375b6 100644 --- a/contrib/python/fonttools/fontTools/mtiLib/__init__.py +++ b/contrib/python/fonttools/fontTools/mtiLib/__init__.py @@ -1,5 +1,3 @@ -#!/usr/bin/python - # FontDame-to-FontTools for OpenType Layout tables # # Source language spec is available at: diff --git a/contrib/python/fonttools/fontTools/otlLib/builder.py b/contrib/python/fonttools/fontTools/otlLib/builder.py index b944ea8c261..064b2fce31c 100644 --- a/contrib/python/fonttools/fontTools/otlLib/builder.py +++ b/contrib/python/fonttools/fontTools/otlLib/builder.py @@ -1,6 +1,8 @@ +from __future__ import annotations + from collections import namedtuple, OrderedDict import itertools -import os +from typing import Dict, Union from fontTools.misc.fixedTools import fixedToFloat from fontTools.misc.roundTools import otRound from fontTools import ttLib @@ -10,15 +12,15 @@ from fontTools.ttLib.tables.otBase import ( valueRecordFormatDict, OTLOffsetOverflowError, OTTableWriter, - CountReference, ) -from fontTools.ttLib.tables import otBase +from fontTools.ttLib.ttFont import TTFont from fontTools.feaLib.ast import STATNameStatement from fontTools.otlLib.optimize.gpos import ( _compression_level_from_env, compact_lookup, ) from fontTools.otlLib.error import OpenTypeLibError +from fontTools.misc.loggingTools import deprecateFunction from functools import reduce import logging import copy @@ -73,7 +75,7 @@ LOOKUP_FLAG_IGNORE_MARKS = 0x0008 LOOKUP_FLAG_USE_MARK_FILTERING_SET = 0x0010 -def buildLookup(subtables, flags=0, markFilterSet=None): +def buildLookup(subtables, flags=0, markFilterSet=None, table=None, extension=False): """Turns a collection of rules into a lookup. A Lookup (as defined in the `OpenType Spec <https://docs.microsoft.com/en-gb/typography/opentype/spec/chapter2#lookupTbl>`__) @@ -98,6 +100,8 @@ def buildLookup(subtables, flags=0, markFilterSet=None): lookup. If a mark filtering set is provided, `LOOKUP_FLAG_USE_MARK_FILTERING_SET` will be set on the lookup's flags. + table (str): The name of the table this lookup belongs to, e.g. "GPOS" or "GSUB". + extension (bool): ``True`` if this is an extension lookup, ``False`` otherwise. Returns: An ``otTables.Lookup`` object or ``None`` if there are no subtables @@ -113,8 +117,21 @@ def buildLookup(subtables, flags=0, markFilterSet=None): ), "all subtables must have the same LookupType; got %s" % repr( [t.LookupType for t in subtables] ) + + if extension: + assert table in ("GPOS", "GSUB") + lookupType = 7 if table == "GSUB" else 9 + extSubTableClass = ot.lookupTypes[table][lookupType] + for i, st in enumerate(subtables): + subtables[i] = extSubTableClass() + subtables[i].Format = 1 + subtables[i].ExtSubTable = st + subtables[i].ExtensionLookupType = st.LookupType + else: + lookupType = subtables[0].LookupType + self = ot.Lookup() - self.LookupType = subtables[0].LookupType + self.LookupType = lookupType self.LookupFlag = flags self.SubTable = subtables self.SubTableCount = len(self.SubTable) @@ -133,7 +150,7 @@ def buildLookup(subtables, flags=0, markFilterSet=None): class LookupBuilder(object): SUBTABLE_BREAK_ = "SUBTABLE_BREAK" - def __init__(self, font, location, table, lookup_type): + def __init__(self, font, location, table, lookup_type, extension=False): self.font = font self.glyphMap = font.getReverseGlyphMap() self.location = location @@ -141,6 +158,7 @@ class LookupBuilder(object): self.lookupflag = 0 self.markFilterSet = None self.lookup_index = None # assigned when making final tables + self.extension = extension assert table in ("GPOS", "GSUB") def equals(self, other): @@ -149,6 +167,7 @@ class LookupBuilder(object): and self.table == other.table and self.lookupflag == other.lookupflag and self.markFilterSet == other.markFilterSet + and self.extension == other.extension ) def inferGlyphClasses(self): @@ -160,7 +179,13 @@ class LookupBuilder(object): return {} def buildLookup_(self, subtables): - return buildLookup(subtables, self.lookupflag, self.markFilterSet) + return buildLookup( + subtables, + self.lookupflag, + self.markFilterSet, + self.table, + self.extension, + ) def buildMarkClasses_(self, marks): """{"cedilla": ("BOTTOM", ast.Anchor), ...} --> {"BOTTOM":0, "TOP":1} @@ -949,8 +974,20 @@ class CursivePosBuilder(LookupBuilder): An ``otTables.Lookup`` object representing the cursive positioning lookup. """ - st = buildCursivePosSubtable(self.attachments, self.glyphMap) - return self.buildLookup_([st]) + attachments = [{}] + for key in self.attachments: + if key[0] == self.SUBTABLE_BREAK_: + attachments.append({}) + else: + attachments[-1][key] = self.attachments[key] + subtables = [buildCursivePosSubtable(s, self.glyphMap) for s in attachments] + return self.buildLookup_(subtables) + + def add_subtable_break(self, location): + self.attachments[(self.SUBTABLE_BREAK_, location)] = ( + self.SUBTABLE_BREAK_, + self.SUBTABLE_BREAK_, + ) class MarkBasePosBuilder(LookupBuilder): @@ -985,17 +1022,25 @@ class MarkBasePosBuilder(LookupBuilder): LookupBuilder.__init__(self, font, location, "GPOS", 4) self.marks = {} # glyphName -> (markClassName, anchor) self.bases = {} # glyphName -> {markClassName: anchor} + self.subtables_ = [] + + def get_subtables_(self): + subtables_ = self.subtables_ + if self.bases or self.marks: + subtables_.append((self.marks, self.bases)) + return subtables_ def equals(self, other): return ( LookupBuilder.equals(self, other) - and self.marks == other.marks - and self.bases == other.bases + and self.get_subtables_() == other.get_subtables_() ) def inferGlyphClasses(self): - result = {glyph: 1 for glyph in self.bases} - result.update({glyph: 3 for glyph in self.marks}) + result = {} + for marks, bases in self.get_subtables_(): + result.update({glyph: 1 for glyph in bases}) + result.update({glyph: 3 for glyph in marks}) return result def build(self): @@ -1005,26 +1050,33 @@ class MarkBasePosBuilder(LookupBuilder): An ``otTables.Lookup`` object representing the mark-to-base positioning lookup. """ - markClasses = self.buildMarkClasses_(self.marks) - marks = {} - for mark, (mc, anchor) in self.marks.items(): - if mc not in markClasses: - raise ValueError( - "Mark class %s not found for mark glyph %s" % (mc, mark) - ) - marks[mark] = (markClasses[mc], anchor) - bases = {} - for glyph, anchors in self.bases.items(): - bases[glyph] = {} - for mc, anchor in anchors.items(): + subtables = [] + for subtable in self.get_subtables_(): + markClasses = self.buildMarkClasses_(subtable[0]) + marks = {} + for mark, (mc, anchor) in subtable[0].items(): if mc not in markClasses: raise ValueError( - "Mark class %s not found for base glyph %s" % (mc, glyph) + "Mark class %s not found for mark glyph %s" % (mc, mark) ) - bases[glyph][markClasses[mc]] = anchor - subtables = buildMarkBasePos(marks, bases, self.glyphMap) + marks[mark] = (markClasses[mc], anchor) + bases = {} + for glyph, anchors in subtable[1].items(): + bases[glyph] = {} + for mc, anchor in anchors.items(): + if mc not in markClasses: + raise ValueError( + "Mark class %s not found for base glyph %s" % (mc, glyph) + ) + bases[glyph][markClasses[mc]] = anchor + subtables.append(buildMarkBasePosSubtable(marks, bases, self.glyphMap)) return self.buildLookup_(subtables) + def add_subtable_break(self, location): + self.subtables_.append((self.marks, self.bases)) + self.marks = {} + self.bases = {} + class MarkLigPosBuilder(LookupBuilder): """Builds a Mark-To-Ligature Positioning (GPOS5) lookup. @@ -1061,17 +1113,25 @@ class MarkLigPosBuilder(LookupBuilder): LookupBuilder.__init__(self, font, location, "GPOS", 5) self.marks = {} # glyphName -> (markClassName, anchor) self.ligatures = {} # glyphName -> [{markClassName: anchor}, ...] + self.subtables_ = [] + + def get_subtables_(self): + subtables_ = self.subtables_ + if self.ligatures or self.marks: + subtables_.append((self.marks, self.ligatures)) + return subtables_ def equals(self, other): return ( LookupBuilder.equals(self, other) - and self.marks == other.marks - and self.ligatures == other.ligatures + and self.get_subtables_() == other.get_subtables_() ) def inferGlyphClasses(self): - result = {glyph: 2 for glyph in self.ligatures} - result.update({glyph: 3 for glyph in self.marks}) + result = {} + for marks, ligatures in self.get_subtables_(): + result.update({glyph: 2 for glyph in ligatures}) + result.update({glyph: 3 for glyph in marks}) return result def build(self): @@ -1081,18 +1141,26 @@ class MarkLigPosBuilder(LookupBuilder): An ``otTables.Lookup`` object representing the mark-to-ligature positioning lookup. """ - markClasses = self.buildMarkClasses_(self.marks) - marks = { - mark: (markClasses[mc], anchor) for mark, (mc, anchor) in self.marks.items() - } - ligs = {} - for lig, components in self.ligatures.items(): - ligs[lig] = [] - for c in components: - ligs[lig].append({markClasses[mc]: a for mc, a in c.items()}) - subtables = buildMarkLigPos(marks, ligs, self.glyphMap) + subtables = [] + for subtable in self.get_subtables_(): + markClasses = self.buildMarkClasses_(subtable[0]) + marks = { + mark: (markClasses[mc], anchor) + for mark, (mc, anchor) in subtable[0].items() + } + ligs = {} + for lig, components in subtable[1].items(): + ligs[lig] = [] + for c in components: + ligs[lig].append({markClasses[mc]: a for mc, a in c.items()}) + subtables.append(buildMarkLigPosSubtable(marks, ligs, self.glyphMap)) return self.buildLookup_(subtables) + def add_subtable_break(self, location): + self.subtables_.append((self.marks, self.ligatures)) + self.marks = {} + self.ligatures = {} + class MarkMarkPosBuilder(LookupBuilder): """Builds a Mark-To-Mark Positioning (GPOS6) lookup. @@ -1125,17 +1193,25 @@ class MarkMarkPosBuilder(LookupBuilder): LookupBuilder.__init__(self, font, location, "GPOS", 6) self.marks = {} # glyphName -> (markClassName, anchor) self.baseMarks = {} # glyphName -> {markClassName: anchor} + self.subtables_ = [] + + def get_subtables_(self): + subtables_ = self.subtables_ + if self.baseMarks or self.marks: + subtables_.append((self.marks, self.baseMarks)) + return subtables_ def equals(self, other): return ( LookupBuilder.equals(self, other) - and self.marks == other.marks - and self.baseMarks == other.baseMarks + and self.get_subtables_() == other.get_subtables_() ) def inferGlyphClasses(self): - result = {glyph: 3 for glyph in self.baseMarks} - result.update({glyph: 3 for glyph in self.marks}) + result = {} + for marks, baseMarks in self.get_subtables_(): + result.update({glyph: 3 for glyph in baseMarks}) + result.update({glyph: 3 for glyph in marks}) return result def build(self): @@ -1145,25 +1221,34 @@ class MarkMarkPosBuilder(LookupBuilder): An ``otTables.Lookup`` object representing the mark-to-mark positioning lookup. """ - markClasses = self.buildMarkClasses_(self.marks) - markClassList = sorted(markClasses.keys(), key=markClasses.get) - marks = { - mark: (markClasses[mc], anchor) for mark, (mc, anchor) in self.marks.items() - } + subtables = [] + for subtable in self.get_subtables_(): + markClasses = self.buildMarkClasses_(subtable[0]) + markClassList = sorted(markClasses.keys(), key=markClasses.get) + marks = { + mark: (markClasses[mc], anchor) + for mark, (mc, anchor) in subtable[0].items() + } + + st = ot.MarkMarkPos() + st.Format = 1 + st.ClassCount = len(markClasses) + st.Mark1Coverage = buildCoverage(marks, self.glyphMap) + st.Mark2Coverage = buildCoverage(subtable[1], self.glyphMap) + st.Mark1Array = buildMarkArray(marks, self.glyphMap) + st.Mark2Array = ot.Mark2Array() + st.Mark2Array.Mark2Count = len(st.Mark2Coverage.glyphs) + st.Mark2Array.Mark2Record = [] + for base in st.Mark2Coverage.glyphs: + anchors = [subtable[1][base].get(mc) for mc in markClassList] + st.Mark2Array.Mark2Record.append(buildMark2Record(anchors)) + subtables.append(st) + return self.buildLookup_(subtables) - st = ot.MarkMarkPos() - st.Format = 1 - st.ClassCount = len(markClasses) - st.Mark1Coverage = buildCoverage(marks, self.glyphMap) - st.Mark2Coverage = buildCoverage(self.baseMarks, self.glyphMap) - st.Mark1Array = buildMarkArray(marks, self.glyphMap) - st.Mark2Array = ot.Mark2Array() - st.Mark2Array.Mark2Count = len(st.Mark2Coverage.glyphs) - st.Mark2Array.Mark2Record = [] - for base in st.Mark2Coverage.glyphs: - anchors = [self.baseMarks[base].get(mc) for mc in markClassList] - st.Mark2Array.Mark2Record.append(buildMark2Record(anchors)) - return self.buildLookup_([st]) + def add_subtable_break(self, location): + self.subtables_.append((self.marks, self.baseMarks)) + self.marks = {} + self.baseMarks = {} class ReverseChainSingleSubstBuilder(LookupBuilder): @@ -1484,6 +1569,8 @@ class SinglePosBuilder(LookupBuilder): otValueRection: A ``otTables.ValueRecord`` used to position the glyph. """ + if otValueRecord is None: + otValueRecord = ValueRecord() if not self.can_add(glyph, otValueRecord): otherLoc = self.locations[glyph] raise OpenTypeLibError( @@ -1900,53 +1987,15 @@ def buildMarkArray(marks, glyphMap): return self +@deprecateFunction( + "use buildMarkBasePosSubtable() instead", category=DeprecationWarning +) def buildMarkBasePos(marks, bases, glyphMap): """Build a list of MarkBasePos (GPOS4) subtables. - This routine turns a set of marks and bases into a list of mark-to-base - positioning subtables. Currently the list will contain a single subtable - containing all marks and bases, although at a later date it may return the - optimal list of subtables subsetting the marks and bases into groups which - save space. See :func:`buildMarkBasePosSubtable` below. - - Note that if you are implementing a layout compiler, you may find it more - flexible to use - :py:class:`fontTools.otlLib.lookupBuilders.MarkBasePosBuilder` instead. - - Example:: - - # a1, a2, a3, a4, a5 = buildAnchor(500, 100), ... - - marks = {"acute": (0, a1), "grave": (0, a1), "cedilla": (1, a2)} - bases = {"a": {0: a3, 1: a5}, "b": {0: a4, 1: a5}} - markbaseposes = buildMarkBasePos(marks, bases, font.getReverseGlyphMap()) - - Args: - marks (dict): A dictionary mapping anchors to glyphs; the keys being - glyph names, and the values being a tuple of mark class number and - an ``otTables.Anchor`` object representing the mark's attachment - point. (See :func:`buildMarkArray`.) - bases (dict): A dictionary mapping anchors to glyphs; the keys being - glyph names, and the values being dictionaries mapping mark class ID - to the appropriate ``otTables.Anchor`` object used for attaching marks - of that class. (See :func:`buildBaseArray`.) - glyphMap: a glyph name to ID map, typically returned from - ``font.getReverseGlyphMap()``. - - Returns: - A list of ``otTables.MarkBasePos`` objects. + .. deprecated:: 4.58.0 + Use :func:`buildMarkBasePosSubtable` instead. """ - # TODO: Consider emitting multiple subtables to save space. - # Partition the marks and bases into disjoint subsets, so that - # MarkBasePos rules would only access glyphs from a single - # subset. This would likely lead to smaller mark/base - # matrices, so we might be able to omit many of the empty - # anchor tables that we currently produce. Of course, this - # would only work if the MarkBasePos rules of real-world fonts - # allow partitioning into multiple subsets. We should find out - # whether this is the case; if so, implement the optimization. - # On the other hand, a very large number of subtables could - # slow down layout engines; so this would need profiling. return [buildMarkBasePosSubtable(marks, bases, glyphMap)] @@ -1954,7 +2003,15 @@ def buildMarkBasePosSubtable(marks, bases, glyphMap): """Build a single MarkBasePos (GPOS4) subtable. This builds a mark-to-base lookup subtable containing all of the referenced - marks and bases. See :func:`buildMarkBasePos`. + marks and bases. + + Example:: + + # a1, a2, a3, a4, a5 = buildAnchor(500, 100), ... + + marks = {"acute": (0, a1), "grave": (0, a1), "cedilla": (1, a2)} + bases = {"a": {0: a3, 1: a5}, "b": {0: a4, 1: a5}} + markbaseposes = [buildMarkBasePosSubtable(marks, bases, font.getReverseGlyphMap())] Args: marks (dict): A dictionary mapping anchors to glyphs; the keys being @@ -1981,14 +2038,21 @@ def buildMarkBasePosSubtable(marks, bases, glyphMap): return self +@deprecateFunction("use buildMarkLigPosSubtable() instead", category=DeprecationWarning) def buildMarkLigPos(marks, ligs, glyphMap): """Build a list of MarkLigPos (GPOS5) subtables. - This routine turns a set of marks and ligatures into a list of mark-to-ligature - positioning subtables. Currently the list will contain a single subtable - containing all marks and ligatures, although at a later date it may return - the optimal list of subtables subsetting the marks and ligatures into groups - which save space. See :func:`buildMarkLigPosSubtable` below. + .. deprecated:: 4.58.0 + Use :func:`buildMarkLigPosSubtable` instead. + """ + return [buildMarkLigPosSubtable(marks, ligs, glyphMap)] + + +def buildMarkLigPosSubtable(marks, ligs, glyphMap): + """Build a single MarkLigPos (GPOS5) subtable. + + This builds a mark-to-base lookup subtable containing all of the referenced + marks and bases. Note that if you are implementing a layout compiler, you may find it more flexible to use @@ -2009,7 +2073,7 @@ def buildMarkLigPos(marks, ligs, glyphMap): ], # "c_t": [{...}, {...}] } - markligposes = buildMarkLigPos(marks, ligs, + markligpose = buildMarkLigPosSubtable(marks, ligs, font.getReverseGlyphMap()) Args: @@ -2024,34 +2088,6 @@ def buildMarkLigPos(marks, ligs, glyphMap): ``font.getReverseGlyphMap()``. Returns: - A list of ``otTables.MarkLigPos`` objects. - - """ - # TODO: Consider splitting into multiple subtables to save space, - # as with MarkBasePos, this would be a trade-off that would need - # profiling. And, depending on how typical fonts are structured, - # it might not be worth doing at all. - return [buildMarkLigPosSubtable(marks, ligs, glyphMap)] - - -def buildMarkLigPosSubtable(marks, ligs, glyphMap): - """Build a single MarkLigPos (GPOS5) subtable. - - This builds a mark-to-base lookup subtable containing all of the referenced - marks and bases. See :func:`buildMarkLigPos`. - - Args: - marks (dict): A dictionary mapping anchors to glyphs; the keys being - glyph names, and the values being a tuple of mark class number and - an ``otTables.Anchor`` object representing the mark's attachment - point. (See :func:`buildMarkArray`.) - ligs (dict): A mapping of ligature names to an array of dictionaries: - for each component glyph in the ligature, an dictionary mapping - mark class IDs to anchors. (See :func:`buildLigatureArray`.) - glyphMap: a glyph name to ID map, typically returned from - ``font.getReverseGlyphMap()``. - - Returns: A ``otTables.MarkLigPos`` object. """ self = ot.MarkLigPos() @@ -2706,10 +2742,18 @@ class ClassDefBuilder(object): AXIS_VALUE_NEGATIVE_INFINITY = fixedToFloat(-0x80000000, 16) AXIS_VALUE_POSITIVE_INFINITY = fixedToFloat(0x7FFFFFFF, 16) +STATName = Union[int, str, Dict[str, str]] +"""A raw name ID, English name, or multilingual name.""" + def buildStatTable( - ttFont, axes, locations=None, elidedFallbackName=2, windowsNames=True, macNames=True -): + ttFont: TTFont, + axes, + locations=None, + elidedFallbackName: Union[STATName, STATNameStatement] = 2, + windowsNames: bool = True, + macNames: bool = True, +) -> None: """Add a 'STAT' table to 'ttFont'. 'axes' is a list of dictionaries describing axes and their @@ -2900,7 +2944,13 @@ def _buildAxisValuesFormat4(locations, axes, ttFont, windowsNames=True, macNames return axisValues -def _addName(ttFont, value, minNameID=0, windows=True, mac=True): +def _addName( + ttFont: TTFont, + value: Union[STATName, STATNameStatement], + minNameID: int = 0, + windows: bool = True, + mac: bool = True, +) -> int: nameTable = ttFont["name"] if isinstance(value, int): # Already a nameID diff --git a/contrib/python/fonttools/fontTools/otlLib/optimize/gpos.py b/contrib/python/fonttools/fontTools/otlLib/optimize/gpos.py index 61ea856d96e..3edbfeb306c 100644 --- a/contrib/python/fonttools/fontTools/otlLib/optimize/gpos.py +++ b/contrib/python/fonttools/fontTools/otlLib/optimize/gpos.py @@ -1,7 +1,8 @@ import logging import os from collections import defaultdict, namedtuple -from functools import reduce +from dataclasses import dataclass +from functools import cached_property, reduce from itertools import chain from math import log2 from typing import DefaultDict, Dict, Iterable, List, Sequence, Tuple @@ -192,79 +193,58 @@ ClusteringContext = namedtuple( ) +@dataclass class Cluster: - # TODO(Python 3.7): Turn this into a dataclass - # ctx: ClusteringContext - # indices: int - # Caches - # TODO(Python 3.8): use functools.cached_property instead of the - # manually cached properties, and remove the cache fields listed below. - # _indices: Optional[List[int]] = None - # _column_indices: Optional[List[int]] = None - # _cost: Optional[int] = None - - __slots__ = "ctx", "indices_bitmask", "_indices", "_column_indices", "_cost" - - def __init__(self, ctx: ClusteringContext, indices_bitmask: int): - self.ctx = ctx - self.indices_bitmask = indices_bitmask - self._indices = None - self._column_indices = None - self._cost = None + ctx: ClusteringContext + indices_bitmask: int - @property + @cached_property def indices(self): - if self._indices is None: - self._indices = bit_indices(self.indices_bitmask) - return self._indices + return bit_indices(self.indices_bitmask) - @property + @cached_property def column_indices(self): - if self._column_indices is None: - # Indices of columns that have a 1 in at least 1 line - # => binary OR all the lines - bitmask = reduce(int.__or__, (self.ctx.lines[i] for i in self.indices)) - self._column_indices = bit_indices(bitmask) - return self._column_indices + # Indices of columns that have a 1 in at least 1 line + # => binary OR all the lines + bitmask = reduce(int.__or__, (self.ctx.lines[i] for i in self.indices)) + return bit_indices(bitmask) @property def width(self): # Add 1 because Class2=0 cannot be used but needs to be encoded. return len(self.column_indices) + 1 - @property + @cached_property def cost(self): - if self._cost is None: - self._cost = ( - # 2 bytes to store the offset to this subtable in the Lookup table above - 2 - # Contents of the subtable - # From: https://docs.microsoft.com/en-us/typography/opentype/spec/gpos#pair-adjustment-positioning-format-2-class-pair-adjustment - # uint16 posFormat Format identifier: format = 2 - + 2 - # Offset16 coverageOffset Offset to Coverage table, from beginning of PairPos subtable. - + 2 - + self.coverage_bytes - # uint16 valueFormat1 ValueRecord definition — for the first glyph of the pair (may be zero). - + 2 - # uint16 valueFormat2 ValueRecord definition — for the second glyph of the pair (may be zero). - + 2 - # Offset16 classDef1Offset Offset to ClassDef table, from beginning of PairPos subtable — for the first glyph of the pair. - + 2 - + self.classDef1_bytes - # Offset16 classDef2Offset Offset to ClassDef table, from beginning of PairPos subtable — for the second glyph of the pair. - + 2 - + self.classDef2_bytes - # uint16 class1Count Number of classes in classDef1 table — includes Class 0. - + 2 - # uint16 class2Count Number of classes in classDef2 table — includes Class 0. - + 2 - # Class1Record class1Records[class1Count] Array of Class1 records, ordered by classes in classDef1. - + (self.ctx.valueFormat1_bytes + self.ctx.valueFormat2_bytes) - * len(self.indices) - * self.width - ) - return self._cost + return ( + # 2 bytes to store the offset to this subtable in the Lookup table above + 2 + # Contents of the subtable + # From: https://docs.microsoft.com/en-us/typography/opentype/spec/gpos#pair-adjustment-positioning-format-2-class-pair-adjustment + # uint16 posFormat Format identifier: format = 2 + + 2 + # Offset16 coverageOffset Offset to Coverage table, from beginning of PairPos subtable. + + 2 + + self.coverage_bytes + # uint16 valueFormat1 ValueRecord definition — for the first glyph of the pair (may be zero). + + 2 + # uint16 valueFormat2 ValueRecord definition — for the second glyph of the pair (may be zero). + + 2 + # Offset16 classDef1Offset Offset to ClassDef table, from beginning of PairPos subtable — for the first glyph of the pair. + + 2 + + self.classDef1_bytes + # Offset16 classDef2Offset Offset to ClassDef table, from beginning of PairPos subtable — for the second glyph of the pair. + + 2 + + self.classDef2_bytes + # uint16 class1Count Number of classes in classDef1 table — includes Class 0. + + 2 + # uint16 class2Count Number of classes in classDef2 table — includes Class 0. + + 2 + # Class1Record class1Records[class1Count] Array of Class1 records, ordered by classes in classDef1. + + (self.ctx.valueFormat1_bytes + self.ctx.valueFormat2_bytes) + * len(self.indices) + * self.width + ) @property def coverage_bytes(self): diff --git a/contrib/python/fonttools/fontTools/pens/pointPen.py b/contrib/python/fonttools/fontTools/pens/pointPen.py index 93a9201c991..843d7a28d31 100644 --- a/contrib/python/fonttools/fontTools/pens/pointPen.py +++ b/contrib/python/fonttools/fontTools/pens/pointPen.py @@ -12,12 +12,14 @@ This allows the caller to provide more data for each point. For instance, whether or not a point is smooth, and its name. """ +from __future__ import annotations + import math -from typing import Any, Optional, Tuple, Dict +from typing import Any, Dict, List, Optional, Tuple from fontTools.misc.loggingTools import LogMixin -from fontTools.pens.basePen import AbstractPen, MissingComponentError, PenError from fontTools.misc.transform import DecomposedTransform, Identity +from fontTools.pens.basePen import AbstractPen, MissingComponentError, PenError __all__ = [ "AbstractPointPen", @@ -28,6 +30,14 @@ __all__ = [ "ReverseContourPointPen", ] +# Some type aliases to make it easier below +Point = Tuple[float, float] +PointName = Optional[str] +# [(pt, smooth, name, kwargs)] +SegmentPointList = List[Tuple[Optional[Point], bool, PointName, Any]] +SegmentType = Optional[str] +SegmentList = List[Tuple[SegmentType, SegmentPointList]] + class AbstractPointPen: """Baseclass for all PointPens.""" @@ -88,7 +98,7 @@ class BasePointToSegmentPen(AbstractPointPen): care of all the edge cases. """ - def __init__(self): + def __init__(self) -> None: self.currentPath = None def beginPath(self, identifier=None, **kwargs): @@ -96,7 +106,7 @@ class BasePointToSegmentPen(AbstractPointPen): raise PenError("Path already begun.") self.currentPath = [] - def _flushContour(self, segments): + def _flushContour(self, segments: SegmentList) -> None: """Override this method. It will be called for each non-empty sub path with a list @@ -124,7 +134,7 @@ class BasePointToSegmentPen(AbstractPointPen): """ raise NotImplementedError - def endPath(self): + def endPath(self) -> None: if self.currentPath is None: raise PenError("Path not begun.") points = self.currentPath @@ -134,7 +144,7 @@ class BasePointToSegmentPen(AbstractPointPen): if len(points) == 1: # Not much more we can do than output a single move segment. pt, segmentType, smooth, name, kwargs = points[0] - segments = [("move", [(pt, smooth, name, kwargs)])] + segments: SegmentList = [("move", [(pt, smooth, name, kwargs)])] self._flushContour(segments) return segments = [] @@ -162,7 +172,7 @@ class BasePointToSegmentPen(AbstractPointPen): else: points = points[firstOnCurve + 1 :] + points[: firstOnCurve + 1] - currentSegment = [] + currentSegment: SegmentPointList = [] for pt, segmentType, smooth, name, kwargs in points: currentSegment.append((pt, smooth, name, kwargs)) if segmentType is None: @@ -189,7 +199,7 @@ class PointToSegmentPen(BasePointToSegmentPen): and kwargs. """ - def __init__(self, segmentPen, outputImpliedClosingLine=False): + def __init__(self, segmentPen, outputImpliedClosingLine: bool = False) -> None: BasePointToSegmentPen.__init__(self) self.pen = segmentPen self.outputImpliedClosingLine = outputImpliedClosingLine @@ -271,14 +281,14 @@ class SegmentToPointPen(AbstractPen): PointPen protocol. """ - def __init__(self, pointPen, guessSmooth=True): + def __init__(self, pointPen, guessSmooth=True) -> None: if guessSmooth: self.pen = GuessSmoothPointPen(pointPen) else: self.pen = pointPen - self.contour = None + self.contour: Optional[List[Tuple[Point, SegmentType]]] = None - def _flushContour(self): + def _flushContour(self) -> None: pen = self.pen pen.beginPath() for pt, segmentType in self.contour: @@ -594,7 +604,6 @@ class DecomposingPointPen(LogMixin, AbstractPointPen): # if the transformation has a negative determinant, it will # reverse the contour direction of the component a, b, c, d = transformation[:4] - det = a * d - b * c if a * d - b * c < 0: pen = ReverseContourPointPen(pen) glyph.drawPoints(pen) diff --git a/contrib/python/fonttools/fontTools/subset/__init__.py b/contrib/python/fonttools/fontTools/subset/__init__.py index 8458edc3592..056ad81babe 100644 --- a/contrib/python/fonttools/fontTools/subset/__init__.py +++ b/contrib/python/fonttools/fontTools/subset/__init__.py @@ -16,6 +16,7 @@ from fontTools.subset.cff import * from fontTools.subset.svg import * from fontTools.varLib import varStore, multiVarStore # For monkey-patching from fontTools.ttLib.tables._n_a_m_e import NameRecordVisitor +from fontTools.unicodedata import mirrored import sys import struct import array @@ -2870,6 +2871,15 @@ def prune_post_subset(self, font, options): def closure_glyphs(self, s): tables = [t for t in self.tables if t.isUnicode()] + # Closure unicodes, which for now is pulling in bidi mirrored variants + if s.options.bidi_closure: + additional_unicodes = set() + for u in s.unicodes_requested: + mirror_u = mirrored(u) + if mirror_u is not None: + additional_unicodes.add(mirror_u) + s.unicodes_requested.update(additional_unicodes) + # Close glyphs for table in tables: if table.format == 14: @@ -3191,6 +3201,7 @@ class Options(object): self.font_number = -1 self.pretty_svg = False self.lazy = True + self.bidi_closure = True self.set(**kwargs) diff --git a/contrib/python/fonttools/fontTools/ttLib/tables/G_V_A_R_.py b/contrib/python/fonttools/fontTools/ttLib/tables/G_V_A_R_.py new file mode 100644 index 00000000000..889b1f2a3bd --- /dev/null +++ b/contrib/python/fonttools/fontTools/ttLib/tables/G_V_A_R_.py @@ -0,0 +1,5 @@ +from ._g_v_a_r import table__g_v_a_r + + +class table_G_V_A_R_(table__g_v_a_r): + gid_size = 3 diff --git a/contrib/python/fonttools/fontTools/ttLib/tables/T_S_I__0.py b/contrib/python/fonttools/fontTools/ttLib/tables/T_S_I__0.py index 0d0e61a1cd3..d60e783c608 100644 --- a/contrib/python/fonttools/fontTools/ttLib/tables/T_S_I__0.py +++ b/contrib/python/fonttools/fontTools/ttLib/tables/T_S_I__0.py @@ -1,4 +1,4 @@ -""" TSI{0,1,2,3,5} are private tables used by Microsoft Visual TrueType (VTT) +"""TSI{0,1,2,3,5} are private tables used by Microsoft Visual TrueType (VTT) tool to store its hinting source data. TSI0 is the index table containing the lengths and offsets for the glyph @@ -8,9 +8,13 @@ in the TSI1 table. See also https://learn.microsoft.com/en-us/typography/tools/vtt/tsi-tables """ -from . import DefaultTable +import logging import struct +from . import DefaultTable + +log = logging.getLogger(__name__) + tsi0Format = ">HHL" @@ -25,7 +29,14 @@ class table_T_S_I__0(DefaultTable.DefaultTable): numGlyphs = ttFont["maxp"].numGlyphs indices = [] size = struct.calcsize(tsi0Format) - for i in range(numGlyphs + 5): + numEntries = len(data) // size + if numEntries != numGlyphs + 5: + diff = numEntries - numGlyphs - 5 + log.warning( + "Number of glyphPrograms differs from the number of glyphs in the font " + f"by {abs(diff)} ({numEntries - 5} programs vs. {numGlyphs} glyphs)." + ) + for _ in range(numEntries): glyphID, textLength, textOffset = fixlongs( *struct.unpack(tsi0Format, data[:size]) ) diff --git a/contrib/python/fonttools/fontTools/ttLib/tables/T_S_I__5.py b/contrib/python/fonttools/fontTools/ttLib/tables/T_S_I__5.py index 24078b22561..6afd76832fe 100644 --- a/contrib/python/fonttools/fontTools/ttLib/tables/T_S_I__5.py +++ b/contrib/python/fonttools/fontTools/ttLib/tables/T_S_I__5.py @@ -1,4 +1,4 @@ -""" TSI{0,1,2,3,5} are private tables used by Microsoft Visual TrueType (VTT) +"""TSI{0,1,2,3,5} are private tables used by Microsoft Visual TrueType (VTT) tool to store its hinting source data. TSI5 contains the VTT character groups. @@ -6,22 +6,33 @@ TSI5 contains the VTT character groups. See also https://learn.microsoft.com/en-us/typography/tools/vtt/tsi-tables """ +import array +import logging +import sys + from fontTools.misc.textTools import safeEval + from . import DefaultTable -import sys -import array + +log = logging.getLogger(__name__) class table_T_S_I__5(DefaultTable.DefaultTable): def decompile(self, data, ttFont): numGlyphs = ttFont["maxp"].numGlyphs - assert len(data) == 2 * numGlyphs a = array.array("H") a.frombytes(data) if sys.byteorder != "big": a.byteswap() self.glyphGrouping = {} - for i in range(numGlyphs): + numEntries = len(data) // 2 + if numEntries != numGlyphs: + diff = numEntries - numGlyphs + log.warning( + "Number of entries differs from the number of glyphs in the font " + f"by {abs(diff)} ({numEntries} entries vs. {numGlyphs} glyphs)." + ) + for i in range(numEntries): self.glyphGrouping[ttFont.getGlyphName(i)] = a[i] def compile(self, ttFont): diff --git a/contrib/python/fonttools/fontTools/ttLib/tables/__init__.py b/contrib/python/fonttools/fontTools/ttLib/tables/__init__.py index e622f1d1349..b111097a804 100644 --- a/contrib/python/fonttools/fontTools/ttLib/tables/__init__.py +++ b/contrib/python/fonttools/fontTools/ttLib/tables/__init__.py @@ -23,6 +23,7 @@ def _moduleFinderHint(): from . import G_P_K_G_ from . import G_P_O_S_ from . import G_S_U_B_ + from . import G_V_A_R_ from . import G__l_a_t from . import G__l_o_c from . import H_V_A_R_ diff --git a/contrib/python/fonttools/fontTools/ttLib/tables/_c_v_t.py b/contrib/python/fonttools/fontTools/ttLib/tables/_c_v_t.py index 92c50a1b8d6..51e2f78df8d 100644 --- a/contrib/python/fonttools/fontTools/ttLib/tables/_c_v_t.py +++ b/contrib/python/fonttools/fontTools/ttLib/tables/_c_v_t.py @@ -21,6 +21,8 @@ class table__c_v_t(DefaultTable.DefaultTable): self.values = values def compile(self, ttFont): + if not hasattr(self, "values"): + return b"" values = self.values[:] if sys.byteorder != "big": values.byteswap() diff --git a/contrib/python/fonttools/fontTools/ttLib/tables/_f_p_g_m.py b/contrib/python/fonttools/fontTools/ttLib/tables/_f_p_g_m.py index ba8e0488deb..c21a9d4b681 100644 --- a/contrib/python/fonttools/fontTools/ttLib/tables/_f_p_g_m.py +++ b/contrib/python/fonttools/fontTools/ttLib/tables/_f_p_g_m.py @@ -20,7 +20,9 @@ class table__f_p_g_m(DefaultTable.DefaultTable): self.program = program def compile(self, ttFont): - return self.program.getBytecode() + if hasattr(self, "program"): + return self.program.getBytecode() + return b"" def toXML(self, writer, ttFont): self.program.toXML(writer, ttFont) diff --git a/contrib/python/fonttools/fontTools/ttLib/tables/_g_l_y_f.py b/contrib/python/fonttools/fontTools/ttLib/tables/_g_l_y_f.py index c05fcea5d35..ea46c9f7971 100644 --- a/contrib/python/fonttools/fontTools/ttLib/tables/_g_l_y_f.py +++ b/contrib/python/fonttools/fontTools/ttLib/tables/_g_l_y_f.py @@ -1225,7 +1225,7 @@ class Glyph(object): if boundsDone is not None: boundsDone.add(glyphName) # empty components shouldn't update the bounds of the parent glyph - if g.numberOfContours == 0: + if g.yMin == g.yMax and g.xMin == g.xMax: continue x, y = compo.x, compo.y @@ -1285,11 +1285,7 @@ class Glyph(object): # however, if the referenced component glyph is another composite, we # must not round here but only at the end, after all the nested # transforms have been applied, or else rounding errors will compound. - if ( - round is not noRound - and g.numberOfContours > 0 - and not compo._hasOnlyIntegerTranslate() - ): + if round is not noRound and g.numberOfContours > 0: coordinates.toInt(round=round) if hasattr(compo, "firstPt"): # component uses two reference points: we apply the transform _before_ diff --git a/contrib/python/fonttools/fontTools/ttLib/tables/_g_v_a_r.py b/contrib/python/fonttools/fontTools/ttLib/tables/_g_v_a_r.py index e942beaf58b..07d3befb7aa 100644 --- a/contrib/python/fonttools/fontTools/ttLib/tables/_g_v_a_r.py +++ b/contrib/python/fonttools/fontTools/ttLib/tables/_g_v_a_r.py @@ -24,19 +24,24 @@ log = logging.getLogger(__name__) # FreeType2 source code for parsing 'gvar': # http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/src/truetype/ttgxvar.c -GVAR_HEADER_FORMAT = """ +GVAR_HEADER_FORMAT_HEAD = """ > # big endian version: H reserved: H axisCount: H sharedTupleCount: H offsetToSharedTuples: I - glyphCount: H +""" +# In between the HEAD and TAIL lies the glyphCount, which is +# of different size: 2 bytes for gvar, and 3 bytes for GVAR. +GVAR_HEADER_FORMAT_TAIL = """ + > # big endian flags: H offsetToGlyphVariationData: I """ -GVAR_HEADER_SIZE = sstruct.calcsize(GVAR_HEADER_FORMAT) +GVAR_HEADER_SIZE_HEAD = sstruct.calcsize(GVAR_HEADER_FORMAT_HEAD) +GVAR_HEADER_SIZE_TAIL = sstruct.calcsize(GVAR_HEADER_FORMAT_TAIL) class table__g_v_a_r(DefaultTable.DefaultTable): @@ -51,6 +56,7 @@ class table__g_v_a_r(DefaultTable.DefaultTable): """ dependencies = ["fvar", "glyf"] + gid_size = 2 def __init__(self, tag=None): DefaultTable.DefaultTable.__init__(self, tag) @@ -74,20 +80,25 @@ class table__g_v_a_r(DefaultTable.DefaultTable): offsets.append(offset) compiledOffsets, tableFormat = self.compileOffsets_(offsets) + GVAR_HEADER_SIZE = GVAR_HEADER_SIZE_HEAD + self.gid_size + GVAR_HEADER_SIZE_TAIL header = {} header["version"] = self.version header["reserved"] = self.reserved header["axisCount"] = len(axisTags) header["sharedTupleCount"] = len(sharedTuples) header["offsetToSharedTuples"] = GVAR_HEADER_SIZE + len(compiledOffsets) - header["glyphCount"] = len(compiledGlyphs) header["flags"] = tableFormat header["offsetToGlyphVariationData"] = ( header["offsetToSharedTuples"] + sharedTupleSize ) - compiledHeader = sstruct.pack(GVAR_HEADER_FORMAT, header) - result = [compiledHeader, compiledOffsets] + result = [ + sstruct.pack(GVAR_HEADER_FORMAT_HEAD, header), + len(compiledGlyphs).to_bytes(self.gid_size, "big"), + sstruct.pack(GVAR_HEADER_FORMAT_TAIL, header), + ] + + result.append(compiledOffsets) result.extend(sharedTuples) result.extend(compiledGlyphs) return b"".join(result) @@ -104,6 +115,7 @@ class table__g_v_a_r(DefaultTable.DefaultTable): pointCountUnused = 0 # pointCount is actually unused by compileGlyph result.append( compileGlyph_( + self.gid_size, variations, pointCountUnused, axisTags, @@ -116,7 +128,19 @@ class table__g_v_a_r(DefaultTable.DefaultTable): def decompile(self, data, ttFont): axisTags = [axis.axisTag for axis in ttFont["fvar"].axes] glyphs = ttFont.getGlyphOrder() - sstruct.unpack(GVAR_HEADER_FORMAT, data[0:GVAR_HEADER_SIZE], self) + + # Parse the header + GVAR_HEADER_SIZE = GVAR_HEADER_SIZE_HEAD + self.gid_size + GVAR_HEADER_SIZE_TAIL + sstruct.unpack(GVAR_HEADER_FORMAT_HEAD, data[:GVAR_HEADER_SIZE_HEAD], self) + self.glyphCount = int.from_bytes( + data[GVAR_HEADER_SIZE_HEAD : GVAR_HEADER_SIZE_HEAD + self.gid_size], "big" + ) + sstruct.unpack( + GVAR_HEADER_FORMAT_TAIL, + data[GVAR_HEADER_SIZE_HEAD + self.gid_size : GVAR_HEADER_SIZE], + self, + ) + assert len(glyphs) == self.glyphCount assert len(axisTags) == self.axisCount sharedCoords = tv.decompileSharedTuples( @@ -146,7 +170,7 @@ class table__g_v_a_r(DefaultTable.DefaultTable): glyph = glyf[glyphName] numPointsInGlyph = self.getNumPoints_(glyph) return decompileGlyph_( - numPointsInGlyph, sharedCoords, axisTags, gvarData + self.gid_size, numPointsInGlyph, sharedCoords, axisTags, gvarData ) return read_item @@ -264,23 +288,42 @@ class table__g_v_a_r(DefaultTable.DefaultTable): def compileGlyph_( - variations, pointCount, axisTags, sharedCoordIndices, *, optimizeSize=True + dataOffsetSize, + variations, + pointCount, + axisTags, + sharedCoordIndices, + *, + optimizeSize=True, ): + assert dataOffsetSize in (2, 3) tupleVariationCount, tuples, data = tv.compileTupleVariationStore( variations, pointCount, axisTags, sharedCoordIndices, optimizeSize=optimizeSize ) if tupleVariationCount == 0: return b"" - result = [struct.pack(">HH", tupleVariationCount, 4 + len(tuples)), tuples, data] - if (len(tuples) + len(data)) % 2 != 0: + + offsetToData = 2 + dataOffsetSize + len(tuples) + + result = [ + tupleVariationCount.to_bytes(2, "big"), + offsetToData.to_bytes(dataOffsetSize, "big"), + tuples, + data, + ] + if (offsetToData + len(data)) % 2 != 0: result.append(b"\0") # padding return b"".join(result) -def decompileGlyph_(pointCount, sharedTuples, axisTags, data): - if len(data) < 4: +def decompileGlyph_(dataOffsetSize, pointCount, sharedTuples, axisTags, data): + assert dataOffsetSize in (2, 3) + if len(data) < 2 + dataOffsetSize: return [] - tupleVariationCount, offsetToData = struct.unpack(">HH", data[:4]) + + tupleVariationCount = int.from_bytes(data[:2], "big") + offsetToData = int.from_bytes(data[2 : 2 + dataOffsetSize], "big") + dataPos = offsetToData return tv.decompileTupleVariationStore( "gvar", @@ -289,6 +332,6 @@ def decompileGlyph_(pointCount, sharedTuples, axisTags, data): pointCount, sharedTuples, data, - 4, + 2 + dataOffsetSize, offsetToData, ) diff --git a/contrib/python/fonttools/fontTools/ttLib/tables/_p_o_s_t.py b/contrib/python/fonttools/fontTools/ttLib/tables/_p_o_s_t.py index fca0812f984..c449e5f0c03 100644 --- a/contrib/python/fonttools/fontTools/ttLib/tables/_p_o_s_t.py +++ b/contrib/python/fonttools/fontTools/ttLib/tables/_p_o_s_t.py @@ -122,13 +122,16 @@ class table__p_o_s_t(DefaultTable.DefaultTable): glyphName = psName = self.glyphOrder[i] if glyphName == "": glyphName = "glyph%.5d" % i + if glyphName in allNames: # make up a new glyphName that's unique n = allNames[glyphName] - while (glyphName + "#" + str(n)) in allNames: + # check if the exists in any of the seen names or later ones + names = set(allNames.keys()) | set(self.glyphOrder) + while (glyphName + "." + str(n)) in names: n += 1 allNames[glyphName] = n + 1 - glyphName = glyphName + "#" + str(n) + glyphName = glyphName + "." + str(n) self.glyphOrder[i] = glyphName allNames[glyphName] = 1 diff --git a/contrib/python/fonttools/fontTools/ttLib/tables/otBase.py b/contrib/python/fonttools/fontTools/ttLib/tables/otBase.py index 8df7c236b1c..582b02024b3 100644 --- a/contrib/python/fonttools/fontTools/ttLib/tables/otBase.py +++ b/contrib/python/fonttools/fontTools/ttLib/tables/otBase.py @@ -398,6 +398,7 @@ class OTTableWriter(object): self.localState = localState self.tableTag = tableTag self.parent = None + self.name = "<none>" def __setitem__(self, name, value): state = self.localState.copy() if self.localState else dict() diff --git a/contrib/python/fonttools/fontTools/ufoLib/__init__.py b/contrib/python/fonttools/fontTools/ufoLib/__init__.py index 42c06734ea9..f76938a8f15 100644 --- a/contrib/python/fonttools/fontTools/ufoLib/__init__.py +++ b/contrib/python/fonttools/fontTools/ufoLib/__init__.py @@ -204,7 +204,7 @@ class UFOReader(_UFOBaseIO): """Read the various components of a .ufo. Attributes: - path: An `os.PathLike` object pointing to the .ufo. + path: An :class:`os.PathLike` object pointing to the .ufo. validate: A boolean indicating if the data read should be validated. Defaults to `True`. @@ -891,7 +891,7 @@ class UFOWriter(UFOReader): """Write the various components of a .ufo. Attributes: - path: An `os.PathLike` object pointing to the .ufo. + path: An :class:`os.PathLike` object pointing to the .ufo. formatVersion: the UFO format version as a tuple of integers (major, minor), or as a single integer for the major digit only (minor is implied to be 0). By default, the latest formatVersion will be used; currently it is 3.0, diff --git a/contrib/python/fonttools/fontTools/ufoLib/converters.py b/contrib/python/fonttools/fontTools/ufoLib/converters.py index 88a26c616a8..4ee6b05e338 100644 --- a/contrib/python/fonttools/fontTools/ufoLib/converters.py +++ b/contrib/python/fonttools/fontTools/ufoLib/converters.py @@ -1,11 +1,33 @@ """ -Conversion functions. +Functions for converting UFO1 or UFO2 files into UFO3 format. + +Currently provides functionality for converting kerning rules +and kerning groups. Conversion is only supported _from_ UFO1 +or UFO2, and _to_ UFO3. """ # adapted from the UFO spec def convertUFO1OrUFO2KerningToUFO3Kerning(kerning, groups, glyphSet=()): + """Convert kerning data in UFO1 or UFO2 syntax into UFO3 syntax. + + Args: + kerning: + A dictionary containing the kerning rules defined in + the UFO font, as used in :class:`.UFOReader` objects. + groups: + A dictionary containing the groups defined in the UFO + font, as used in :class:`.UFOReader` objects. + glyphSet: + Optional; a set of glyph objects to skip (default: None). + + Returns: + 1. A dictionary representing the converted kerning data. + 2. A copy of the groups dictionary, with all groups renamed to UFO3 syntax. + 3. A dictionary containing the mapping of old group names to new group names. + + """ # gather known kerning groups based on the prefixes firstReferencedGroups, secondReferencedGroups = findKnownKerningGroups(groups) # Make lists of groups referenced in kerning pairs. @@ -63,35 +85,54 @@ def convertUFO1OrUFO2KerningToUFO3Kerning(kerning, groups, glyphSet=()): def findKnownKerningGroups(groups): - """ - This will find kerning groups with known prefixes. - In some cases not all kerning groups will be referenced - by the kerning pairs. The algorithm for locating groups - in convertUFO1OrUFO2KerningToUFO3Kerning will miss these - unreferenced groups. By scanning for known prefixes + """Find all kerning groups in a UFO1 or UFO2 font that use known prefixes. + + In some cases, not all kerning groups will be referenced + by the kerning pairs in a UFO. The algorithm for locating + groups in :func:`convertUFO1OrUFO2KerningToUFO3Kerning` will + miss these unreferenced groups. By scanning for known prefixes, this function will catch all of the prefixed groups. - These are the prefixes and sides that are handled: + The prefixes and sides by this function are: + @MMK_L_ - side 1 @MMK_R_ - side 2 - >>> testGroups = { - ... "@MMK_L_1" : None, - ... "@MMK_L_2" : None, - ... "@MMK_L_3" : None, - ... "@MMK_R_1" : None, - ... "@MMK_R_2" : None, - ... "@MMK_R_3" : None, - ... "@MMK_l_1" : None, - ... "@MMK_r_1" : None, - ... "@MMK_X_1" : None, - ... "foo" : None, - ... } - >>> first, second = findKnownKerningGroups(testGroups) - >>> sorted(first) == ['@MMK_L_1', '@MMK_L_2', '@MMK_L_3'] - True - >>> sorted(second) == ['@MMK_R_1', '@MMK_R_2', '@MMK_R_3'] - True + as defined in the UFO1 specification. + + Args: + groups: + A dictionary containing the groups defined in the UFO + font, as read by :class:`.UFOReader`. + + Returns: + Two sets; the first containing the names of all + first-side kerning groups identified in the ``groups`` + dictionary, and the second containing the names of all + second-side kerning groups identified. + + "First-side" and "second-side" are with respect to the + writing direction of the script. + + Example:: + + >>> testGroups = { + ... "@MMK_L_1" : None, + ... "@MMK_L_2" : None, + ... "@MMK_L_3" : None, + ... "@MMK_R_1" : None, + ... "@MMK_R_2" : None, + ... "@MMK_R_3" : None, + ... "@MMK_l_1" : None, + ... "@MMK_r_1" : None, + ... "@MMK_X_1" : None, + ... "foo" : None, + ... } + >>> first, second = findKnownKerningGroups(testGroups) + >>> sorted(first) == ['@MMK_L_1', '@MMK_L_2', '@MMK_L_3'] + True + >>> sorted(second) == ['@MMK_R_1', '@MMK_R_2', '@MMK_R_3'] + True """ knownFirstGroupPrefixes = ["@MMK_L_"] knownSecondGroupPrefixes = ["@MMK_R_"] @@ -110,6 +151,27 @@ def findKnownKerningGroups(groups): def makeUniqueGroupName(name, groupNames, counter=0): + """Make a kerning group name that will be unique within the set of group names. + + If the requested kerning group name already exists within the set, this + will return a new name by adding an incremented counter to the end + of the requested name. + + Args: + name: + The requested kerning group name. + groupNames: + A list of the existing kerning group names. + counter: + Optional; a counter of group names already seen (default: 0). If + :attr:`.counter` is not provided, the function will recurse, + incrementing the value of :attr:`.counter` until it finds the + first unused ``name+counter`` combination, and return that result. + + Returns: + A unique kerning group name composed of the requested name suffixed + by the smallest available integer counter. + """ # Add a number to the name if the counter is higher than zero. newName = name if counter > 0: @@ -123,6 +185,8 @@ def makeUniqueGroupName(name, groupNames, counter=0): def test(): """ + Tests for :func:`.convertUFO1OrUFO2KerningToUFO3Kerning`. + No known prefixes. >>> testKerning = { diff --git a/contrib/python/fonttools/fontTools/ufoLib/errors.py b/contrib/python/fonttools/fontTools/ufoLib/errors.py index e05dd438b43..6cc9fec3994 100644 --- a/contrib/python/fonttools/fontTools/ufoLib/errors.py +++ b/contrib/python/fonttools/fontTools/ufoLib/errors.py @@ -10,6 +10,14 @@ class UnsupportedUFOFormat(UFOLibError): class GlifLibError(UFOLibError): + """An error raised by glifLib. + + This class is a loose backport of PEP 678, adding a :attr:`.note` + attribute that can hold additional context for errors encountered. + + It will be maintained until only Python 3.11-and-later are supported. + """ + def _add_note(self, note: str) -> None: # Loose backport of PEP 678 until we only support Python 3.11+, used for # adding additional context to errors. diff --git a/contrib/python/fonttools/fontTools/ufoLib/etree.py b/contrib/python/fonttools/fontTools/ufoLib/etree.py index 77e3c16e2b4..07b924ac852 100644 --- a/contrib/python/fonttools/fontTools/ufoLib/etree.py +++ b/contrib/python/fonttools/fontTools/ufoLib/etree.py @@ -1,5 +1,5 @@ """DEPRECATED - This module is kept here only as a backward compatibility shim -for the old ufoLib.etree module, which was moved to fontTools.misc.etree. +for the old ufoLib.etree module, which was moved to :mod:`fontTools.misc.etree`. Please use the latter instead. """ diff --git a/contrib/python/fonttools/fontTools/ufoLib/filenames.py b/contrib/python/fonttools/fontTools/ufoLib/filenames.py index 7f1af58ee84..83442f1c8ce 100644 --- a/contrib/python/fonttools/fontTools/ufoLib/filenames.py +++ b/contrib/python/fonttools/fontTools/ufoLib/filenames.py @@ -1,6 +1,22 @@ """ -User name to file name conversion. -This was taken from the UFO 3 spec. +Convert user-provided internal UFO names to spec-compliant filenames. + +This module implements the algorithm for converting between a "user name" - +something that a user can choose arbitrarily inside a font editor - and a file +name suitable for use in a wide range of operating systems and filesystems. + +The `UFO 3 specification <http://unifiedfontobject.org/versions/ufo3/conventions/>`_ +provides an example of an algorithm for such conversion, which avoids illegal +characters, reserved file names, ambiguity between upper- and lower-case +characters, and clashes with existing files. + +This code was originally copied from +`ufoLib <https://github.com/unified-font-object/ufoLib/blob/8747da7/Lib/ufoLib/filenames.py>`_ +by Tal Leming and is copyright (c) 2005-2016, The RoboFab Developers: + +- Erik van Blokland +- Tal Leming +- Just van Rossum """ # Restrictions are taken mostly from @@ -93,53 +109,69 @@ class NameTranslationError(Exception): def userNameToFileName(userName: str, existing=(), prefix="", suffix=""): - """ - `existing` should be a set-like object. - - >>> userNameToFileName("a") == "a" - True - >>> userNameToFileName("A") == "A_" - True - >>> userNameToFileName("AE") == "A_E_" - True - >>> userNameToFileName("Ae") == "A_e" - True - >>> userNameToFileName("ae") == "ae" - True - >>> userNameToFileName("aE") == "aE_" - True - >>> userNameToFileName("a.alt") == "a.alt" - True - >>> userNameToFileName("A.alt") == "A_.alt" - True - >>> userNameToFileName("A.Alt") == "A_.A_lt" - True - >>> userNameToFileName("A.aLt") == "A_.aL_t" - True - >>> userNameToFileName(u"A.alT") == "A_.alT_" - True - >>> userNameToFileName("T_H") == "T__H_" - True - >>> userNameToFileName("T_h") == "T__h" - True - >>> userNameToFileName("t_h") == "t_h" - True - >>> userNameToFileName("F_F_I") == "F__F__I_" - True - >>> userNameToFileName("f_f_i") == "f_f_i" - True - >>> userNameToFileName("Aacute_V.swash") == "A_acute_V_.swash" - True - >>> userNameToFileName(".notdef") == "_notdef" - True - >>> userNameToFileName("con") == "_con" - True - >>> userNameToFileName("CON") == "C_O_N_" - True - >>> userNameToFileName("con.alt") == "_con.alt" - True - >>> userNameToFileName("alt.con") == "alt._con" - True + """Converts from a user name to a file name. + + Takes care to avoid illegal characters, reserved file names, ambiguity between + upper- and lower-case characters, and clashes with existing files. + + Args: + userName (str): The input file name. + existing: A case-insensitive list of all existing file names. + prefix: Prefix to be prepended to the file name. + suffix: Suffix to be appended to the file name. + + Returns: + A suitable filename. + + Raises: + NameTranslationError: If no suitable name could be generated. + + Examples:: + + >>> userNameToFileName("a") == "a" + True + >>> userNameToFileName("A") == "A_" + True + >>> userNameToFileName("AE") == "A_E_" + True + >>> userNameToFileName("Ae") == "A_e" + True + >>> userNameToFileName("ae") == "ae" + True + >>> userNameToFileName("aE") == "aE_" + True + >>> userNameToFileName("a.alt") == "a.alt" + True + >>> userNameToFileName("A.alt") == "A_.alt" + True + >>> userNameToFileName("A.Alt") == "A_.A_lt" + True + >>> userNameToFileName("A.aLt") == "A_.aL_t" + True + >>> userNameToFileName(u"A.alT") == "A_.alT_" + True + >>> userNameToFileName("T_H") == "T__H_" + True + >>> userNameToFileName("T_h") == "T__h" + True + >>> userNameToFileName("t_h") == "t_h" + True + >>> userNameToFileName("F_F_I") == "F__F__I_" + True + >>> userNameToFileName("f_f_i") == "f_f_i" + True + >>> userNameToFileName("Aacute_V.swash") == "A_acute_V_.swash" + True + >>> userNameToFileName(".notdef") == "_notdef" + True + >>> userNameToFileName("con") == "_con" + True + >>> userNameToFileName("CON") == "C_O_N_" + True + >>> userNameToFileName("con.alt") == "_con.alt" + True + >>> userNameToFileName("alt.con") == "alt._con" + True """ # the incoming name must be a string if not isinstance(userName, str): @@ -181,33 +213,42 @@ def userNameToFileName(userName: str, existing=(), prefix="", suffix=""): def handleClash1(userName, existing=[], prefix="", suffix=""): - """ - existing should be a case-insensitive list - of all existing file names. - - >>> prefix = ("0" * 5) + "." - >>> suffix = "." + ("0" * 10) - >>> existing = ["a" * 5] - - >>> e = list(existing) - >>> handleClash1(userName="A" * 5, existing=e, - ... prefix=prefix, suffix=suffix) == ( - ... '00000.AAAAA000000000000001.0000000000') - True - - >>> e = list(existing) - >>> e.append(prefix + "aaaaa" + "1".zfill(15) + suffix) - >>> handleClash1(userName="A" * 5, existing=e, - ... prefix=prefix, suffix=suffix) == ( - ... '00000.AAAAA000000000000002.0000000000') - True - - >>> e = list(existing) - >>> e.append(prefix + "AAAAA" + "2".zfill(15) + suffix) - >>> handleClash1(userName="A" * 5, existing=e, - ... prefix=prefix, suffix=suffix) == ( - ... '00000.AAAAA000000000000001.0000000000') - True + """A helper function that resolves collisions with existing names when choosing a filename. + + This function attempts to append an unused integer counter to the filename. + + Args: + userName (str): The input file name. + existing: A case-insensitive list of all existing file names. + prefix: Prefix to be prepended to the file name. + suffix: Suffix to be appended to the file name. + + Returns: + A suitable filename. + + >>> prefix = ("0" * 5) + "." + >>> suffix = "." + ("0" * 10) + >>> existing = ["a" * 5] + + >>> e = list(existing) + >>> handleClash1(userName="A" * 5, existing=e, + ... prefix=prefix, suffix=suffix) == ( + ... '00000.AAAAA000000000000001.0000000000') + True + + >>> e = list(existing) + >>> e.append(prefix + "aaaaa" + "1".zfill(15) + suffix) + >>> handleClash1(userName="A" * 5, existing=e, + ... prefix=prefix, suffix=suffix) == ( + ... '00000.AAAAA000000000000002.0000000000') + True + + >>> e = list(existing) + >>> e.append(prefix + "AAAAA" + "2".zfill(15) + suffix) + >>> handleClash1(userName="A" * 5, existing=e, + ... prefix=prefix, suffix=suffix) == ( + ... '00000.AAAAA000000000000001.0000000000') + True """ # if the prefix length + user name length + suffix length + 15 is at # or past the maximum length, silce 15 characters off of the user name @@ -238,30 +279,44 @@ def handleClash1(userName, existing=[], prefix="", suffix=""): def handleClash2(existing=[], prefix="", suffix=""): - """ - existing should be a case-insensitive list - of all existing file names. - - >>> prefix = ("0" * 5) + "." - >>> suffix = "." + ("0" * 10) - >>> existing = [prefix + str(i) + suffix for i in range(100)] - - >>> e = list(existing) - >>> handleClash2(existing=e, prefix=prefix, suffix=suffix) == ( - ... '00000.100.0000000000') - True - - >>> e = list(existing) - >>> e.remove(prefix + "1" + suffix) - >>> handleClash2(existing=e, prefix=prefix, suffix=suffix) == ( - ... '00000.1.0000000000') - True - - >>> e = list(existing) - >>> e.remove(prefix + "2" + suffix) - >>> handleClash2(existing=e, prefix=prefix, suffix=suffix) == ( - ... '00000.2.0000000000') - True + """A helper function that resolves collisions with existing names when choosing a filename. + + This function is a fallback to :func:`handleClash1`. It attempts to append an unused integer counter to the filename. + + Args: + userName (str): The input file name. + existing: A case-insensitive list of all existing file names. + prefix: Prefix to be prepended to the file name. + suffix: Suffix to be appended to the file name. + + Returns: + A suitable filename. + + Raises: + NameTranslationError: If no suitable name could be generated. + + Examples:: + + >>> prefix = ("0" * 5) + "." + >>> suffix = "." + ("0" * 10) + >>> existing = [prefix + str(i) + suffix for i in range(100)] + + >>> e = list(existing) + >>> handleClash2(existing=e, prefix=prefix, suffix=suffix) == ( + ... '00000.100.0000000000') + True + + >>> e = list(existing) + >>> e.remove(prefix + "1" + suffix) + >>> handleClash2(existing=e, prefix=prefix, suffix=suffix) == ( + ... '00000.1.0000000000') + True + + >>> e = list(existing) + >>> e.remove(prefix + "2" + suffix) + >>> handleClash2(existing=e, prefix=prefix, suffix=suffix) == ( + ... '00000.2.0000000000') + True """ # calculate the longest possible string maxLength = maxFileNameLength - len(prefix) - len(suffix) diff --git a/contrib/python/fonttools/fontTools/ufoLib/glifLib.py b/contrib/python/fonttools/fontTools/ufoLib/glifLib.py index abbda491463..a5a05003ee8 100644 --- a/contrib/python/fonttools/fontTools/ufoLib/glifLib.py +++ b/contrib/python/fonttools/fontTools/ufoLib/glifLib.py @@ -1,11 +1,11 @@ """ -glifLib.py -- Generic module for reading and writing the .glif format. +Generic module for reading and writing the .glif format. More info about the .glif format (GLyphInterchangeFormat) can be found here: http://unifiedfontobject.org -The main class in this module is GlyphSet. It manages a set of .glif files +The main class in this module is :class:`GlyphSet`. It manages a set of .glif files in a folder. It offers two ways to read glyph data, and one way to write glyph data. See the class doc string for details. """ @@ -60,6 +60,13 @@ LAYERINFO_FILENAME = "layerinfo.plist" class GLIFFormatVersion(tuple, _VersionTupleEnumMixin, enum.Enum): + """Class representing the versions of the .glif format supported by the UFO version in use. + + For a given :mod:`fontTools.ufoLib.UFOFormatVersion`, the :func:`supported_versions` method will + return the supported versions of the GLIF file format. If the UFO version is unspecified, the + :func:`supported_versions` method will return all available GLIF format versions. + """ + FORMAT_1_0 = (1, 0) FORMAT_2_0 = (2, 0) diff --git a/contrib/python/fonttools/fontTools/ufoLib/kerning.py b/contrib/python/fonttools/fontTools/ufoLib/kerning.py index 8a1dca5b680..5c84dd720af 100644 --- a/contrib/python/fonttools/fontTools/ufoLib/kerning.py +++ b/contrib/python/fonttools/fontTools/ufoLib/kerning.py @@ -1,43 +1,73 @@ def lookupKerningValue( pair, kerning, groups, fallback=0, glyphToFirstGroup=None, glyphToSecondGroup=None ): - """ - Note: This expects kerning to be a flat dictionary - of kerning pairs, not the nested structure used - in kerning.plist. + """Retrieve the kerning value (if any) between a pair of elements. + + The elments can be either individual glyphs (by name) or kerning + groups (by name), or any combination of the two. + + Args: + pair: + A tuple, in logical order (first, second) with respect + to the reading direction, to query the font for kerning + information on. Each element in the tuple can be either + a glyph name or a kerning group name. + kerning: + A dictionary of kerning pairs. + groups: + A set of kerning groups. + fallback: + The fallback value to return if no kern is found between + the elements in ``pair``. Defaults to 0. + glyphToFirstGroup: + A dictionary mapping glyph names to the first-glyph kerning + groups to which they belong. Defaults to ``None``. + glyphToSecondGroup: + A dictionary mapping glyph names to the second-glyph kerning + groups to which they belong. Defaults to ``None``. + + Returns: + The kerning value between the element pair. If no kerning for + the pair is found, the fallback value is returned. + + Note: This function expects the ``kerning`` argument to be a flat + dictionary of kerning pairs, not the nested structure used in a + kerning.plist file. + + Examples:: - >>> groups = { - ... "public.kern1.O" : ["O", "D", "Q"], - ... "public.kern2.E" : ["E", "F"] - ... } - >>> kerning = { - ... ("public.kern1.O", "public.kern2.E") : -100, - ... ("public.kern1.O", "F") : -200, - ... ("D", "F") : -300 - ... } - >>> lookupKerningValue(("D", "F"), kerning, groups) - -300 - >>> lookupKerningValue(("O", "F"), kerning, groups) - -200 - >>> lookupKerningValue(("O", "E"), kerning, groups) - -100 - >>> lookupKerningValue(("O", "O"), kerning, groups) - 0 - >>> lookupKerningValue(("E", "E"), kerning, groups) - 0 - >>> lookupKerningValue(("E", "O"), kerning, groups) - 0 - >>> lookupKerningValue(("X", "X"), kerning, groups) - 0 - >>> lookupKerningValue(("public.kern1.O", "public.kern2.E"), - ... kerning, groups) - -100 - >>> lookupKerningValue(("public.kern1.O", "F"), kerning, groups) - -200 - >>> lookupKerningValue(("O", "public.kern2.E"), kerning, groups) - -100 - >>> lookupKerningValue(("public.kern1.X", "public.kern2.X"), kerning, groups) - 0 + >>> groups = { + ... "public.kern1.O" : ["O", "D", "Q"], + ... "public.kern2.E" : ["E", "F"] + ... } + >>> kerning = { + ... ("public.kern1.O", "public.kern2.E") : -100, + ... ("public.kern1.O", "F") : -200, + ... ("D", "F") : -300 + ... } + >>> lookupKerningValue(("D", "F"), kerning, groups) + -300 + >>> lookupKerningValue(("O", "F"), kerning, groups) + -200 + >>> lookupKerningValue(("O", "E"), kerning, groups) + -100 + >>> lookupKerningValue(("O", "O"), kerning, groups) + 0 + >>> lookupKerningValue(("E", "E"), kerning, groups) + 0 + >>> lookupKerningValue(("E", "O"), kerning, groups) + 0 + >>> lookupKerningValue(("X", "X"), kerning, groups) + 0 + >>> lookupKerningValue(("public.kern1.O", "public.kern2.E"), + ... kerning, groups) + -100 + >>> lookupKerningValue(("public.kern1.O", "F"), kerning, groups) + -200 + >>> lookupKerningValue(("O", "public.kern2.E"), kerning, groups) + -100 + >>> lookupKerningValue(("public.kern1.X", "public.kern2.X"), kerning, groups) + 0 """ # quickly check to see if the pair is in the kerning dictionary if pair in kerning: diff --git a/contrib/python/fonttools/fontTools/ufoLib/utils.py b/contrib/python/fonttools/fontTools/ufoLib/utils.py index 45ec1c564b7..45ae5e812eb 100644 --- a/contrib/python/fonttools/fontTools/ufoLib/utils.py +++ b/contrib/python/fonttools/fontTools/ufoLib/utils.py @@ -1,5 +1,8 @@ -"""The module contains miscellaneous helpers. -It's not considered part of the public ufoLib API. +"""This module contains miscellaneous helpers. + +It is not considered part of the public ufoLib API. It does, however, +define the :py:obj:`.deprecated` decorator that is used elsewhere in +the module. """ import warnings diff --git a/contrib/python/fonttools/fontTools/unicodedata/Mirrored.py b/contrib/python/fonttools/fontTools/unicodedata/Mirrored.py new file mode 100644 index 00000000000..75b51a90123 --- /dev/null +++ b/contrib/python/fonttools/fontTools/unicodedata/Mirrored.py @@ -0,0 +1,446 @@ +# -*- coding: utf-8 -*- +# +# NOTE: The mappings in this file were generated from the command line: +# cat BidiMirroring.txt | grep "^[0-9A-F]" | sed "s/;//" | awk '{print " 0x"$1": 0x"$2","}' +# +# Source: http://www.unicode.org/Public/UNIDATA/BidiMirroring.txt +# License: http://unicode.org/copyright.html#License +# +# BidiMirroring-16.0.0.txt +# Date: 2024-01-30 +# © 2024 Unicode®, Inc. +# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. +# For terms of use and license, see https://www.unicode.org/terms_of_use.html +# +# Unicode Character Database +# For documentation, see https://www.unicode.org/reports/tr44/ +MIRRORED = { + 0x0028: 0x0029, + 0x0029: 0x0028, + 0x003C: 0x003E, + 0x003E: 0x003C, + 0x005B: 0x005D, + 0x005D: 0x005B, + 0x007B: 0x007D, + 0x007D: 0x007B, + 0x00AB: 0x00BB, + 0x00BB: 0x00AB, + 0x0F3A: 0x0F3B, + 0x0F3B: 0x0F3A, + 0x0F3C: 0x0F3D, + 0x0F3D: 0x0F3C, + 0x169B: 0x169C, + 0x169C: 0x169B, + 0x2039: 0x203A, + 0x203A: 0x2039, + 0x2045: 0x2046, + 0x2046: 0x2045, + 0x207D: 0x207E, + 0x207E: 0x207D, + 0x208D: 0x208E, + 0x208E: 0x208D, + 0x2208: 0x220B, + 0x2209: 0x220C, + 0x220A: 0x220D, + 0x220B: 0x2208, + 0x220C: 0x2209, + 0x220D: 0x220A, + 0x2215: 0x29F5, + 0x221F: 0x2BFE, + 0x2220: 0x29A3, + 0x2221: 0x299B, + 0x2222: 0x29A0, + 0x2224: 0x2AEE, + 0x223C: 0x223D, + 0x223D: 0x223C, + 0x2243: 0x22CD, + 0x2245: 0x224C, + 0x224C: 0x2245, + 0x2252: 0x2253, + 0x2253: 0x2252, + 0x2254: 0x2255, + 0x2255: 0x2254, + 0x2264: 0x2265, + 0x2265: 0x2264, + 0x2266: 0x2267, + 0x2267: 0x2266, + 0x2268: 0x2269, + 0x2269: 0x2268, + 0x226A: 0x226B, + 0x226B: 0x226A, + 0x226E: 0x226F, + 0x226F: 0x226E, + 0x2270: 0x2271, + 0x2271: 0x2270, + 0x2272: 0x2273, + 0x2273: 0x2272, + 0x2274: 0x2275, + 0x2275: 0x2274, + 0x2276: 0x2277, + 0x2277: 0x2276, + 0x2278: 0x2279, + 0x2279: 0x2278, + 0x227A: 0x227B, + 0x227B: 0x227A, + 0x227C: 0x227D, + 0x227D: 0x227C, + 0x227E: 0x227F, + 0x227F: 0x227E, + 0x2280: 0x2281, + 0x2281: 0x2280, + 0x2282: 0x2283, + 0x2283: 0x2282, + 0x2284: 0x2285, + 0x2285: 0x2284, + 0x2286: 0x2287, + 0x2287: 0x2286, + 0x2288: 0x2289, + 0x2289: 0x2288, + 0x228A: 0x228B, + 0x228B: 0x228A, + 0x228F: 0x2290, + 0x2290: 0x228F, + 0x2291: 0x2292, + 0x2292: 0x2291, + 0x2298: 0x29B8, + 0x22A2: 0x22A3, + 0x22A3: 0x22A2, + 0x22A6: 0x2ADE, + 0x22A8: 0x2AE4, + 0x22A9: 0x2AE3, + 0x22AB: 0x2AE5, + 0x22B0: 0x22B1, + 0x22B1: 0x22B0, + 0x22B2: 0x22B3, + 0x22B3: 0x22B2, + 0x22B4: 0x22B5, + 0x22B5: 0x22B4, + 0x22B6: 0x22B7, + 0x22B7: 0x22B6, + 0x22B8: 0x27DC, + 0x22C9: 0x22CA, + 0x22CA: 0x22C9, + 0x22CB: 0x22CC, + 0x22CC: 0x22CB, + 0x22CD: 0x2243, + 0x22D0: 0x22D1, + 0x22D1: 0x22D0, + 0x22D6: 0x22D7, + 0x22D7: 0x22D6, + 0x22D8: 0x22D9, + 0x22D9: 0x22D8, + 0x22DA: 0x22DB, + 0x22DB: 0x22DA, + 0x22DC: 0x22DD, + 0x22DD: 0x22DC, + 0x22DE: 0x22DF, + 0x22DF: 0x22DE, + 0x22E0: 0x22E1, + 0x22E1: 0x22E0, + 0x22E2: 0x22E3, + 0x22E3: 0x22E2, + 0x22E4: 0x22E5, + 0x22E5: 0x22E4, + 0x22E6: 0x22E7, + 0x22E7: 0x22E6, + 0x22E8: 0x22E9, + 0x22E9: 0x22E8, + 0x22EA: 0x22EB, + 0x22EB: 0x22EA, + 0x22EC: 0x22ED, + 0x22ED: 0x22EC, + 0x22F0: 0x22F1, + 0x22F1: 0x22F0, + 0x22F2: 0x22FA, + 0x22F3: 0x22FB, + 0x22F4: 0x22FC, + 0x22F6: 0x22FD, + 0x22F7: 0x22FE, + 0x22FA: 0x22F2, + 0x22FB: 0x22F3, + 0x22FC: 0x22F4, + 0x22FD: 0x22F6, + 0x22FE: 0x22F7, + 0x2308: 0x2309, + 0x2309: 0x2308, + 0x230A: 0x230B, + 0x230B: 0x230A, + 0x2329: 0x232A, + 0x232A: 0x2329, + 0x2768: 0x2769, + 0x2769: 0x2768, + 0x276A: 0x276B, + 0x276B: 0x276A, + 0x276C: 0x276D, + 0x276D: 0x276C, + 0x276E: 0x276F, + 0x276F: 0x276E, + 0x2770: 0x2771, + 0x2771: 0x2770, + 0x2772: 0x2773, + 0x2773: 0x2772, + 0x2774: 0x2775, + 0x2775: 0x2774, + 0x27C3: 0x27C4, + 0x27C4: 0x27C3, + 0x27C5: 0x27C6, + 0x27C6: 0x27C5, + 0x27C8: 0x27C9, + 0x27C9: 0x27C8, + 0x27CB: 0x27CD, + 0x27CD: 0x27CB, + 0x27D5: 0x27D6, + 0x27D6: 0x27D5, + 0x27DC: 0x22B8, + 0x27DD: 0x27DE, + 0x27DE: 0x27DD, + 0x27E2: 0x27E3, + 0x27E3: 0x27E2, + 0x27E4: 0x27E5, + 0x27E5: 0x27E4, + 0x27E6: 0x27E7, + 0x27E7: 0x27E6, + 0x27E8: 0x27E9, + 0x27E9: 0x27E8, + 0x27EA: 0x27EB, + 0x27EB: 0x27EA, + 0x27EC: 0x27ED, + 0x27ED: 0x27EC, + 0x27EE: 0x27EF, + 0x27EF: 0x27EE, + 0x2983: 0x2984, + 0x2984: 0x2983, + 0x2985: 0x2986, + 0x2986: 0x2985, + 0x2987: 0x2988, + 0x2988: 0x2987, + 0x2989: 0x298A, + 0x298A: 0x2989, + 0x298B: 0x298C, + 0x298C: 0x298B, + 0x298D: 0x2990, + 0x298E: 0x298F, + 0x298F: 0x298E, + 0x2990: 0x298D, + 0x2991: 0x2992, + 0x2992: 0x2991, + 0x2993: 0x2994, + 0x2994: 0x2993, + 0x2995: 0x2996, + 0x2996: 0x2995, + 0x2997: 0x2998, + 0x2998: 0x2997, + 0x299B: 0x2221, + 0x29A0: 0x2222, + 0x29A3: 0x2220, + 0x29A4: 0x29A5, + 0x29A5: 0x29A4, + 0x29A8: 0x29A9, + 0x29A9: 0x29A8, + 0x29AA: 0x29AB, + 0x29AB: 0x29AA, + 0x29AC: 0x29AD, + 0x29AD: 0x29AC, + 0x29AE: 0x29AF, + 0x29AF: 0x29AE, + 0x29B8: 0x2298, + 0x29C0: 0x29C1, + 0x29C1: 0x29C0, + 0x29C4: 0x29C5, + 0x29C5: 0x29C4, + 0x29CF: 0x29D0, + 0x29D0: 0x29CF, + 0x29D1: 0x29D2, + 0x29D2: 0x29D1, + 0x29D4: 0x29D5, + 0x29D5: 0x29D4, + 0x29D8: 0x29D9, + 0x29D9: 0x29D8, + 0x29DA: 0x29DB, + 0x29DB: 0x29DA, + 0x29E8: 0x29E9, + 0x29E9: 0x29E8, + 0x29F5: 0x2215, + 0x29F8: 0x29F9, + 0x29F9: 0x29F8, + 0x29FC: 0x29FD, + 0x29FD: 0x29FC, + 0x2A2B: 0x2A2C, + 0x2A2C: 0x2A2B, + 0x2A2D: 0x2A2E, + 0x2A2E: 0x2A2D, + 0x2A34: 0x2A35, + 0x2A35: 0x2A34, + 0x2A3C: 0x2A3D, + 0x2A3D: 0x2A3C, + 0x2A64: 0x2A65, + 0x2A65: 0x2A64, + 0x2A79: 0x2A7A, + 0x2A7A: 0x2A79, + 0x2A7B: 0x2A7C, + 0x2A7C: 0x2A7B, + 0x2A7D: 0x2A7E, + 0x2A7E: 0x2A7D, + 0x2A7F: 0x2A80, + 0x2A80: 0x2A7F, + 0x2A81: 0x2A82, + 0x2A82: 0x2A81, + 0x2A83: 0x2A84, + 0x2A84: 0x2A83, + 0x2A85: 0x2A86, + 0x2A86: 0x2A85, + 0x2A87: 0x2A88, + 0x2A88: 0x2A87, + 0x2A89: 0x2A8A, + 0x2A8A: 0x2A89, + 0x2A8B: 0x2A8C, + 0x2A8C: 0x2A8B, + 0x2A8D: 0x2A8E, + 0x2A8E: 0x2A8D, + 0x2A8F: 0x2A90, + 0x2A90: 0x2A8F, + 0x2A91: 0x2A92, + 0x2A92: 0x2A91, + 0x2A93: 0x2A94, + 0x2A94: 0x2A93, + 0x2A95: 0x2A96, + 0x2A96: 0x2A95, + 0x2A97: 0x2A98, + 0x2A98: 0x2A97, + 0x2A99: 0x2A9A, + 0x2A9A: 0x2A99, + 0x2A9B: 0x2A9C, + 0x2A9C: 0x2A9B, + 0x2A9D: 0x2A9E, + 0x2A9E: 0x2A9D, + 0x2A9F: 0x2AA0, + 0x2AA0: 0x2A9F, + 0x2AA1: 0x2AA2, + 0x2AA2: 0x2AA1, + 0x2AA6: 0x2AA7, + 0x2AA7: 0x2AA6, + 0x2AA8: 0x2AA9, + 0x2AA9: 0x2AA8, + 0x2AAA: 0x2AAB, + 0x2AAB: 0x2AAA, + 0x2AAC: 0x2AAD, + 0x2AAD: 0x2AAC, + 0x2AAF: 0x2AB0, + 0x2AB0: 0x2AAF, + 0x2AB1: 0x2AB2, + 0x2AB2: 0x2AB1, + 0x2AB3: 0x2AB4, + 0x2AB4: 0x2AB3, + 0x2AB5: 0x2AB6, + 0x2AB6: 0x2AB5, + 0x2AB7: 0x2AB8, + 0x2AB8: 0x2AB7, + 0x2AB9: 0x2ABA, + 0x2ABA: 0x2AB9, + 0x2ABB: 0x2ABC, + 0x2ABC: 0x2ABB, + 0x2ABD: 0x2ABE, + 0x2ABE: 0x2ABD, + 0x2ABF: 0x2AC0, + 0x2AC0: 0x2ABF, + 0x2AC1: 0x2AC2, + 0x2AC2: 0x2AC1, + 0x2AC3: 0x2AC4, + 0x2AC4: 0x2AC3, + 0x2AC5: 0x2AC6, + 0x2AC6: 0x2AC5, + 0x2AC7: 0x2AC8, + 0x2AC8: 0x2AC7, + 0x2AC9: 0x2ACA, + 0x2ACA: 0x2AC9, + 0x2ACB: 0x2ACC, + 0x2ACC: 0x2ACB, + 0x2ACD: 0x2ACE, + 0x2ACE: 0x2ACD, + 0x2ACF: 0x2AD0, + 0x2AD0: 0x2ACF, + 0x2AD1: 0x2AD2, + 0x2AD2: 0x2AD1, + 0x2AD3: 0x2AD4, + 0x2AD4: 0x2AD3, + 0x2AD5: 0x2AD6, + 0x2AD6: 0x2AD5, + 0x2ADE: 0x22A6, + 0x2AE3: 0x22A9, + 0x2AE4: 0x22A8, + 0x2AE5: 0x22AB, + 0x2AEC: 0x2AED, + 0x2AED: 0x2AEC, + 0x2AEE: 0x2224, + 0x2AF7: 0x2AF8, + 0x2AF8: 0x2AF7, + 0x2AF9: 0x2AFA, + 0x2AFA: 0x2AF9, + 0x2BFE: 0x221F, + 0x2E02: 0x2E03, + 0x2E03: 0x2E02, + 0x2E04: 0x2E05, + 0x2E05: 0x2E04, + 0x2E09: 0x2E0A, + 0x2E0A: 0x2E09, + 0x2E0C: 0x2E0D, + 0x2E0D: 0x2E0C, + 0x2E1C: 0x2E1D, + 0x2E1D: 0x2E1C, + 0x2E20: 0x2E21, + 0x2E21: 0x2E20, + 0x2E22: 0x2E23, + 0x2E23: 0x2E22, + 0x2E24: 0x2E25, + 0x2E25: 0x2E24, + 0x2E26: 0x2E27, + 0x2E27: 0x2E26, + 0x2E28: 0x2E29, + 0x2E29: 0x2E28, + 0x2E55: 0x2E56, + 0x2E56: 0x2E55, + 0x2E57: 0x2E58, + 0x2E58: 0x2E57, + 0x2E59: 0x2E5A, + 0x2E5A: 0x2E59, + 0x2E5B: 0x2E5C, + 0x2E5C: 0x2E5B, + 0x3008: 0x3009, + 0x3009: 0x3008, + 0x300A: 0x300B, + 0x300B: 0x300A, + 0x300C: 0x300D, + 0x300D: 0x300C, + 0x300E: 0x300F, + 0x300F: 0x300E, + 0x3010: 0x3011, + 0x3011: 0x3010, + 0x3014: 0x3015, + 0x3015: 0x3014, + 0x3016: 0x3017, + 0x3017: 0x3016, + 0x3018: 0x3019, + 0x3019: 0x3018, + 0x301A: 0x301B, + 0x301B: 0x301A, + 0xFE59: 0xFE5A, + 0xFE5A: 0xFE59, + 0xFE5B: 0xFE5C, + 0xFE5C: 0xFE5B, + 0xFE5D: 0xFE5E, + 0xFE5E: 0xFE5D, + 0xFE64: 0xFE65, + 0xFE65: 0xFE64, + 0xFF08: 0xFF09, + 0xFF09: 0xFF08, + 0xFF1C: 0xFF1E, + 0xFF1E: 0xFF1C, + 0xFF3B: 0xFF3D, + 0xFF3D: 0xFF3B, + 0xFF5B: 0xFF5D, + 0xFF5D: 0xFF5B, + 0xFF5F: 0xFF60, + 0xFF60: 0xFF5F, + 0xFF62: 0xFF63, + 0xFF63: 0xFF62, +} diff --git a/contrib/python/fonttools/fontTools/unicodedata/__init__.py b/contrib/python/fonttools/fontTools/unicodedata/__init__.py index edae44ec718..1adb07d2896 100644 --- a/contrib/python/fonttools/fontTools/unicodedata/__init__.py +++ b/contrib/python/fonttools/fontTools/unicodedata/__init__.py @@ -15,8 +15,7 @@ except ImportError: # pragma: no cover # fall back to built-in unicodedata (possibly outdated) from unicodedata import * -from . import Blocks, Scripts, ScriptExtensions, OTTags - +from . import Blocks, Mirrored, Scripts, ScriptExtensions, OTTags __all__ = [ # names from built-in unicodedata module @@ -46,6 +45,11 @@ __all__ = [ ] +def mirrored(code): + """If code (unicode codepoint) has a mirrored version returns it, otherwise None.""" + return Mirrored.MIRRORED.get(code) + + def script(char): """Return the four-letter script code assigned to the Unicode character 'char' as string. diff --git a/contrib/python/fonttools/fontTools/varLib/__init__.py b/contrib/python/fonttools/fontTools/varLib/__init__.py index f3d8d59dc5a..e3c00d73fa9 100644 --- a/contrib/python/fonttools/fontTools/varLib/__init__.py +++ b/contrib/python/fonttools/fontTools/varLib/__init__.py @@ -555,6 +555,8 @@ def _add_VHVAR(font, axisTags, tableFields, getAdvanceMetrics): varData.addItem(vhAdvanceDeltasAndSupports[glyphName][0], round=noRound) varData.optimize() directStore = builder.buildVarStore(varTupleList, [varData]) + # remove unused regions from VarRegionList + directStore.prune_regions() # Build optimized indirect mapping storeBuilder = varStore.OnlineVarStoreBuilder(axisTags) diff --git a/contrib/python/fonttools/fontTools/voltLib/__main__.py b/contrib/python/fonttools/fontTools/voltLib/__main__.py new file mode 100644 index 00000000000..aa2c3b3e610 --- /dev/null +++ b/contrib/python/fonttools/fontTools/voltLib/__main__.py @@ -0,0 +1,206 @@ +import argparse +import logging +import sys +from io import StringIO +from pathlib import Path + +from fontTools import configLogger +from fontTools.feaLib.builder import addOpenTypeFeaturesFromString +from fontTools.feaLib.error import FeatureLibError +from fontTools.feaLib.lexer import Lexer +from fontTools.misc.cliTools import makeOutputFileName +from fontTools.ttLib import TTFont, TTLibError +from fontTools.voltLib.parser import Parser +from fontTools.voltLib.voltToFea import TABLES, VoltToFea + +log = logging.getLogger("fontTools.feaLib") + +SUPPORTED_TABLES = TABLES + ["cmap"] + + +def invalid_fea_glyph_name(name): + """Check if the glyph name is valid according to FEA syntax.""" + if name[0] not in Lexer.CHAR_NAME_START_: + return True + if any(c not in Lexer.CHAR_NAME_CONTINUATION_ for c in name[1:]): + return True + return False + + +def sanitize_glyph_name(name): + """Sanitize the glyph name to ensure it is valid according to FEA syntax.""" + sanitized = "" + for i, c in enumerate(name): + if i == 0 and c not in Lexer.CHAR_NAME_START_: + sanitized += "a" + c + elif c not in Lexer.CHAR_NAME_CONTINUATION_: + sanitized += "_" + else: + sanitized += c + + return sanitized + + +def main(args=None): + """Build tables from a MS VOLT project into an OTF font""" + parser = argparse.ArgumentParser( + description="Use fontTools to compile MS VOLT projects." + ) + parser.add_argument( + "input", + metavar="INPUT", + help="Path to the input font/VTP file to process", + type=Path, + ) + parser.add_argument( + "-f", + "--font", + metavar="INPUT_FONT", + help="Path to the input font (if INPUT is a VTP file)", + type=Path, + ) + parser.add_argument( + "-o", + "--output", + dest="output", + metavar="OUTPUT", + help="Path to the output font.", + type=Path, + ) + parser.add_argument( + "-t", + "--tables", + metavar="TABLE_TAG", + choices=SUPPORTED_TABLES, + nargs="+", + help="Specify the table(s) to be built.", + ) + parser.add_argument( + "-F", + "--debug-feature-file", + help="Write the generated feature file to disk.", + action="store_true", + ) + parser.add_argument( + "--ship", + help="Remove source VOLT tables from output font.", + action="store_true", + ) + parser.add_argument( + "-v", + "--verbose", + help="Increase the logger verbosity. Multiple -v options are allowed.", + action="count", + default=0, + ) + parser.add_argument( + "-T", + "--traceback", + help="show traceback for exceptions.", + action="store_true", + ) + options = parser.parse_args(args) + + levels = ["WARNING", "INFO", "DEBUG"] + configLogger(level=levels[min(len(levels) - 1, options.verbose)]) + + output_font = options.output or Path( + makeOutputFileName(options.font or options.input) + ) + log.info(f"Compiling MS VOLT to '{output_font}'") + + file_or_path = options.input + font = None + + # If the input is a font file, extract the VOLT data from the "TSIV" table + try: + font = TTFont(file_or_path) + if "TSIV" in font: + file_or_path = StringIO(font["TSIV"].data.decode("utf-8")) + else: + log.error('"TSIV" table is missing') + return 1 + except TTLibError: + pass + + # If input is not a font file, the font must be provided + if font is None: + if not options.font: + log.error("Please provide an input font") + return 1 + font = TTFont(options.font) + + # FEA syntax does not allow some glyph names that VOLT accepts, so if we + # found such glyph name we will temporarily rename such glyphs. + glyphOrder = font.getGlyphOrder() + tempGlyphOrder = None + if any(invalid_fea_glyph_name(n) for n in glyphOrder): + tempGlyphOrder = [] + for n in glyphOrder: + if invalid_fea_glyph_name(n): + n = sanitize_glyph_name(n) + existing = set(tempGlyphOrder) | set(glyphOrder) + while n in existing: + n = "a" + n + tempGlyphOrder.append(n) + font.setGlyphOrder(tempGlyphOrder) + + doc = Parser(file_or_path).parse() + + log.info("Converting VTP data to FEA") + converter = VoltToFea(doc, font) + try: + fea = converter.convert(options.tables, ignore_unsupported_settings=True) + except NotImplementedError as e: + if options.traceback: + raise + location = getattr(e.args[0], "location", None) + message = f'"{e}" is not supported' + if location: + path, line, column = location + log.error(f"{path}:{line}:{column}: {message}") + else: + log.error(message) + return 1 + + fea_filename = options.input + if options.debug_feature_file: + fea_filename = output_font.with_suffix(".fea") + log.info(f"Writing FEA to '{fea_filename}'") + with open(fea_filename, "w") as fp: + fp.write(fea) + + log.info("Compiling FEA to OpenType tables") + try: + addOpenTypeFeaturesFromString( + font, + fea, + filename=fea_filename, + tables=options.tables, + ) + except FeatureLibError as e: + if options.traceback: + raise + log.error(e) + return 1 + + if options.ship: + for tag in ["TSIV", "TSIS", "TSIP", "TSID"]: + if tag in font: + del font[tag] + + # Restore original glyph names. + if tempGlyphOrder: + import io + + f = io.BytesIO() + font.save(f) + font = TTFont(f) + font.setGlyphOrder(glyphOrder) + font["post"].extraNames = [] + + font.save(output_font) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/contrib/python/fonttools/fontTools/voltLib/ast.py b/contrib/python/fonttools/fontTools/voltLib/ast.py index 82c2cca8b7f..dba8f4d45b3 100644 --- a/contrib/python/fonttools/fontTools/voltLib/ast.py +++ b/contrib/python/fonttools/fontTools/voltLib/ast.py @@ -317,6 +317,10 @@ class SubstitutionLigatureDefinition(SubstitutionDefinition): pass +class SubstitutionAlternateDefinition(SubstitutionDefinition): + pass + + class SubstitutionReverseChainingSingleDefinition(SubstitutionDefinition): pass diff --git a/contrib/python/fonttools/fontTools/voltLib/parser.py b/contrib/python/fonttools/fontTools/voltLib/parser.py index 1fa6b11d027..31a76e69bff 100644 --- a/contrib/python/fonttools/fontTools/voltLib/parser.py +++ b/contrib/python/fonttools/fontTools/voltLib/parser.py @@ -313,19 +313,27 @@ class Parser(object): self.expect_keyword_("END_SUBSTITUTION") max_src = max([len(cov) for cov in src]) max_dest = max([len(cov) for cov in dest]) + # many to many or mixed is invalid - if (max_src > 1 and max_dest > 1) or ( - reversal and (max_src > 1 or max_dest > 1) - ): + if max_src > 1 and max_dest > 1: raise VoltLibError("Invalid substitution type", location) + mapping = dict(zip(tuple(src), tuple(dest))) if max_src == 1 and max_dest == 1: - if reversal: - sub = ast.SubstitutionReverseChainingSingleDefinition( - mapping, location=location - ) + # Alternate substitutions are represented by adding multiple + # substitutions for the same glyph, so we detect that here + glyphs = [x.glyphSet() for cov in src for x in cov] # flatten src + if len(set(glyphs)) != len(glyphs): # src has duplicates + sub = ast.SubstitutionAlternateDefinition(mapping, location=location) else: - sub = ast.SubstitutionSingleDefinition(mapping, location=location) + if reversal: + # Reversal is valid only for single glyph substitutions + # and VOLT ignores it otherwise. + sub = ast.SubstitutionReverseChainingSingleDefinition( + mapping, location=location + ) + else: + sub = ast.SubstitutionSingleDefinition(mapping, location=location) elif max_src == 1 and max_dest > 1: sub = ast.SubstitutionMultipleDefinition(mapping, location=location) elif max_src > 1 and max_dest == 1: diff --git a/contrib/python/fonttools/fontTools/voltLib/voltToFea.py b/contrib/python/fonttools/fontTools/voltLib/voltToFea.py index c77d5ad1111..d552f4b52a8 100644 --- a/contrib/python/fonttools/fontTools/voltLib/voltToFea.py +++ b/contrib/python/fonttools/fontTools/voltLib/voltToFea.py @@ -46,6 +46,7 @@ Limitations import logging import re from io import StringIO +from graphlib import TopologicalSorter from fontTools.feaLib import ast from fontTools.ttLib import TTFont, TTLibError @@ -57,32 +58,39 @@ log = logging.getLogger("fontTools.voltLib.voltToFea") TABLES = ["GDEF", "GSUB", "GPOS"] -class MarkClassDefinition(ast.MarkClassDefinition): - def asFea(self, indent=""): - res = "" - if not getattr(self, "used", False): - res += "#" - res += ast.MarkClassDefinition.asFea(self, indent) - return res - - -# For sorting voltLib.ast.GlyphDefinition, see its use below. -class Group: - def __init__(self, group): - self.name = group.name.lower() - self.groups = [ - x.group.lower() for x in group.enum.enum if isinstance(x, VAst.GroupName) +def _flatten_group(group): + ret = [] + if isinstance(group, (tuple, list)): + for item in group: + ret.extend(_flatten_group(item)) + elif hasattr(group, "enum"): + ret.extend(_flatten_group(group.enum)) + else: + ret.append(group) + return ret + + +# Topologically sort of group definitions to ensure that all groups are defined +# before they are referenced. This is necessary because FEA requires it but +# VOLT does not, see below. +def sort_groups(groups): + group_map = {group.name.lower(): group for group in groups} + graph = { + group.name.lower(): [ + x.group.lower() + for x in _flatten_group(group) + if isinstance(x, VAst.GroupName) ] + for group in groups + } + sorter = TopologicalSorter(graph) + return [group_map[name] for name in sorter.static_order()] + - def __lt__(self, other): - if self.name in other.groups: - return True - if other.name in self.groups: - return False - if self.groups and not other.groups: - return False - if not self.groups and other.groups: - return True +class Lookup(ast.LookupBlock): + def __init__(self, name, use_extension=False, location=None): + super().__init__(name, use_extension, location) + self.chained = [] class VoltToFea: @@ -90,7 +98,10 @@ class VoltToFea: _NOT_CLASS_NAME_RE = re.compile(r"[^A-Za-z_0-9.\-]") def __init__(self, file_or_path, font=None): - self._file_or_path = file_or_path + if isinstance(file_or_path, VAst.VoltFile): + self._doc, self._file_or_path = file_or_path, None + else: + self._doc, self._file_or_path = None, file_or_path self._font = font self._glyph_map = {} @@ -128,23 +139,26 @@ class VoltToFea: self._class_names[name] = res return self._class_names[name] - def _collectStatements(self, doc, tables): + def _collectStatements(self, doc, tables, ignore_unsupported_settings=False): + # Collect glyph difinitions first, as we need them to map VOLT glyph names to font glyph name. + for statement in doc.statements: + if isinstance(statement, VAst.GlyphDefinition): + self._glyphDefinition(statement) + # Collect and sort group definitions first, to make sure a group # definition that references other groups comes after them since VOLT # does not enforce such ordering, and feature file require it. groups = [s for s in doc.statements if isinstance(s, VAst.GroupDefinition)] - for statement in sorted(groups, key=lambda x: Group(x)): - self._groupDefinition(statement) + for group in sort_groups(groups): + self._groupDefinition(group) for statement in doc.statements: - if isinstance(statement, VAst.GlyphDefinition): - self._glyphDefinition(statement) - elif isinstance(statement, VAst.AnchorDefinition): + if isinstance(statement, VAst.AnchorDefinition): if "GPOS" in tables: self._anchorDefinition(statement) elif isinstance(statement, VAst.SettingDefinition): - self._settingDefinition(statement) - elif isinstance(statement, VAst.GroupDefinition): + self._settingDefinition(statement, ignore_unsupported_settings) + elif isinstance(statement, (VAst.GlyphDefinition, VAst.GroupDefinition)): pass # Handled above elif isinstance(statement, VAst.ScriptDefinition): self._scriptDefinition(statement) @@ -176,35 +190,57 @@ class VoltToFea: if self._lookups: statements.append(ast.Comment("\n# Lookups")) for lookup in self._lookups.values(): - statements.extend(getattr(lookup, "targets", [])) + statements.extend(lookup.chained) statements.append(lookup) # Prune features features = self._features.copy() - for ftag in features: - scripts = features[ftag] - for stag in scripts: - langs = scripts[stag] - for ltag in langs: - langs[ltag] = [l for l in langs[ltag] if l.lower() in self._lookups] - scripts[stag] = {t: l for t, l in langs.items() if l} - features[ftag] = {t: s for t, s in scripts.items() if s} + for feature_tag in features: + scripts = features[feature_tag] + for script_tag in scripts: + langs = scripts[script_tag] + for language_tag in langs: + langs[language_tag] = [ + l for l in langs[language_tag] if l.lower() in self._lookups + ] + scripts[script_tag] = {t: l for t, l in langs.items() if l} + features[feature_tag] = {t: s for t, s in scripts.items() if s} features = {t: f for t, f in features.items() if f} if features: statements.append(ast.Comment("# Features")) - for ftag, scripts in features.items(): - feature = ast.FeatureBlock(ftag) - stags = sorted(scripts, key=lambda k: 0 if k == "DFLT" else 1) - for stag in stags: - feature.statements.append(ast.ScriptStatement(stag)) - ltags = sorted(scripts[stag], key=lambda k: 0 if k == "dflt" else 1) - for ltag in ltags: - include_default = True if ltag == "dflt" else False - feature.statements.append( - ast.LanguageStatement(ltag, include_default=include_default) + for feature_tag, scripts in features.items(): + feature = ast.FeatureBlock(feature_tag) + script_tags = sorted(scripts, key=lambda k: 0 if k == "DFLT" else 1) + if feature_tag == "aalt" and len(script_tags) > 1: + log.warning( + "FEA syntax does not allow script statements in 'aalt' feature, " + "so only lookups from the first script will be included." + ) + script_tags = script_tags[:1] + for script_tag in script_tags: + if feature_tag != "aalt": + feature.statements.append(ast.ScriptStatement(script_tag)) + language_tags = sorted( + scripts[script_tag], + key=lambda k: 0 if k == "dflt" else 1, + ) + if feature_tag == "aalt" and len(language_tags) > 1: + log.warning( + "FEA syntax does not allow language statements in 'aalt' feature, " + "so only lookups from the first language will be included." ) - for name in scripts[stag][ltag]: + language_tags = language_tags[:1] + for language_tag in language_tags: + if feature_tag != "aalt": + include_default = True if language_tag == "dflt" else False + feature.statements.append( + ast.LanguageStatement( + language_tag.ljust(4), + include_default=include_default, + ) + ) + for name in scripts[script_tag][language_tag]: lookup = self._lookups[name.lower()] lookupref = ast.LookupReferenceStatement(lookup) feature.statements.append(lookupref) @@ -227,15 +263,17 @@ class VoltToFea: return doc - def convert(self, tables=None): - doc = VoltParser(self._file_or_path).parse() + def convert(self, tables=None, ignore_unsupported_settings=False): + if self._doc is None: + self._doc = VoltParser(self._file_or_path).parse() + doc = self._doc if tables is None: tables = TABLES if self._font is not None: self._glyph_order = self._font.getGlyphOrder() - self._collectStatements(doc, tables) + self._collectStatements(doc, tables, ignore_unsupported_settings) fea = self._buildFeatureFile(tables) return fea.asFea() @@ -253,7 +291,13 @@ class VoltToFea: name = group return ast.GlyphClassName(self._glyphclasses[name.lower()]) - def _coverage(self, coverage): + def _glyphSet(self, item): + return [ + (self._glyphName(x) if isinstance(x, (str, VAst.GlyphName)) else x) + for x in item.glyphSet() + ] + + def _coverage(self, coverage, flatten=False): items = [] for item in coverage: if isinstance(item, VAst.GlyphName): @@ -261,31 +305,38 @@ class VoltToFea: elif isinstance(item, VAst.GroupName): items.append(self._groupName(item)) elif isinstance(item, VAst.Enum): - items.append(self._enum(item)) + item = self._coverage(item.enum, flatten=True) + if flatten: + items.extend(item) + else: + items.append(ast.GlyphClass(item)) elif isinstance(item, VAst.Range): - items.append((item.start, item.end)) + item = self._glyphSet(item) + if flatten: + items.extend(item) + else: + items.append(ast.GlyphClass(item)) else: raise NotImplementedError(item) return items - def _enum(self, enum): - return ast.GlyphClass(self._coverage(enum.enum)) - def _context(self, context): out = [] for item in context: - coverage = self._coverage(item) - if not isinstance(coverage, (tuple, list)): - coverage = [coverage] - out.extend(coverage) + coverage = self._coverage(item, flatten=True) + if len(coverage) > 1: + coverage = ast.GlyphClass(coverage) + else: + coverage = coverage[0] + out.append(coverage) return out def _groupDefinition(self, group): name = self._className(group.name) - glyphs = self._enum(group.enum) - glyphclass = ast.GlyphClassDefinition(name, glyphs) - - self._glyphclasses[group.name.lower()] = glyphclass + glyphs = self._coverage(group.enum.enum, flatten=True) + glyphclass = ast.GlyphClass(glyphs) + classdef = ast.GlyphClassDefinition(name, glyphclass) + self._glyphclasses[group.name.lower()] = classdef def _glyphDefinition(self, glyph): try: @@ -317,10 +368,10 @@ class VoltToFea: assert ltag not in self._features[ftag][stag] self._features[ftag][stag][ltag] = lookups.keys() - def _settingDefinition(self, setting): + def _settingDefinition(self, setting, ignore_unsupported=False): if setting.name.startswith("COMPILER_"): self._settings[setting.name] = setting.value - else: + elif not ignore_unsupported: log.warning(f"Unsupported setting ignored: {setting.name}") def _adjustment(self, adjustment): @@ -358,18 +409,15 @@ class VoltToFea: glyphname = anchordef.glyph_name anchor = self._anchor(anchordef.pos) + if glyphname not in self._anchors: + self._anchors[glyphname] = {} if anchorname.startswith("MARK_"): - name = "_".join(anchorname.split("_")[1:]) - markclass = ast.MarkClass(self._className(name)) - glyph = self._glyphName(glyphname) - markdef = MarkClassDefinition(markclass, anchor, glyph) - self._markclasses[(glyphname, anchorname)] = markdef + anchorname = anchorname[:5] + anchorname[5:].lower() else: - if glyphname not in self._anchors: - self._anchors[glyphname] = {} - if anchorname not in self._anchors[glyphname]: - self._anchors[glyphname][anchorname] = {} - self._anchors[glyphname][anchorname][anchordef.component] = anchor + anchorname = anchorname.lower() + if anchorname not in self._anchors[glyphname]: + self._anchors[glyphname][anchorname] = {} + self._anchors[glyphname][anchorname][anchordef.component] = anchor def _gposLookup(self, lookup, fealookup): statements = fealookup.statements @@ -408,43 +456,66 @@ class VoltToFea: ) elif isinstance(pos, VAst.PositionAttachDefinition): anchors = {} - for marks, classname in pos.coverage_to: - for mark in marks: - # Set actually used mark classes. Basically a hack to get - # around the feature file syntax limitation of making mark - # classes global and not allowing mark positioning to - # specify mark coverage. - for name in mark.glyphSet(): - key = (name, "MARK_" + classname) - self._markclasses[key].used = True - markclass = ast.MarkClass(self._className(classname)) + allmarks = set() + for coverage, anchorname in pos.coverage_to: + # In feature files mark classes are global, but in VOLT they + # are defined per-lookup. If we output mark class definitions + # for all marks that use a given anchor, we might end up with a + # mark used in two different classes in the same lookup, which + # is causes feature file compilation error. + # At the expense of uglier feature code, we make the mark class + # name by appending the current lookup name not the anchor + # name, and output mark class definitions only for marks used + # in this lookup. + classname = self._className(f"{anchorname}.{lookup.name}") + markclass = ast.MarkClass(classname) + + # Anchor names are case-insensitive in VOLT + anchorname = anchorname.lower() + + # We might still end in marks used in two different anchor + # classes, so we filter out already used marks. + marks = set() + for mark in coverage: + marks.update(mark.glyphSet()) + if not marks.isdisjoint(allmarks): + marks.difference_update(allmarks) + if not marks: + continue + allmarks.update(marks) + + for glyphname in marks: + glyph = self._glyphName(glyphname) + anchor = self._anchors[glyphname][f"MARK_{anchorname}"][1] + markdef = ast.MarkClassDefinition(markclass, anchor, glyph) + self._markclasses[(glyphname, classname)] = markdef + for base in pos.coverage: for name in base.glyphSet(): if name not in anchors: anchors[name] = [] - if classname not in anchors[name]: - anchors[name].append(classname) + if (anchorname, classname) not in anchors[name]: + anchors[name].append((anchorname, classname)) + is_ligature = all(n in self._ligatures for n in anchors) + is_mark = all(n in self._marks for n in anchors) for name in anchors: components = 1 - if name in self._ligatures: + if is_ligature: components = self._ligatures[name] - marks = [] - for mark in anchors[name]: - markclass = ast.MarkClass(self._className(mark)) + marks = [[] for _ in range(components)] + for mark, classname in anchors[name]: + markclass = ast.MarkClass(classname) for component in range(1, components + 1): - if len(marks) < component: - marks.append([]) - anchor = None if component in self._anchors[name][mark]: anchor = self._anchors[name][mark][component] - marks[component - 1].append((anchor, markclass)) + marks[component - 1].append((anchor, markclass)) base = self._glyphName(name) - if name in self._marks: + if is_mark: mark = ast.MarkMarkPosStatement(base, marks[0]) - elif name in self._ligatures: + elif is_ligature: mark = ast.MarkLigPosStatement(base, marks) else: mark = ast.MarkBasePosStatement(base, marks[0]) @@ -481,13 +552,9 @@ class VoltToFea: else: raise NotImplementedError(pos) - def _gposContextLookup( - self, lookup, prefix, suffix, ignore, fealookup, targetlookup - ): + def _gposContextLookup(self, lookup, prefix, suffix, ignore, fealookup, chained): statements = fealookup.statements - assert not lookup.reversal - pos = lookup.pos if isinstance(pos, VAst.PositionAdjustPairDefinition): for (idx1, idx2), (pos1, pos2) in pos.adjust_pair.items(): @@ -500,79 +567,181 @@ class VoltToFea: if ignore: statement = ast.IgnorePosStatement([(prefix, glyphs, suffix)]) else: - lookups = (targetlookup, targetlookup) statement = ast.ChainContextPosStatement( - prefix, glyphs, suffix, lookups + prefix, glyphs, suffix, [chained, chained] ) statements.append(statement) elif isinstance(pos, VAst.PositionAdjustSingleDefinition): glyphs = [ast.GlyphClass()] - for a, b in pos.adjust_single: - glyph = self._coverage(a) - glyphs[0].extend(glyph) + for a, _ in pos.adjust_single: + glyphs[0].extend(self._coverage(a, flatten=True)) if ignore: statement = ast.IgnorePosStatement([(prefix, glyphs, suffix)]) else: statement = ast.ChainContextPosStatement( - prefix, glyphs, suffix, [targetlookup] + prefix, glyphs, suffix, [chained] ) statements.append(statement) elif isinstance(pos, VAst.PositionAttachDefinition): glyphs = [ast.GlyphClass()] for coverage, _ in pos.coverage_to: - glyphs[0].extend(self._coverage(coverage)) + glyphs[0].extend(self._coverage(coverage, flatten=True)) if ignore: statement = ast.IgnorePosStatement([(prefix, glyphs, suffix)]) else: statement = ast.ChainContextPosStatement( - prefix, glyphs, suffix, [targetlookup] + prefix, glyphs, suffix, [chained] ) statements.append(statement) else: raise NotImplementedError(pos) - def _gsubLookup(self, lookup, prefix, suffix, ignore, chain, fealookup): + def _gsubLookup(self, lookup, fealookup): statements = fealookup.statements sub = lookup.sub + + # Alternate substitutions are represented by adding multiple + # substitutions for the same glyph, so we need to collect them into one + # to many mapping. + if isinstance(sub, VAst.SubstitutionAlternateDefinition): + alternates = {} + for key, val in sub.mapping.items(): + if not key or not val: + path, line, column = sub.location + log.warning(f"{path}:{line}:{column}: Ignoring empty substitution") + continue + glyphs = self._coverage(key) + replacements = self._coverage(val) + assert len(glyphs) == 1 + for src_glyph, repl_glyph in zip( + glyphs[0].glyphSet(), replacements[0].glyphSet() + ): + alternates.setdefault(str(self._glyphName(src_glyph)), []).append( + str(self._glyphName(repl_glyph)) + ) + + for glyph, replacements in alternates.items(): + statement = ast.AlternateSubstStatement( + [], glyph, [], ast.GlyphClass(replacements) + ) + statements.append(statement) + return + for key, val in sub.mapping.items(): if not key or not val: path, line, column = sub.location log.warning(f"{path}:{line}:{column}: Ignoring empty substitution") continue - statement = None glyphs = self._coverage(key) replacements = self._coverage(val) - if ignore: - chain_context = (prefix, glyphs, suffix) - statement = ast.IgnoreSubstStatement([chain_context]) - elif isinstance(sub, VAst.SubstitutionSingleDefinition): + if isinstance(sub, VAst.SubstitutionSingleDefinition): assert len(glyphs) == 1 assert len(replacements) == 1 - statement = ast.SingleSubstStatement( - glyphs, replacements, prefix, suffix, chain + statements.append( + ast.SingleSubstStatement(glyphs, replacements, [], [], False) ) elif isinstance(sub, VAst.SubstitutionReverseChainingSingleDefinition): - assert len(glyphs) == 1 - assert len(replacements) == 1 - statement = ast.ReverseChainSingleSubstStatement( - prefix, suffix, glyphs, replacements - ) + # This is handled in gsubContextLookup() + pass elif isinstance(sub, VAst.SubstitutionMultipleDefinition): assert len(glyphs) == 1 - statement = ast.MultipleSubstStatement( - prefix, glyphs[0], suffix, replacements, chain + statements.append( + ast.MultipleSubstStatement([], glyphs[0], [], replacements) ) elif isinstance(sub, VAst.SubstitutionLigatureDefinition): assert len(replacements) == 1 statement = ast.LigatureSubstStatement( - prefix, glyphs, suffix, replacements[0], chain + [], glyphs, [], replacements[0], False ) + + # If any of the input glyphs is a group, we need to + # explode the substitution into multiple ligature substitutions + # since feature file syntax does not support classes in + # ligature substitutions. + n = max(len(x.glyphSet()) for x in glyphs) + if n > 1: + # All input should either be groups of the same length or single glyphs + assert all(len(x.glyphSet()) in (n, 1) for x in glyphs) + glyphs = [x.glyphSet() for x in glyphs] + glyphs = [([x[0]] * n if len(x) == 1 else x) for x in glyphs] + + # In this case ligature replacements must be a group of the same length + # as the input groups, or a single glyph. VOLT + # allows the replacement glyphs to be longer and truncates them. + # So well allow that and zip() below will do the truncation + # for us. + replacement = replacements[0].glyphSet() + if len(replacement) == 1: + replacement = [replacement[0]] * n + assert len(replacement) >= n + + # Add the unexploded statement commented out for reference. + statements.append(ast.Comment(f"# {statement}")) + + for zipped in zip(*glyphs, replacement): + zipped = [self._glyphName(x) for x in zipped] + statements.append( + ast.LigatureSubstStatement( + [], zipped[:-1], [], zipped[-1], False + ) + ) + else: + statements.append(statement) else: raise NotImplementedError(sub) - statements.append(statement) + + def _gsubContextLookup(self, lookup, prefix, suffix, ignore, fealookup, chained): + statements = fealookup.statements + + sub = lookup.sub + + if isinstance(sub, VAst.SubstitutionReverseChainingSingleDefinition): + # Reverse substitutions is a special case, it can’t use chained lookups. + for key, val in sub.mapping.items(): + if not key or not val: + path, line, column = sub.location + log.warning(f"{path}:{line}:{column}: Ignoring empty substitution") + continue + glyphs = self._coverage(key) + replacements = self._coverage(val) + statements.append( + ast.ReverseChainSingleSubstStatement( + prefix, suffix, glyphs, replacements + ) + ) + fealookup.chained = [] + return + + if not isinstance( + sub, + ( + VAst.SubstitutionSingleDefinition, + VAst.SubstitutionMultipleDefinition, + VAst.SubstitutionLigatureDefinition, + VAst.SubstitutionAlternateDefinition, + ), + ): + raise NotImplementedError(type(sub)) + + glyphs = [] + for key, val in sub.mapping.items(): + if not key or not val: + path, line, column = sub.location + log.warning(f"{path}:{line}:{column}: Ignoring empty substitution") + continue + glyphs.extend(self._coverage(key, flatten=True)) + + if len(glyphs) > 1: + glyphs = [ast.GlyphClass(glyphs)] + if ignore: + statements.append(ast.IgnoreSubstStatement([(prefix, glyphs, suffix)])) + else: + statements.append( + ast.ChainContextSubstStatement(prefix, glyphs, suffix, [chained]) + ) def _lookupDefinition(self, lookup): mark_attachement = None @@ -598,13 +767,21 @@ class VoltToFea: lookupflags = ast.LookupFlagStatement( flags, mark_attachement, mark_filtering ) + + use_extension = False + if self._settings.get("COMPILER_USEEXTENSIONLOOKUPS"): + use_extension = True + if "\\" in lookup.name: # Merge sub lookups as subtables (lookups named “base\sub”), # makeotf/feaLib will issue a warning and ignore the subtable # statement if it is not a pairpos lookup, though. name = lookup.name.split("\\")[0] if name.lower() not in self._lookups: - fealookup = ast.LookupBlock(self._lookupName(name)) + fealookup = Lookup( + self._lookupName(name), + use_extension=use_extension, + ) if lookupflags is not None: fealookup.statements.append(lookupflags) fealookup.statements.append(ast.Comment("# " + lookup.name)) @@ -614,7 +791,10 @@ class VoltToFea: fealookup.statements.append(ast.Comment("# " + lookup.name)) self._lookups[name.lower()] = fealookup else: - fealookup = ast.LookupBlock(self._lookupName(lookup.name)) + fealookup = Lookup( + self._lookupName(lookup.name), + use_extension=use_extension, + ) if lookupflags is not None: fealookup.statements.append(lookupflags) self._lookups[lookup.name.lower()] = fealookup @@ -623,39 +803,40 @@ class VoltToFea: fealookup.statements.append(ast.Comment("# " + lookup.comments)) contexts = [] - if lookup.context: - for context in lookup.context: - prefix = self._context(context.left) - suffix = self._context(context.right) - ignore = context.ex_or_in == "EXCEPT_CONTEXT" - contexts.append([prefix, suffix, ignore, False]) - # It seems that VOLT will create contextual substitution using - # only the input if there is no other contexts in this lookup. - if ignore and len(lookup.context) == 1: - contexts.append([[], [], False, True]) - else: - contexts.append([[], [], False, False]) - - targetlookup = None - for prefix, suffix, ignore, chain in contexts: + for context in lookup.context: + prefix = self._context(context.left) + suffix = self._context(context.right) + ignore = context.ex_or_in == "EXCEPT_CONTEXT" + contexts.append([prefix, suffix, ignore]) + # It seems that VOLT will create contextual substitution using + # only the input if there is no other contexts in this lookup. + if ignore and len(lookup.context) == 1: + contexts.append([[], [], False]) + + if contexts: + chained = ast.LookupBlock( + self._lookupName(lookup.name + " chained"), + use_extension=use_extension, + ) + fealookup.chained.append(chained) if lookup.sub is not None: - self._gsubLookup(lookup, prefix, suffix, ignore, chain, fealookup) - - if lookup.pos is not None: - if self._settings.get("COMPILER_USEEXTENSIONLOOKUPS"): - fealookup.use_extension = True - if prefix or suffix or chain or ignore: - if not ignore and targetlookup is None: - targetname = self._lookupName(lookup.name + " target") - targetlookup = ast.LookupBlock(targetname) - fealookup.targets = getattr(fealookup, "targets", []) - fealookup.targets.append(targetlookup) - self._gposLookup(lookup, targetlookup) + self._gsubLookup(lookup, chained) + elif lookup.pos is not None: + self._gposLookup(lookup, chained) + for prefix, suffix, ignore in contexts: + if lookup.sub is not None: + self._gsubContextLookup( + lookup, prefix, suffix, ignore, fealookup, chained + ) + elif lookup.pos is not None: self._gposContextLookup( - lookup, prefix, suffix, ignore, fealookup, targetlookup + lookup, prefix, suffix, ignore, fealookup, chained ) - else: - self._gposLookup(lookup, fealookup) + else: + if lookup.sub is not None: + self._gsubLookup(lookup, fealookup) + elif lookup.pos is not None: + self._gposLookup(lookup, fealookup) def main(args=None): diff --git a/contrib/python/fonttools/ya.make b/contrib/python/fonttools/ya.make index 2c3763e30b8..f8dcab48540 100644 --- a/contrib/python/fonttools/ya.make +++ b/contrib/python/fonttools/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(4.57.0) +VERSION(4.58.0) LICENSE(MIT) @@ -192,6 +192,7 @@ PY_SRCS( fontTools/ttLib/tables/G_P_K_G_.py fontTools/ttLib/tables/G_P_O_S_.py fontTools/ttLib/tables/G_S_U_B_.py + fontTools/ttLib/tables/G_V_A_R_.py fontTools/ttLib/tables/G__l_a_t.py fontTools/ttLib/tables/G__l_o_c.py fontTools/ttLib/tables/H_V_A_R_.py @@ -289,6 +290,7 @@ PY_SRCS( fontTools/ufoLib/validators.py fontTools/unicode.py fontTools/unicodedata/Blocks.py + fontTools/unicodedata/Mirrored.py fontTools/unicodedata/OTTags.py fontTools/unicodedata/ScriptExtensions.py fontTools/unicodedata/Scripts.py @@ -323,6 +325,7 @@ PY_SRCS( fontTools/varLib/stat.py fontTools/varLib/varStore.py fontTools/voltLib/__init__.py + fontTools/voltLib/__main__.py fontTools/voltLib/ast.py fontTools/voltLib/error.py fontTools/voltLib/lexer.py diff --git a/contrib/python/platformdirs/.dist-info/METADATA b/contrib/python/platformdirs/.dist-info/METADATA index 91c59c9a280..d8668fed1ea 100644 --- a/contrib/python/platformdirs/.dist-info/METADATA +++ b/contrib/python/platformdirs/.dist-info/METADATA @@ -1,6 +1,6 @@ -Metadata-Version: 2.3 +Metadata-Version: 2.4 Name: platformdirs -Version: 4.3.6 +Version: 4.3.8 Summary: A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`. Project-URL: Changelog, https://github.com/tox-dev/platformdirs/releases Project-URL: Documentation, https://platformdirs.readthedocs.io @@ -17,7 +17,6 @@ Classifier: License :: OSI Approved :: MIT License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.8 Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 @@ -26,20 +25,20 @@ Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Software Development :: Libraries :: Python Modules -Requires-Python: >=3.8 +Requires-Python: >=3.9 Provides-Extra: docs Requires-Dist: furo>=2024.8.6; extra == 'docs' Requires-Dist: proselint>=0.14; extra == 'docs' -Requires-Dist: sphinx-autodoc-typehints>=2.4; extra == 'docs' -Requires-Dist: sphinx>=8.0.2; extra == 'docs' +Requires-Dist: sphinx-autodoc-typehints>=3; extra == 'docs' +Requires-Dist: sphinx>=8.1.3; extra == 'docs' Provides-Extra: test Requires-Dist: appdirs==1.4.4; extra == 'test' Requires-Dist: covdefaults>=2.3; extra == 'test' -Requires-Dist: pytest-cov>=5; extra == 'test' +Requires-Dist: pytest-cov>=6; extra == 'test' Requires-Dist: pytest-mock>=3.14; extra == 'test' -Requires-Dist: pytest>=8.3.2; extra == 'test' +Requires-Dist: pytest>=8.3.4; extra == 'test' Provides-Extra: type -Requires-Dist: mypy>=1.11.2; extra == 'type' +Requires-Dist: mypy>=1.14.1; extra == 'type' Description-Content-Type: text/x-rst The problem @@ -115,10 +114,14 @@ On macOS: >>> appauthor = "Acme" >>> user_data_dir(appname, appauthor) '/Users/trentm/Library/Application Support/SuperApp' - >>> site_data_dir(appname, appauthor) - '/Library/Application Support/SuperApp' + >>> user_config_dir(appname, appauthor) + '/Users/trentm/Library/Application Support/SuperApp' >>> user_cache_dir(appname, appauthor) '/Users/trentm/Library/Caches/SuperApp' + >>> site_data_dir(appname, appauthor) + '/Library/Application Support/SuperApp' + >>> site_config_dir(appname, appauthor) + '/Library/Application Support/SuperApp' >>> user_log_dir(appname, appauthor) '/Users/trentm/Library/Logs/SuperApp' >>> user_documents_dir() @@ -147,8 +150,14 @@ On Windows: 'C:\\Users\\trentm\\AppData\\Local\\Acme\\SuperApp' >>> user_data_dir(appname, appauthor, roaming=True) 'C:\\Users\\trentm\\AppData\\Roaming\\Acme\\SuperApp' + >>> user_config_dir(appname, appauthor) + 'C:\\Users\\trentm\\AppData\\Local\\Acme\\SuperApp' >>> user_cache_dir(appname, appauthor) 'C:\\Users\\trentm\\AppData\\Local\\Acme\\SuperApp\\Cache' + >>> site_data_dir(appname, appauthor) + 'C:\\ProgramData\\Acme\\SuperApp' + >>> site_config_dir(appname, appauthor) + 'C:\\ProgramData\\Acme\\SuperApp' >>> user_log_dir(appname, appauthor) 'C:\\Users\\trentm\\AppData\\Local\\Acme\\SuperApp\\Logs' >>> user_documents_dir() @@ -175,16 +184,21 @@ On Linux: >>> appauthor = "Acme" >>> user_data_dir(appname, appauthor) '/home/trentm/.local/share/SuperApp' + >>> user_config_dir(appname) + '/home/trentm/.config/SuperApp' + >>> user_cache_dir(appname, appauthor) + '/home/trentm/.cache/SuperApp' >>> site_data_dir(appname, appauthor) '/usr/local/share/SuperApp' >>> site_data_dir(appname, appauthor, multipath=True) '/usr/local/share/SuperApp:/usr/share/SuperApp' - >>> user_cache_dir(appname, appauthor) - '/home/trentm/.cache/SuperApp' + >>> site_config_dir(appname) + '/etc/xdg/SuperApp' + >>> os.environ["XDG_CONFIG_DIRS"] = "/etc:/usr/local/etc" + >>> site_config_dir(appname, multipath=True) + '/etc/SuperApp:/usr/local/etc/SuperApp' >>> user_log_dir(appname, appauthor) '/home/trentm/.local/state/SuperApp/log' - >>> user_config_dir(appname) - '/home/trentm/.config/SuperApp' >>> user_documents_dir() '/home/trentm/Documents' >>> user_downloads_dir() @@ -199,11 +213,6 @@ On Linux: '/home/trentm/Desktop' >>> user_runtime_dir(appname, appauthor) '/run/user/{os.getuid()}/SuperApp' - >>> site_config_dir(appname) - '/etc/xdg/SuperApp' - >>> os.environ["XDG_CONFIG_DIRS"] = "/etc:/usr/local/etc" - >>> site_config_dir(appname, multipath=True) - '/etc/SuperApp:/usr/local/etc/SuperApp' On Android:: @@ -212,12 +221,16 @@ On Android:: >>> appauthor = "Acme" >>> user_data_dir(appname, appauthor) '/data/data/com.myApp/files/SuperApp' + >>> user_config_dir(appname) + '/data/data/com.myApp/shared_prefs/SuperApp' >>> user_cache_dir(appname, appauthor) '/data/data/com.myApp/cache/SuperApp' + >>> site_data_dir(appname, appauthor) + '/data/data/com.myApp/files/SuperApp' + >>> site_config_dir(appname) + '/data/data/com.myApp/shared_prefs/SuperApp' >>> user_log_dir(appname, appauthor) '/data/data/com.myApp/cache/SuperApp/log' - >>> user_config_dir(appname) - '/data/data/com.myApp/shared_prefs/SuperApp' >>> user_documents_dir() '/storage/emulated/0/Documents' >>> user_downloads_dir() @@ -249,8 +262,14 @@ apps also support ``XDG_*`` environment variables. >>> dirs = PlatformDirs("SuperApp", "Acme") >>> dirs.user_data_dir '/Users/trentm/Library/Application Support/SuperApp' + >>> dirs.user_config_dir + '/Users/trentm/Library/Application Support/SuperApp' + >>> dirs.user_cache_dir + '/Users/trentm/Library/Caches/SuperApp' >>> dirs.site_data_dir '/Library/Application Support/SuperApp' + >>> dirs.site_config_dir + '/Library/Application Support/SuperApp' >>> dirs.user_cache_dir '/Users/trentm/Library/Caches/SuperApp' >>> dirs.user_log_dir @@ -281,10 +300,14 @@ dirs:: >>> dirs = PlatformDirs("SuperApp", "Acme", version="1.0") >>> dirs.user_data_dir '/Users/trentm/Library/Application Support/SuperApp/1.0' - >>> dirs.site_data_dir - '/Library/Application Support/SuperApp/1.0' + >>> dirs.user_config_dir + '/Users/trentm/Library/Application Support/SuperApp/1.0' >>> dirs.user_cache_dir '/Users/trentm/Library/Caches/SuperApp/1.0' + >>> dirs.site_data_dir + '/Library/Application Support/SuperApp/1.0' + >>> dirs.site_config_dir + '/Library/Application Support/SuperApp/1.0' >>> dirs.user_log_dir '/Users/trentm/Library/Logs/SuperApp/1.0' >>> dirs.user_documents_dir diff --git a/contrib/python/platformdirs/README.rst b/contrib/python/platformdirs/README.rst index 1562ecb5906..b83425e17c6 100644 --- a/contrib/python/platformdirs/README.rst +++ b/contrib/python/platformdirs/README.rst @@ -71,10 +71,14 @@ On macOS: >>> appauthor = "Acme" >>> user_data_dir(appname, appauthor) '/Users/trentm/Library/Application Support/SuperApp' - >>> site_data_dir(appname, appauthor) - '/Library/Application Support/SuperApp' + >>> user_config_dir(appname, appauthor) + '/Users/trentm/Library/Application Support/SuperApp' >>> user_cache_dir(appname, appauthor) '/Users/trentm/Library/Caches/SuperApp' + >>> site_data_dir(appname, appauthor) + '/Library/Application Support/SuperApp' + >>> site_config_dir(appname, appauthor) + '/Library/Application Support/SuperApp' >>> user_log_dir(appname, appauthor) '/Users/trentm/Library/Logs/SuperApp' >>> user_documents_dir() @@ -103,8 +107,14 @@ On Windows: 'C:\\Users\\trentm\\AppData\\Local\\Acme\\SuperApp' >>> user_data_dir(appname, appauthor, roaming=True) 'C:\\Users\\trentm\\AppData\\Roaming\\Acme\\SuperApp' + >>> user_config_dir(appname, appauthor) + 'C:\\Users\\trentm\\AppData\\Local\\Acme\\SuperApp' >>> user_cache_dir(appname, appauthor) 'C:\\Users\\trentm\\AppData\\Local\\Acme\\SuperApp\\Cache' + >>> site_data_dir(appname, appauthor) + 'C:\\ProgramData\\Acme\\SuperApp' + >>> site_config_dir(appname, appauthor) + 'C:\\ProgramData\\Acme\\SuperApp' >>> user_log_dir(appname, appauthor) 'C:\\Users\\trentm\\AppData\\Local\\Acme\\SuperApp\\Logs' >>> user_documents_dir() @@ -131,16 +141,21 @@ On Linux: >>> appauthor = "Acme" >>> user_data_dir(appname, appauthor) '/home/trentm/.local/share/SuperApp' + >>> user_config_dir(appname) + '/home/trentm/.config/SuperApp' + >>> user_cache_dir(appname, appauthor) + '/home/trentm/.cache/SuperApp' >>> site_data_dir(appname, appauthor) '/usr/local/share/SuperApp' >>> site_data_dir(appname, appauthor, multipath=True) '/usr/local/share/SuperApp:/usr/share/SuperApp' - >>> user_cache_dir(appname, appauthor) - '/home/trentm/.cache/SuperApp' + >>> site_config_dir(appname) + '/etc/xdg/SuperApp' + >>> os.environ["XDG_CONFIG_DIRS"] = "/etc:/usr/local/etc" + >>> site_config_dir(appname, multipath=True) + '/etc/SuperApp:/usr/local/etc/SuperApp' >>> user_log_dir(appname, appauthor) '/home/trentm/.local/state/SuperApp/log' - >>> user_config_dir(appname) - '/home/trentm/.config/SuperApp' >>> user_documents_dir() '/home/trentm/Documents' >>> user_downloads_dir() @@ -155,11 +170,6 @@ On Linux: '/home/trentm/Desktop' >>> user_runtime_dir(appname, appauthor) '/run/user/{os.getuid()}/SuperApp' - >>> site_config_dir(appname) - '/etc/xdg/SuperApp' - >>> os.environ["XDG_CONFIG_DIRS"] = "/etc:/usr/local/etc" - >>> site_config_dir(appname, multipath=True) - '/etc/SuperApp:/usr/local/etc/SuperApp' On Android:: @@ -168,12 +178,16 @@ On Android:: >>> appauthor = "Acme" >>> user_data_dir(appname, appauthor) '/data/data/com.myApp/files/SuperApp' + >>> user_config_dir(appname) + '/data/data/com.myApp/shared_prefs/SuperApp' >>> user_cache_dir(appname, appauthor) '/data/data/com.myApp/cache/SuperApp' + >>> site_data_dir(appname, appauthor) + '/data/data/com.myApp/files/SuperApp' + >>> site_config_dir(appname) + '/data/data/com.myApp/shared_prefs/SuperApp' >>> user_log_dir(appname, appauthor) '/data/data/com.myApp/cache/SuperApp/log' - >>> user_config_dir(appname) - '/data/data/com.myApp/shared_prefs/SuperApp' >>> user_documents_dir() '/storage/emulated/0/Documents' >>> user_downloads_dir() @@ -205,8 +219,14 @@ apps also support ``XDG_*`` environment variables. >>> dirs = PlatformDirs("SuperApp", "Acme") >>> dirs.user_data_dir '/Users/trentm/Library/Application Support/SuperApp' + >>> dirs.user_config_dir + '/Users/trentm/Library/Application Support/SuperApp' + >>> dirs.user_cache_dir + '/Users/trentm/Library/Caches/SuperApp' >>> dirs.site_data_dir '/Library/Application Support/SuperApp' + >>> dirs.site_config_dir + '/Library/Application Support/SuperApp' >>> dirs.user_cache_dir '/Users/trentm/Library/Caches/SuperApp' >>> dirs.user_log_dir @@ -237,10 +257,14 @@ dirs:: >>> dirs = PlatformDirs("SuperApp", "Acme", version="1.0") >>> dirs.user_data_dir '/Users/trentm/Library/Application Support/SuperApp/1.0' - >>> dirs.site_data_dir - '/Library/Application Support/SuperApp/1.0' + >>> dirs.user_config_dir + '/Users/trentm/Library/Application Support/SuperApp/1.0' >>> dirs.user_cache_dir '/Users/trentm/Library/Caches/SuperApp/1.0' + >>> dirs.site_data_dir + '/Library/Application Support/SuperApp/1.0' + >>> dirs.site_config_dir + '/Library/Application Support/SuperApp/1.0' >>> dirs.user_log_dir '/Users/trentm/Library/Logs/SuperApp/1.0' >>> dirs.user_documents_dir diff --git a/contrib/python/platformdirs/platformdirs/__init__.py b/contrib/python/platformdirs/platformdirs/__init__.py index afe8351d203..02daa5914a8 100644 --- a/contrib/python/platformdirs/platformdirs/__init__.py +++ b/contrib/python/platformdirs/platformdirs/__init__.py @@ -52,7 +52,7 @@ AppDirs = PlatformDirs #: Backwards compatibility with appdirs def user_data_dir( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, roaming: bool = False, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -76,7 +76,7 @@ def user_data_dir( def site_data_dir( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, multipath: bool = False, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -100,7 +100,7 @@ def site_data_dir( def user_config_dir( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, roaming: bool = False, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -124,7 +124,7 @@ def user_config_dir( def site_config_dir( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, multipath: bool = False, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -148,7 +148,7 @@ def site_config_dir( def user_cache_dir( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, opinion: bool = True, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -172,7 +172,7 @@ def user_cache_dir( def site_cache_dir( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, opinion: bool = True, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -196,7 +196,7 @@ def site_cache_dir( def user_state_dir( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, roaming: bool = False, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -220,7 +220,7 @@ def user_state_dir( def user_log_dir( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, opinion: bool = True, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -274,7 +274,7 @@ def user_desktop_dir() -> str: def user_runtime_dir( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, opinion: bool = True, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -298,7 +298,7 @@ def user_runtime_dir( def site_runtime_dir( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, opinion: bool = True, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -322,7 +322,7 @@ def site_runtime_dir( def user_data_path( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, roaming: bool = False, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -346,7 +346,7 @@ def user_data_path( def site_data_path( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, multipath: bool = False, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -370,7 +370,7 @@ def site_data_path( def user_config_path( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, roaming: bool = False, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -394,7 +394,7 @@ def user_config_path( def site_config_path( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, multipath: bool = False, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -418,7 +418,7 @@ def site_config_path( def site_cache_path( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, opinion: bool = True, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -442,7 +442,7 @@ def site_cache_path( def user_cache_path( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, opinion: bool = True, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -466,7 +466,7 @@ def user_cache_path( def user_state_path( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, roaming: bool = False, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -490,7 +490,7 @@ def user_state_path( def user_log_path( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, opinion: bool = True, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -544,7 +544,7 @@ def user_desktop_path() -> Path: def user_runtime_path( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, opinion: bool = True, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 @@ -568,7 +568,7 @@ def user_runtime_path( def site_runtime_path( appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, opinion: bool = True, # noqa: FBT001, FBT002 ensure_exists: bool = False, # noqa: FBT001, FBT002 diff --git a/contrib/python/platformdirs/platformdirs/android.py b/contrib/python/platformdirs/platformdirs/android.py index 7004a852422..92efc852d38 100644 --- a/contrib/python/platformdirs/platformdirs/android.py +++ b/contrib/python/platformdirs/platformdirs/android.py @@ -23,7 +23,7 @@ class Android(PlatformDirsABC): @property def user_data_dir(self) -> str: """:return: data directory tied to the user, e.g. ``/data/user/<userid>/<packagename>/files/<AppName>``""" - return self._append_app_name_and_version(cast(str, _android_folder()), "files") + return self._append_app_name_and_version(cast("str", _android_folder()), "files") @property def site_data_dir(self) -> str: @@ -36,7 +36,7 @@ class Android(PlatformDirsABC): :return: config directory tied to the user, e.g. \ ``/data/user/<userid>/<packagename>/shared_prefs/<AppName>`` """ - return self._append_app_name_and_version(cast(str, _android_folder()), "shared_prefs") + return self._append_app_name_and_version(cast("str", _android_folder()), "shared_prefs") @property def site_config_dir(self) -> str: @@ -46,7 +46,7 @@ class Android(PlatformDirsABC): @property def user_cache_dir(self) -> str: """:return: cache directory tied to the user, e.g.,``/data/user/<userid>/<packagename>/cache/<AppName>``""" - return self._append_app_name_and_version(cast(str, _android_folder()), "cache") + return self._append_app_name_and_version(cast("str", _android_folder()), "cache") @property def site_cache_dir(self) -> str: diff --git a/contrib/python/platformdirs/platformdirs/api.py b/contrib/python/platformdirs/platformdirs/api.py index 18d660e4f8c..a352035ec69 100644 --- a/contrib/python/platformdirs/platformdirs/api.py +++ b/contrib/python/platformdirs/platformdirs/api.py @@ -8,7 +8,8 @@ from pathlib import Path from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Iterator, Literal + from collections.abc import Iterator + from typing import Literal class PlatformDirsABC(ABC): # noqa: PLR0904 @@ -17,7 +18,7 @@ class PlatformDirsABC(ABC): # noqa: PLR0904 def __init__( # noqa: PLR0913, PLR0917 self, appname: str | None = None, - appauthor: str | None | Literal[False] = None, + appauthor: str | Literal[False] | None = None, version: str | None = None, roaming: bool = False, # noqa: FBT001, FBT002 multipath: bool = False, # noqa: FBT001, FBT002 diff --git a/contrib/python/platformdirs/platformdirs/unix.py b/contrib/python/platformdirs/platformdirs/unix.py index f1942e92ef4..fc75d8d0747 100644 --- a/contrib/python/platformdirs/platformdirs/unix.py +++ b/contrib/python/platformdirs/platformdirs/unix.py @@ -6,10 +6,13 @@ import os import sys from configparser import ConfigParser from pathlib import Path -from typing import Iterator, NoReturn +from typing import TYPE_CHECKING, NoReturn from .api import PlatformDirsABC +if TYPE_CHECKING: + from collections.abc import Iterator + if sys.platform == "win32": def getuid() -> NoReturn: diff --git a/contrib/python/platformdirs/platformdirs/version.py b/contrib/python/platformdirs/platformdirs/version.py index afb49243e3d..611ac615443 100644 --- a/contrib/python/platformdirs/platformdirs/version.py +++ b/contrib/python/platformdirs/platformdirs/version.py @@ -1,8 +1,13 @@ -# file generated by setuptools_scm +# file generated by setuptools-scm # don't change, don't track in version control + +__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"] + TYPE_CHECKING = False if TYPE_CHECKING: - from typing import Tuple, Union + from typing import Tuple + from typing import Union + VERSION_TUPLE = Tuple[Union[int, str], ...] else: VERSION_TUPLE = object @@ -12,5 +17,5 @@ __version__: str __version_tuple__: VERSION_TUPLE version_tuple: VERSION_TUPLE -__version__ = version = '4.3.6' -__version_tuple__ = version_tuple = (4, 3, 6) +__version__ = version = '4.3.8' +__version_tuple__ = version_tuple = (4, 3, 8) diff --git a/contrib/python/platformdirs/ya.make b/contrib/python/platformdirs/ya.make index 66109cf6a8b..1a33df4191e 100644 --- a/contrib/python/platformdirs/ya.make +++ b/contrib/python/platformdirs/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(4.3.6) +VERSION(4.3.8) LICENSE(MIT) diff --git a/contrib/python/types-protobuf/.dist-info/METADATA b/contrib/python/types-protobuf/.dist-info/METADATA index d0b3a5d6eb4..0fd53bbd2c4 100644 --- a/contrib/python/types-protobuf/.dist-info/METADATA +++ b/contrib/python/types-protobuf/.dist-info/METADATA @@ -1,27 +1,19 @@ -Metadata-Version: 2.2 +Metadata-Version: 2.4 Name: types-protobuf -Version: 5.29.1.20250208 +Version: 6.30.2.20250506 Summary: Typing stubs for protobuf -Home-page: https://github.com/python/typeshed -License: Apache-2.0 +License-Expression: Apache-2.0 +Project-URL: Homepage, https://github.com/python/typeshed Project-URL: GitHub, https://github.com/python/typeshed Project-URL: Changes, https://github.com/typeshed-internal/stub_uploader/blob/main/data/changelogs/protobuf.md Project-URL: Issue tracker, https://github.com/python/typeshed/issues Project-URL: Chat, https://gitter.im/python/typing -Classifier: License :: OSI Approved :: Apache Software License Classifier: Programming Language :: Python :: 3 Classifier: Typing :: Stubs Only Requires-Python: >=3.9 Description-Content-Type: text/markdown License-File: LICENSE -Dynamic: classifier -Dynamic: description -Dynamic: description-content-type -Dynamic: home-page -Dynamic: license -Dynamic: project-url -Dynamic: requires-python -Dynamic: summary +Dynamic: license-file ## Typing stubs for protobuf @@ -34,9 +26,9 @@ It can be used by type-checking tools like [Pyre](https://pyre-check.org/), PyCharm, etc. to check code that uses `protobuf`. This version of `types-protobuf` aims to provide accurate annotations for -`protobuf~=5.29.1`. +`protobuf~=6.30.2`. -Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 28.1 on [protobuf v29.1](https://github.com/protocolbuffers/protobuf/releases/tag/v29.1) (python `protobuf==5.29.1`). +Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 29.0 on [protobuf v30.2](https://github.com/protocolbuffers/protobuf/releases/tag/v30.2) (python `protobuf==6.30.2`). This stub package is marked as [partial](https://peps.python.org/pep-0561/#partial-stub-packages). If you find that annotations are missing, feel free to contribute and help complete them. @@ -51,7 +43,7 @@ directory. This package was tested with mypy 1.15.0, -pyright 1.1.389, +pyright 1.1.400, and pytype 2024.10.11. It was generated from typeshed commit -[`73ebb9dfd7dfce93c5becde4dcdd51d5626853b8`](https://github.com/python/typeshed/commit/73ebb9dfd7dfce93c5becde4dcdd51d5626853b8). +[`4265ee7c72f476e7156949e55784fd82b40e6953`](https://github.com/python/typeshed/commit/4265ee7c72f476e7156949e55784fd82b40e6953). diff --git a/contrib/python/types-protobuf/README.md b/contrib/python/types-protobuf/README.md index 6b33aeb7279..c35bf581ad2 100644 --- a/contrib/python/types-protobuf/README.md +++ b/contrib/python/types-protobuf/README.md @@ -9,9 +9,9 @@ It can be used by type-checking tools like [Pyre](https://pyre-check.org/), PyCharm, etc. to check code that uses `protobuf`. This version of `types-protobuf` aims to provide accurate annotations for -`protobuf~=5.29.1`. +`protobuf~=6.30.2`. -Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 28.1 on [protobuf v29.1](https://github.com/protocolbuffers/protobuf/releases/tag/v29.1) (python `protobuf==5.29.1`). +Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 29.0 on [protobuf v30.2](https://github.com/protocolbuffers/protobuf/releases/tag/v30.2) (python `protobuf==6.30.2`). This stub package is marked as [partial](https://peps.python.org/pep-0561/#partial-stub-packages). If you find that annotations are missing, feel free to contribute and help complete them. @@ -26,7 +26,7 @@ directory. This package was tested with mypy 1.15.0, -pyright 1.1.389, +pyright 1.1.400, and pytype 2024.10.11. It was generated from typeshed commit -[`73ebb9dfd7dfce93c5becde4dcdd51d5626853b8`](https://github.com/python/typeshed/commit/73ebb9dfd7dfce93c5becde4dcdd51d5626853b8).
\ No newline at end of file +[`4265ee7c72f476e7156949e55784fd82b40e6953`](https://github.com/python/typeshed/commit/4265ee7c72f476e7156949e55784fd82b40e6953).
\ No newline at end of file diff --git a/contrib/python/types-protobuf/google-stubs/METADATA.toml b/contrib/python/types-protobuf/google-stubs/METADATA.toml index 04c674e5fbc..a7416507152 100644 --- a/contrib/python/types-protobuf/google-stubs/METADATA.toml +++ b/contrib/python/types-protobuf/google-stubs/METADATA.toml @@ -1,7 +1,8 @@ # Using an exact number in the specifier for scripts/sync_protobuf/google_protobuf.py -version = "~=5.29.1" +# When updating, also re-run the script +version = "~=6.30.2" upstream_repository = "https://github.com/protocolbuffers/protobuf" -extra_description = "Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 28.1 on [protobuf v29.1](https://github.com/protocolbuffers/protobuf/releases/tag/v29.1) (python `protobuf==5.29.1`)." +extra_description = "Partially generated using [mypy-protobuf==3.6.0](https://github.com/nipunn1313/mypy-protobuf/tree/v3.6.0) and libprotoc 29.0 on [protobuf v30.2](https://github.com/protocolbuffers/protobuf/releases/tag/v30.2) (python `protobuf==6.30.2`)." partial_stub = true [tool.stubtest] diff --git a/contrib/python/types-protobuf/google-stubs/_upb/_message.pyi b/contrib/python/types-protobuf/google-stubs/_upb/_message.pyi new file mode 100644 index 00000000000..bb188bcbf4f --- /dev/null +++ b/contrib/python/types-protobuf/google-stubs/_upb/_message.pyi @@ -0,0 +1,310 @@ +from _typeshed import Incomplete +from typing import ClassVar, final + +default_pool: DescriptorPool + +@final +class Arena: ... + +@final +class Descriptor: + containing_type: Incomplete + enum_types: Incomplete + enum_types_by_name: Incomplete + enum_values_by_name: Incomplete + extension_ranges: Incomplete + extensions: Incomplete + extensions_by_name: Incomplete + fields: Incomplete + fields_by_camelcase_name: Incomplete + fields_by_name: Incomplete + fields_by_number: Incomplete + file: Incomplete + full_name: Incomplete + has_options: Incomplete + is_extendable: Incomplete + name: Incomplete + nested_types: Incomplete + nested_types_by_name: Incomplete + oneofs: Incomplete + oneofs_by_name: Incomplete + def __init__(self, *args, **kwargs) -> None: ... # incomplete + def CopyToProto(self, object, /): ... + def EnumValueName(self, *args, **kwargs): ... # incomplete + def GetOptions(self): ... + +@final +class DescriptorPool: + def __init__(self, *args, **kwargs) -> None: ... # incomplete + def Add(self, object, /): ... + def AddSerializedFile(self, object, /): ... + def FindAllExtensions(self, object, /): ... + def FindEnumTypeByName(self, object, /): ... + def FindExtensionByName(self, object, /): ... + def FindExtensionByNumber(self, *args, **kwargs): ... # incomplete + def FindFieldByName(self, object, /): ... + def FindFileByName(self, object, /): ... + def FindFileContainingSymbol(self, object, /): ... + def FindMessageTypeByName(self, object, /): ... + def FindMethodByName(self, object, /): ... + def FindOneofByName(self, object, /): ... + def FindServiceByName(self, object, /): ... + def SetFeatureSetDefaults(self, object, /): ... + +@final +class EnumDescriptor: + containing_type: Incomplete + file: Incomplete + full_name: Incomplete + has_options: Incomplete + is_closed: Incomplete + name: Incomplete + values: Incomplete + values_by_name: Incomplete + values_by_number: Incomplete + def __init__(self, *args, **kwargs) -> None: ... # incomplete + def CopyToProto(self, object, /): ... + def GetOptions(self): ... + +@final +class EnumValueDescriptor: + has_options: Incomplete + index: Incomplete + name: Incomplete + number: Incomplete + type: Incomplete + def __init__(self, *args, **kwargs) -> None: ... # incomplete + def GetOptions(self): ... + +@final +class ExtensionDict: + def __contains__(self, other) -> bool: ... + def __delitem__(self, other) -> None: ... + def __eq__(self, other: object) -> bool: ... + def __ge__(self, other: object) -> bool: ... + def __getitem__(self, index): ... + def __gt__(self, other: object) -> bool: ... + def __iter__(self): ... + def __le__(self, other: object) -> bool: ... + def __len__(self) -> int: ... + def __lt__(self, other: object) -> bool: ... + def __ne__(self, other: object) -> bool: ... + def __setitem__(self, index, object) -> None: ... + +@final +class ExtensionIterator: + def __iter__(self): ... + def __next__(self): ... + +@final +class FieldDescriptor: + CPPTYPE_BOOL: ClassVar[int] = ... + CPPTYPE_BYTES: ClassVar[int] = ... + CPPTYPE_DOUBLE: ClassVar[int] = ... + CPPTYPE_ENUM: ClassVar[int] = ... + CPPTYPE_FLOAT: ClassVar[int] = ... + CPPTYPE_INT32: ClassVar[int] = ... + CPPTYPE_INT64: ClassVar[int] = ... + CPPTYPE_MESSAGE: ClassVar[int] = ... + CPPTYPE_STRING: ClassVar[int] = ... + CPPTYPE_UINT32: ClassVar[int] = ... + CPPTYPE_UINT64: ClassVar[int] = ... + LABEL_OPTIONAL: ClassVar[int] = ... + LABEL_REPEATED: ClassVar[int] = ... + LABEL_REQUIRED: ClassVar[int] = ... + TYPE_BOOL: ClassVar[int] = ... + TYPE_BYTES: ClassVar[int] = ... + TYPE_DOUBLE: ClassVar[int] = ... + TYPE_ENUM: ClassVar[int] = ... + TYPE_FIXED32: ClassVar[int] = ... + TYPE_FIXED64: ClassVar[int] = ... + TYPE_FLOAT: ClassVar[int] = ... + TYPE_GROUP: ClassVar[int] = ... + TYPE_INT32: ClassVar[int] = ... + TYPE_INT64: ClassVar[int] = ... + TYPE_MESSAGE: ClassVar[int] = ... + TYPE_SFIXED32: ClassVar[int] = ... + TYPE_SFIXED64: ClassVar[int] = ... + TYPE_SINT32: ClassVar[int] = ... + TYPE_SINT64: ClassVar[int] = ... + TYPE_STRING: ClassVar[int] = ... + TYPE_UINT32: ClassVar[int] = ... + TYPE_UINT64: ClassVar[int] = ... + camelcase_name: Incomplete + containing_oneof: Incomplete + containing_type: Incomplete + cpp_type: Incomplete + default_value: Incomplete + enum_type: Incomplete + extension_scope: Incomplete + file: Incomplete + full_name: Incomplete + has_default_value: Incomplete + has_options: Incomplete + has_presence: Incomplete + index: Incomplete + is_extension: Incomplete + is_packed: Incomplete + json_name: Incomplete + label: Incomplete + message_type: Incomplete + name: Incomplete + number: Incomplete + type: Incomplete + def __init__(self, *args, **kwargs) -> None: ... # incomplete + def GetOptions(self): ... + +@final +class FileDescriptor: + dependencies: Incomplete + enum_types_by_name: Incomplete + extensions_by_name: Incomplete + has_options: Incomplete + message_types_by_name: Incomplete + name: Incomplete + package: Incomplete + pool: Incomplete + public_dependencies: Incomplete + serialized_pb: Incomplete + services_by_name: Incomplete + def __init__(self, *args, **kwargs) -> None: ... # incomplete + def CopyToProto(self, object, /): ... + def GetOptions(self): ... + +@final +class MapIterator: + def __iter__(self): ... + def __next__(self): ... + +@final +class Message: + Extensions: Incomplete + def __init__(self, *args, **kwargs) -> None: ... # incomplete # incomplete + def ByteSize(self): ... + def Clear(self): ... + def ClearExtension(self, object, /): ... + def ClearField(self, object, /): ... + def CopyFrom(self, object, /): ... + def DiscardUnknownFields(self): ... + def FindInitializationErrors(self): ... + @classmethod + def FromString(cls, object, /): ... + def HasExtension(self, object, /): ... + def HasField(self, object, /): ... + def IsInitialized(self, *args, **kwargs): ... # incomplete + def ListFields(self): ... + def MergeFrom(self, object, /): ... + def MergeFromString(self, object, /): ... + def ParseFromString(self, object, /): ... + def SerializePartialToString(self, *args, **kwargs): ... # incomplete + def SerializeToString(self, *args, **kwargs): ... # incomplete + def SetInParent(self): ... + def UnknownFields(self): ... + def WhichOneof(self, object, /): ... + def __contains__(self, other) -> bool: ... + def __deepcopy__(self, memo=None): ... + def __delattr__(self, name): ... + def __eq__(self, other: object) -> bool: ... + def __ge__(self, other: object) -> bool: ... + def __gt__(self, other: object) -> bool: ... + def __le__(self, other: object) -> bool: ... + def __lt__(self, other: object) -> bool: ... + def __ne__(self, other: object) -> bool: ... + def __setattr__(self, name, value): ... + +@final +class MessageMeta(type): ... + +@final +class MethodDescriptor: + client_streaming: Incomplete + containing_service: Incomplete + full_name: Incomplete + index: Incomplete + input_type: Incomplete + name: Incomplete + output_type: Incomplete + server_streaming: Incomplete + def __init__(self, *args, **kwargs) -> None: ... # incomplete + def CopyToProto(self, object, /): ... + def GetOptions(self): ... + +@final +class OneofDescriptor: + containing_type: Incomplete + fields: Incomplete + full_name: Incomplete + has_options: Incomplete + index: Incomplete + name: Incomplete + def __init__(self, *args, **kwargs) -> None: ... # incomplete + def GetOptions(self): ... + +@final +class RepeatedCompositeContainer: + def __init__(self, *args, **kwargs) -> None: ... # incomplete + def MergeFrom(self, object, /): ... + def add(self, *args, **kwargs): ... # incomplete + def append(self, object, /): ... + def extend(self, object, /): ... + def insert(self, *args, **kwargs): ... # incomplete + def pop(self, *args, **kwargs): ... # incomplete + def remove(self, object, /): ... + def reverse(self): ... + def sort(self, *args, **kwargs): ... # incomplete + def __deepcopy__(self, memo=None): ... + def __delitem__(self, other) -> None: ... + def __eq__(self, other: object) -> bool: ... + def __ge__(self, other: object) -> bool: ... + def __getitem__(self, index): ... + def __gt__(self, other: object) -> bool: ... + def __le__(self, other: object) -> bool: ... + def __len__(self) -> int: ... + def __lt__(self, other: object) -> bool: ... + def __ne__(self, other: object) -> bool: ... + def __setitem__(self, index, object) -> None: ... + +@final +class RepeatedScalarContainer: + def __init__(self, *args, **kwargs) -> None: ... # incomplete + def MergeFrom(self, object, /): ... + def append(self, object, /): ... + def extend(self, object, /): ... + def insert(self, *args, **kwargs): ... # incomplete + def pop(self, *args, **kwargs): ... # incomplete + def remove(self, object, /): ... + def reverse(self): ... + def sort(self, *args, **kwargs): ... # incomplete + def __deepcopy__(self, memo=None): ... + def __delitem__(self, other) -> None: ... + def __eq__(self, other: object) -> bool: ... + def __ge__(self, other: object) -> bool: ... + def __getitem__(self, index): ... + def __gt__(self, other: object) -> bool: ... + def __le__(self, other: object) -> bool: ... + def __len__(self) -> int: ... + def __lt__(self, other: object) -> bool: ... + def __ne__(self, other: object) -> bool: ... + def __reduce__(self): ... + def __setitem__(self, index, object) -> None: ... + +@final +class ServiceDescriptor: + file: Incomplete + full_name: Incomplete + index: Incomplete + methods: Incomplete + methods_by_name: Incomplete + name: Incomplete + def __init__(self, *args, **kwargs) -> None: ... # incomplete + def CopyToProto(self, object, /): ... + def FindMethodByName(self, object, /): ... + def GetOptions(self): ... + +@final +class UnknownFieldSet: + def __init__(self, *args, **kwargs) -> None: ... # incomplete + def __getitem__(self, index): ... + def __len__(self) -> int: ... + +def SetAllowOversizeProtos(object, /): ... # incomplete diff --git a/contrib/python/types-protobuf/google-stubs/protobuf/__init__.pyi b/contrib/python/types-protobuf/google-stubs/protobuf/__init__.pyi index bda5b5a7f4c..c5dd9546606 100644 --- a/contrib/python/types-protobuf/google-stubs/protobuf/__init__.pyi +++ b/contrib/python/types-protobuf/google-stubs/protobuf/__init__.pyi @@ -1 +1,3 @@ -__version__: str +from typing import Final + +__version__: Final[str] diff --git a/contrib/python/types-protobuf/google-stubs/protobuf/descriptor.pyi b/contrib/python/types-protobuf/google-stubs/protobuf/descriptor.pyi index 291ca0c7296..45891fda76c 100644 --- a/contrib/python/types-protobuf/google-stubs/protobuf/descriptor.pyi +++ b/contrib/python/types-protobuf/google-stubs/protobuf/descriptor.pyi @@ -156,6 +156,12 @@ class FieldDescriptor(DescriptorBase): cpp_type: Any @property def label(self): ... + @property + def camelcase_name(self) -> str: ... + @property + def has_presence(self) -> bool: ... + @property + def is_packed(self) -> bool: ... has_default_value: Any default_value: Any containing_type: Any diff --git a/contrib/python/types-protobuf/google-stubs/protobuf/descriptor_database.pyi b/contrib/python/types-protobuf/google-stubs/protobuf/descriptor_database.pyi new file mode 100644 index 00000000000..6568fc2159c --- /dev/null +++ b/contrib/python/types-protobuf/google-stubs/protobuf/descriptor_database.pyi @@ -0,0 +1,16 @@ +from typing import Final + +from google.protobuf.descriptor_pb2 import FileDescriptorProto + +__author__: Final[str] + +class Error(Exception): ... +class DescriptorDatabaseConflictingDefinitionError(Error): ... + +class DescriptorDatabase: + def __init__(self) -> None: ... + def Add(self, file_desc_proto) -> None: ... + def FindFileByName(self, name): ... + def FindFileContainingSymbol(self, symbol): ... + def FindFileContainingExtension(self, extendee_name, extension_number) -> FileDescriptorProto | None: ... + def FindAllExtensionNumbers(self, extendee_name) -> list[int]: ... diff --git a/contrib/python/types-protobuf/google-stubs/protobuf/internal/containers.pyi b/contrib/python/types-protobuf/google-stubs/protobuf/internal/containers.pyi index aaa97043921..75261371607 100644 --- a/contrib/python/types-protobuf/google-stubs/protobuf/internal/containers.pyi +++ b/contrib/python/types-protobuf/google-stubs/protobuf/internal/containers.pyi @@ -1,5 +1,6 @@ from collections.abc import Callable, Iterable, Iterator, MutableMapping, Sequence from typing import Any, Protocol, SupportsIndex, TypeVar, overload +from typing_extensions import Self from google.protobuf.descriptor import Descriptor from google.protobuf.internal.message_listener import MessageListener @@ -10,7 +11,6 @@ _T = TypeVar("_T") _K = TypeVar("_K", bound=bool | int | str) _ScalarV = TypeVar("_ScalarV", bound=bool | int | float | str | bytes) _MessageV = TypeVar("_MessageV", bound=Message) -_M = TypeVar("_M") class _ValueChecker(Protocol[_T]): def CheckValue(self, proposed_value: _T) -> _T: ... @@ -33,7 +33,7 @@ class RepeatedScalarFieldContainer(BaseContainer[_ScalarV]): def append(self, value: _ScalarV) -> None: ... def insert(self, key: int, value: _ScalarV) -> None: ... def extend(self, elem_seq: Iterable[_ScalarV] | None) -> None: ... - def MergeFrom(self: _M, other: _M | Iterable[_ScalarV]) -> None: ... + def MergeFrom(self, other: Self | Iterable[_ScalarV]) -> None: ... def remove(self, elem: _ScalarV) -> None: ... def pop(self, key: int = -1) -> _ScalarV: ... @overload @@ -49,7 +49,7 @@ class RepeatedCompositeFieldContainer(BaseContainer[_MessageV]): def append(self, value: _MessageV) -> None: ... def insert(self, key: int, value: _MessageV) -> None: ... def extend(self, elem_seq: Iterable[_MessageV]) -> None: ... - def MergeFrom(self: _M, other: _M | Iterable[_MessageV]) -> None: ... + def MergeFrom(self, other: Self | Iterable[_MessageV]) -> None: ... def remove(self, elem: _MessageV) -> None: ... def pop(self, key: int = -1) -> _MessageV: ... def __delitem__(self, key: int | slice) -> None: ... @@ -73,7 +73,8 @@ class ScalarMap(MutableMapping[_K, _ScalarV]): def get(self, key: _K, default: None = None) -> _ScalarV | None: ... @overload def get(self, key: _K, default: _ScalarV | _T) -> _ScalarV | _T: ... - def MergeFrom(self: _M, other: _M): ... + def setdefault(self, key: _K, value: _ScalarV | None = None) -> _ScalarV: ... + def MergeFrom(self, other: Self): ... def InvalidateIterators(self) -> None: ... def GetEntryClass(self) -> GeneratedProtocolMessageType: ... @@ -96,6 +97,7 @@ class MessageMap(MutableMapping[_K, _MessageV]): @overload def get(self, key: _K, default: _MessageV | _T) -> _MessageV | _T: ... def get_or_create(self, key: _K) -> _MessageV: ... - def MergeFrom(self: _M, other: _M): ... + def setdefault(self, key: _K, value: _MessageV | None = None) -> _MessageV: ... + def MergeFrom(self, other: Self): ... def InvalidateIterators(self) -> None: ... def GetEntryClass(self) -> GeneratedProtocolMessageType: ... diff --git a/contrib/python/types-protobuf/google-stubs/protobuf/internal/decoder.pyi b/contrib/python/types-protobuf/google-stubs/protobuf/internal/decoder.pyi index ce74e9318c6..94ed7f61536 100644 --- a/contrib/python/types-protobuf/google-stubs/protobuf/internal/decoder.pyi +++ b/contrib/python/types-protobuf/google-stubs/protobuf/internal/decoder.pyi @@ -59,5 +59,3 @@ MESSAGE_SET_ITEM_TAG: bytes def MessageSetItemDecoder(descriptor: Descriptor) -> _Decoder: ... def MapDecoder(field_descriptor, new_default, is_message_map) -> _Decoder: ... - -SkipField: Any diff --git a/contrib/python/types-protobuf/google-stubs/protobuf/message.pyi b/contrib/python/types-protobuf/google-stubs/protobuf/message.pyi index 819ad7aad5d..ea1d636ee26 100644 --- a/contrib/python/types-protobuf/google-stubs/protobuf/message.pyi +++ b/contrib/python/types-protobuf/google-stubs/protobuf/message.pyi @@ -1,7 +1,9 @@ from collections.abc import Sequence -from typing import Any, TypeVar +from typing import Any from typing_extensions import Self +from google._upb._message import Descriptor as _upb_Descriptor + from .descriptor import Descriptor, FieldDescriptor from .internal.extension_dict import _ExtensionDict, _ExtensionFieldDescriptor @@ -9,10 +11,8 @@ class Error(Exception): ... class DecodeError(Error): ... class EncodeError(Error): ... -_M = TypeVar("_M", bound=Message) # message type (of self) - class Message: - DESCRIPTOR: Descriptor + DESCRIPTOR: Descriptor | _upb_Descriptor def __deepcopy__(self, memo: Any = None) -> Self: ... def __eq__(self, other_msg): ... def __ne__(self, other_msg): ... @@ -26,12 +26,11 @@ class Message: def SerializeToString(self, *, deterministic: bool = ...) -> bytes: ... def SerializePartialToString(self, *, deterministic: bool = ...) -> bytes: ... def ListFields(self) -> Sequence[tuple[FieldDescriptor, Any]]: ... - # The TypeVar must be bound to `Message` or we get mypy errors, so we cannot use `Self` for `HasExtension` & `ClearExtension` - def HasExtension(self: _M, field_descriptor: _ExtensionFieldDescriptor[_M, Any]) -> bool: ... - def ClearExtension(self: _M, field_descriptor: _ExtensionFieldDescriptor[_M, Any]) -> None: ... + def HasExtension(self, field_descriptor: _ExtensionFieldDescriptor[Self, Any]) -> bool: ... + def ClearExtension(self, field_descriptor: _ExtensionFieldDescriptor[Self, Any]) -> None: ... # The TypeVar must be bound to `Message` or we get mypy errors, so we cannot use `Self` for `Extensions` @property - def Extensions(self: _M) -> _ExtensionDict[_M]: ... + def Extensions(self) -> _ExtensionDict[Self]: ... def ByteSize(self) -> int: ... @classmethod def FromString(cls, s: bytes) -> Self: ... diff --git a/contrib/python/types-protobuf/google-stubs/protobuf/message_factory.pyi b/contrib/python/types-protobuf/google-stubs/protobuf/message_factory.pyi index 518e1251955..6422284aaad 100644 --- a/contrib/python/types-protobuf/google-stubs/protobuf/message_factory.pyi +++ b/contrib/python/types-protobuf/google-stubs/protobuf/message_factory.pyi @@ -9,8 +9,6 @@ from google.protobuf.message import Message class MessageFactory: pool: Any def __init__(self, pool: DescriptorPool | None = None) -> None: ... - def GetPrototype(self, descriptor: Descriptor) -> type[Message]: ... - def GetMessages(self, files: Iterable[str]) -> dict[str, type[Message]]: ... def GetMessageClass(descriptor: Descriptor) -> type[Message]: ... def GetMessageClassesForFiles(files: Iterable[str], pool: DescriptorPool) -> dict[str, type[Message]]: ... diff --git a/contrib/python/types-protobuf/google-stubs/protobuf/reflection.pyi b/contrib/python/types-protobuf/google-stubs/protobuf/reflection.pyi index 5f7822363b1..2836b3fcf11 100644 --- a/contrib/python/types-protobuf/google-stubs/protobuf/reflection.pyi +++ b/contrib/python/types-protobuf/google-stubs/protobuf/reflection.pyi @@ -1,5 +1,2 @@ class GeneratedProtocolMessageType(type): def __new__(cls, name, bases, dictionary): ... - -def ParseMessage(descriptor, byte_str): ... -def MakeClass(descriptor): ... diff --git a/contrib/python/types-protobuf/google-stubs/protobuf/service.pyi b/contrib/python/types-protobuf/google-stubs/protobuf/service.pyi deleted file mode 100644 index 1123b6134dd..00000000000 --- a/contrib/python/types-protobuf/google-stubs/protobuf/service.pyi +++ /dev/null @@ -1,39 +0,0 @@ -from collections.abc import Callable -from concurrent.futures import Future - -from google.protobuf.descriptor import MethodDescriptor, ServiceDescriptor -from google.protobuf.message import Message - -class RpcException(Exception): ... - -class Service: - @staticmethod - def GetDescriptor() -> ServiceDescriptor: ... - def CallMethod( - self, - method_descriptor: MethodDescriptor, - rpc_controller: RpcController, - request: Message, - done: Callable[[Message], None] | None, - ) -> Future[Message] | None: ... - def GetRequestClass(self, method_descriptor: MethodDescriptor) -> type[Message]: ... - def GetResponseClass(self, method_descriptor: MethodDescriptor) -> type[Message]: ... - -class RpcController: - def Reset(self) -> None: ... - def Failed(self) -> bool: ... - def ErrorText(self) -> str | None: ... - def StartCancel(self) -> None: ... - def SetFailed(self, reason: str) -> None: ... - def IsCanceled(self) -> bool: ... - def NotifyOnCancel(self, callback: Callable[[], None]) -> None: ... - -class RpcChannel: - def CallMethod( - self, - method_descriptor: MethodDescriptor, - rpc_controller: RpcController, - request: Message, - response_class: type[Message], - done: Callable[[Message], None] | None, - ) -> Future[Message] | None: ... diff --git a/contrib/python/types-protobuf/ya.make b/contrib/python/types-protobuf/ya.make index 14901831773..e93e7b16e84 100644 --- a/contrib/python/types-protobuf/ya.make +++ b/contrib/python/types-protobuf/ya.make @@ -2,7 +2,7 @@ PY3_LIBRARY() -VERSION(5.29.1.20250208) +VERSION(6.30.2.20250506) LICENSE(Apache-2.0) @@ -10,9 +10,11 @@ NO_LINT() PY_SRCS( TOP_LEVEL + google-stubs/_upb/_message.pyi google-stubs/protobuf/__init__.pyi google-stubs/protobuf/compiler/__init__.pyi google-stubs/protobuf/descriptor.pyi + google-stubs/protobuf/descriptor_database.pyi google-stubs/protobuf/descriptor_pool.pyi google-stubs/protobuf/internal/__init__.pyi google-stubs/protobuf/internal/api_implementation.pyi @@ -32,7 +34,6 @@ PY_SRCS( google-stubs/protobuf/message_factory.pyi google-stubs/protobuf/reflection.pyi google-stubs/protobuf/runtime_version.pyi - google-stubs/protobuf/service.pyi google-stubs/protobuf/symbol_database.pyi google-stubs/protobuf/text_format.pyi google-stubs/protobuf/util/__init__.pyi diff --git a/contrib/restricted/abseil-cpp/.yandex_meta/__init__.py b/contrib/restricted/abseil-cpp/.yandex_meta/__init__.py index fd7a5f8b449..47edca28489 100644 --- a/contrib/restricted/abseil-cpp/.yandex_meta/__init__.py +++ b/contrib/restricted/abseil-cpp/.yandex_meta/__init__.py @@ -231,7 +231,7 @@ abseil_cpp = CMakeNinjaNixProject( "absl_strings": [ # FIXME thegeorg@: # put crc libraries together with strings libraries - # to resolve dependency loop around absl_crc_cor + # to resolve dependency loop around absl_crc_cord_state "absl_crc32c", "absl_crc_cpu_detect", "absl_crc_internal", diff --git a/contrib/tools/cython/.dist-info/METADATA b/contrib/tools/cython/.dist-info/METADATA index 197a6e14edc..6364708211f 100644 --- a/contrib/tools/cython/.dist-info/METADATA +++ b/contrib/tools/cython/.dist-info/METADATA @@ -1,6 +1,6 @@ Metadata-Version: 2.1 Name: Cython -Version: 3.0.11 +Version: 3.0.12 Summary: The Cython compiler for writing C extensions in the Python language. Home-page: https://cython.org/ Author: Robert Bradshaw, Stefan Behnel, Dag Seljebotn, Greg Ewing, et al. @@ -28,9 +28,12 @@ Classifier: Programming Language :: Python :: 3.9 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Programming Language :: Python :: Implementation :: Stackless Classifier: Programming Language :: C +Classifier: Programming Language :: C++ Classifier: Programming Language :: Cython Classifier: Topic :: Software Development :: Code Generators Classifier: Topic :: Software Development :: Compilers diff --git a/contrib/tools/cython/.yandex_meta/override.nix b/contrib/tools/cython/.yandex_meta/override.nix index 68cfcf5e2ba..bdcb08cfc03 100644 --- a/contrib/tools/cython/.yandex_meta/override.nix +++ b/contrib/tools/cython/.yandex_meta/override.nix @@ -1,10 +1,10 @@ pkgs: attrs: with pkgs; with pkgs.python311.pkgs; with attrs; rec { - version = "3.0.11"; + version = "3.0.12"; src = fetchPypi { pname = "cython"; inherit version; - hash = "sha256-cUbdKvhoK0ymEzGFHmrrzp/lFY51MAND+AwHyoCx+v8="; + hash = "sha256-uYi7KXznbGceKMl9AXuVQRAQ98d/pmI90LtH7tGu4bw="; }; patches = []; diff --git a/contrib/tools/cython/CHANGES.rst b/contrib/tools/cython/CHANGES.rst index 895bc335486..6d1ccad7e6d 100644 --- a/contrib/tools/cython/CHANGES.rst +++ b/contrib/tools/cython/CHANGES.rst @@ -2,6 +2,35 @@ Cython Changelog ================ +3.0.12 (2025-02-11) +=================== + +Bugs fixed +---------- + +* Release 3.0.11 introduced some incorrect ``noexcept`` warnings. + (Github issue :issue:`6335`) + +* Conditional assignments to variables using the walrus operator could crash. + (Github issue :issue:`6094`) + +* Dict assignments to struct members with reserved C names could generate invalid C code. + +* Fused ctuples with the same entry types but different sizes could fail to compile. + (Github issue :issue:`6328`) + +* In Py3, `pyximport` was not searching `sys.path` when looking for importable source files. + (Github issue :issue:`5615`) + +* Using `& 0` on integers produced with `int.from_bytes()` could read invalid memory on Python 3.10. + (Github issue :issue:`6480`) + +* Modules could fail to compile in PyPy 3.11 due to missing CPython specific header files. + Patch by Matti Picus. (Github issue :issue:`6482`) + +* Minor fix in C++ ``partial_sum()`` declaration. + + 3.0.11 (2024-08-05) =================== diff --git a/contrib/tools/cython/Cython/Compiler/Code.py b/contrib/tools/cython/Cython/Compiler/Code.py index cd7ca03443d..395dbbd5ff8 100644 --- a/contrib/tools/cython/Cython/Compiler/Code.py +++ b/contrib/tools/cython/Cython/Compiler/Code.py @@ -140,7 +140,7 @@ class IncludeCode(object): # order int: sorting order (automatically set by increasing counter) # Constants for location. If the same include occurs with different - # locations, the earliest one takes precedense. + # locations, the earliest one takes precedence. INITIAL = 0 EARLY = 1 LATE = 2 diff --git a/contrib/tools/cython/Cython/Compiler/ExprNodes.py b/contrib/tools/cython/Cython/Compiler/ExprNodes.py index 952617e375d..756c5f90ad5 100644 --- a/contrib/tools/cython/Cython/Compiler/ExprNodes.py +++ b/contrib/tools/cython/Cython/Compiler/ExprNodes.py @@ -1052,7 +1052,7 @@ class ExprNode(Node): src = CoerceToPyTypeNode(src, env, type=dst_type) # FIXME: I would expect that CoerceToPyTypeNode(type=dst_type) returns a value of type dst_type # but it doesn't for ctuples. Thus, we add a PyTypeTestNode which then triggers the - # Python conversion and becomes useless. That sems backwards and inefficient. + # Python conversion and becomes useless. That seems backwards and inefficient. # We should not need a PyTypeTestNode after a previous conversion above. if not src.type.subtype_of(dst_type): src = PyTypeTestNode(src, dst_type, env) @@ -9296,6 +9296,9 @@ class DictNode(ExprNode): len(self.key_value_pairs), code.error_goto_if_null(self.result(), self.pos))) self.generate_gotref(code) + struct_scope = None + else: + struct_scope = self.type.scope keys_seen = set() key_type = None @@ -9344,17 +9347,16 @@ class DictNode(ExprNode): if self.exclude_null_values: code.putln('}') else: + key = str(item.key.value) + member = struct_scope.lookup_here(key) + assert member is not None, "struct member %s not found, error was not handled during coercion" % key + key_cname = member.cname + value_cname = item.value.result() if item.value.type.is_array: code.putln("memcpy(%s.%s, %s, sizeof(%s));" % ( - self.result(), - item.key.value, - item.value.result(), - item.value.result())) + self.result(), key_cname, value_cname, value_cname)) else: - code.putln("%s.%s = %s;" % ( - self.result(), - item.key.value, - item.value.result())) + code.putln("%s.%s = %s;" % (self.result(), key_cname, value_cname)) item.generate_disposal_code(code) item.free_temps(code) diff --git a/contrib/tools/cython/Cython/Compiler/FlowControl.py b/contrib/tools/cython/Cython/Compiler/FlowControl.py index c8575435738..a9965b76c69 100644 --- a/contrib/tools/cython/Cython/Compiler/FlowControl.py +++ b/contrib/tools/cython/Cython/Compiler/FlowControl.py @@ -1381,3 +1381,45 @@ class ControlFlowAnalysis(CythonTransform): self.mark_assignment(node.operand, fake_rhs_expr) self.visitchildren(node) return node + + def visit_BoolBinopNode(self, node): + # Note - I don't believe BoolBinopResultNode needs special handling beyond this + assert len(node.subexprs) == 2 # operand1 and operand2 only + + next_block = self.flow.newblock() + parent = self.flow.block + + self._visit(node.operand1) + + self.flow.nextblock() + self._visit(node.operand2) + if self.flow.block: + self.flow.block.add_child(next_block) + + parent.add_child(next_block) + + if next_block.parents: + self.flow.block = next_block + else: + self.flow.block = None + return node + + def visit_CondExprNode(self, node): + assert len(node.subexprs) == 3 + self._visit(node.test) + parent = self.flow.block + next_block = self.flow.newblock() + self.flow.nextblock() + self._visit(node.true_val) + if self.flow.block: + self.flow.block.add_child(next_block) + self.flow.nextblock(parent=parent) + self._visit(node.false_val) + if self.flow.block: + self.flow.block.add_child(next_block) + + if next_block.parents: + self.flow.block = next_block + else: + self.flow.block = None + return node diff --git a/contrib/tools/cython/Cython/Compiler/Nodes.py b/contrib/tools/cython/Cython/Compiler/Nodes.py index e1fe6ee56c9..dde377c323a 100644 --- a/contrib/tools/cython/Cython/Compiler/Nodes.py +++ b/contrib/tools/cython/Cython/Compiler/Nodes.py @@ -3126,7 +3126,11 @@ class DefNode(FuncDefNode): if scope is None: scope = cfunc.scope cfunc_type = cfunc.type - has_explicit_exc_clause=True + if cfunc_type.exception_check: + # this ensures `legacy_implicit_noexcept` does not trigger + # as it would result in a mismatch + # (declaration with except, definition with implicit noexcept) + has_explicit_exc_clause = True if len(self.args) != len(cfunc_type.args) or cfunc_type.has_varargs: error(self.pos, "wrong number of arguments") error(cfunc.pos, "previous declaration here") diff --git a/contrib/tools/cython/Cython/Compiler/PyrexTypes.py b/contrib/tools/cython/Cython/Compiler/PyrexTypes.py index b522a131751..1080b2ef039 100644 --- a/contrib/tools/cython/Cython/Compiler/PyrexTypes.py +++ b/contrib/tools/cython/Cython/Compiler/PyrexTypes.py @@ -4678,7 +4678,8 @@ class CTupleType(CType): def c_tuple_type(components): components = tuple(components) if any(c.is_fused for c in components): - cname = "<dummy fused ctuple>" # should never end up in code + # should never end up in code but should be unique + cname = "<dummy fused ctuple %s>" % repr(components) else: cname = Naming.ctuple_type_prefix + type_list_identifier(components) tuple_type = CTupleType(cname, components) diff --git a/contrib/tools/cython/Cython/Compiler/Visitor.py b/contrib/tools/cython/Cython/Compiler/Visitor.py index 92e2eb9c0d3..105347b36d5 100644 --- a/contrib/tools/cython/Cython/Compiler/Visitor.py +++ b/contrib/tools/cython/Cython/Compiler/Visitor.py @@ -847,7 +847,8 @@ class PrintTree(TreeVisitor): elif isinstance(node, Nodes.DefNode): result += "(name=\"%s\")" % node.name elif isinstance(node, Nodes.CFuncDefNode): - result += "(name=\"%s\")" % node.declared_name() + result += "(name=\"%s\", type=\"%s\")" % ( + node.declared_name(), getattr(node, "type", None)) elif isinstance(node, ExprNodes.AttributeNode): result += "(type=%s, attribute=\"%s\")" % (repr(node.type), node.attribute) elif isinstance(node, (ExprNodes.ConstNode, ExprNodes.PyConstNode)): diff --git a/contrib/tools/cython/Cython/Includes/libcpp/numeric.pxd b/contrib/tools/cython/Cython/Includes/libcpp/numeric.pxd index a9fb37205a5..99175f82cfc 100644 --- a/contrib/tools/cython/Cython/Includes/libcpp/numeric.pxd +++ b/contrib/tools/cython/Cython/Includes/libcpp/numeric.pxd @@ -17,7 +17,7 @@ cdef extern from "<numeric>" namespace "std" nogil: void adjacent_difference[InputIt, OutputIt, BinaryOperation](InputIt in_first, InputIt in_last, OutputIt out_first, BinaryOperation op) - void partial_sum[InputIt, OutputIt](InputIt in_first, OutputIt in_last, OutputIt out_first) + void partial_sum[InputIt, OutputIt](InputIt in_first, InputIt in_last, OutputIt out_first) void partial_sum[InputIt, OutputIt, BinaryOperation](InputIt in_first, InputIt in_last, OutputIt out_first, BinaryOperation op) diff --git a/contrib/tools/cython/Cython/Shadow.py b/contrib/tools/cython/Cython/Shadow.py index 1400f1657c8..1b0953614ad 100644 --- a/contrib/tools/cython/Cython/Shadow.py +++ b/contrib/tools/cython/Cython/Shadow.py @@ -2,7 +2,7 @@ from __future__ import absolute_import # Possible version formats: "3.1.0", "3.1.0a1", "3.1.0a1.dev0" -__version__ = "3.0.11" +__version__ = "3.0.12" try: from __builtin__ import basestring diff --git a/contrib/tools/cython/Cython/Utility/Coroutine.c b/contrib/tools/cython/Cython/Utility/Coroutine.c index c7ec8ee9ba0..1aecf1d5e6b 100644 --- a/contrib/tools/cython/Cython/Utility/Coroutine.c +++ b/contrib/tools/cython/Cython/Utility/Coroutine.c @@ -501,7 +501,7 @@ static int __pyx_Generator_init(PyObject *module); /*proto*/ //@requires: ModuleSetupCode.c::IncludeStructmemberH #include <frameobject.h> -#if PY_VERSION_HEX >= 0x030b00a6 +#if PY_VERSION_HEX >= 0x030b00a6 && !defined(PYPY_VERSION) #ifndef Py_BUILD_CORE #define Py_BUILD_CORE 1 #endif diff --git a/contrib/tools/cython/Cython/Utility/Exceptions.c b/contrib/tools/cython/Cython/Utility/Exceptions.c index 46f7dd57812..ab6f69fb182 100644 --- a/contrib/tools/cython/Cython/Utility/Exceptions.c +++ b/contrib/tools/cython/Cython/Utility/Exceptions.c @@ -914,7 +914,7 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, #include "compile.h" #include "frameobject.h" #include "traceback.h" -#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) #ifndef Py_BUILD_CORE #define Py_BUILD_CORE 1 #endif diff --git a/contrib/tools/cython/Cython/Utility/ObjectHandling.c b/contrib/tools/cython/Cython/Utility/ObjectHandling.c index b0a8554a7c5..8147a907457 100644 --- a/contrib/tools/cython/Cython/Utility/ObjectHandling.c +++ b/contrib/tools/cython/Cython/Utility/ObjectHandling.c @@ -2530,7 +2530,7 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, #if !CYTHON_VECTORCALL #if PY_VERSION_HEX >= 0x03080000 #include "frameobject.h" -#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) #ifndef Py_BUILD_CORE #define Py_BUILD_CORE 1 #endif diff --git a/contrib/tools/cython/Cython/Utility/Optimize.c b/contrib/tools/cython/Cython/Utility/Optimize.c index 99e9a8db375..daa936dc83a 100644 --- a/contrib/tools/cython/Cython/Utility/Optimize.c +++ b/contrib/tools/cython/Cython/Utility/Optimize.c @@ -1281,15 +1281,6 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, long intval, PY_LONG_LONG ll{{ival}}, llx; #endif {{endif}} - {{if c_op == '&'}} - // special case for &-ing arbitrarily large numbers with known single digit operands - if ((intval & PyLong_MASK) == intval) { - // Calling PyLong_CompactValue() requires the PyLong value to be compact, we only need the last digit. - long last_digit = (long) __Pyx_PyLong_Digits({{pyval}})[0]; - long result = intval & (likely(__Pyx_PyLong_IsPos({{pyval}})) ? last_digit : (PyLong_MASK - last_digit + 1)); - return PyLong_FromLong(result); - } - {{endif}} // special cases for 0: + - * % / // | ^ & >> << if (unlikely(__Pyx_PyLong_IsZero({{pyval}}))) { {{if order == 'CObj' and c_op in '%/'}} @@ -1312,6 +1303,15 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, long intval, return __Pyx_NewRef(op1); {{endif}} } + {{if c_op == '&'}} + // special case for &-ing arbitrarily large numbers with known single digit operands + if ((intval & PyLong_MASK) == intval) { + // Calling PyLong_CompactValue() requires the PyLong value to be compact, we only need the last digit. + long last_digit = (long) __Pyx_PyLong_Digits({{pyval}})[0]; + long result = intval & (likely(__Pyx_PyLong_IsPos({{pyval}})) ? last_digit : (PyLong_MASK - last_digit + 1)); + return PyLong_FromLong(result); + } + {{endif}} // handle most common case first to avoid indirect branch and optimise branch prediction if (likely(__Pyx_PyLong_IsCompact({{pyval}}))) { {{ival}} = __Pyx_PyLong_CompactValue({{pyval}}); diff --git a/contrib/tools/cython/Cython/Utility/Overflow.c b/contrib/tools/cython/Cython/Utility/Overflow.c index 395456c8721..0ae375d78c3 100644 --- a/contrib/tools/cython/Cython/Utility/Overflow.c +++ b/contrib/tools/cython/Cython/Utility/Overflow.c @@ -1,7 +1,7 @@ /* These functions provide integer arithmetic with integer checking. They do not actually raise an exception when an overflow is detected, but rather set a bit -in the overflow parameter. (This parameter may be re-used across several +in the overflow parameter. (This parameter may be reused across several arithmetic operations, so should be or-ed rather than assigned to.) The implementation is divided into two parts, the signed and unsigned basecases, diff --git a/contrib/tools/cython/Cython/Utility/Profile.c b/contrib/tools/cython/Cython/Utility/Profile.c index 2b8564b226f..3b4d8514949 100644 --- a/contrib/tools/cython/Cython/Utility/Profile.c +++ b/contrib/tools/cython/Cython/Utility/Profile.c @@ -38,7 +38,7 @@ #include "compile.h" #include "frameobject.h" #include "traceback.h" -#if PY_VERSION_HEX >= 0x030b00a6 +#if PY_VERSION_HEX >= 0x030b00a6 && !defined(PYPY_VERSION) #ifndef Py_BUILD_CORE #define Py_BUILD_CORE 1 #endif diff --git a/contrib/tools/cython/cython.py b/contrib/tools/cython/cython.py index d8ab80706b9..951184229b0 100755 --- a/contrib/tools/cython/cython.py +++ b/contrib/tools/cython/cython.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Change content of this file to change uids for cython programs - cython 3.0.11 r0 +# Change content of this file to change uids for cython programs - cython 3.0.12 r0 # # Cython -- Main Program, generic diff --git a/contrib/tools/cython/patches/pr6343.patch b/contrib/tools/cython/patches/pr6343.patch deleted file mode 100644 index faffef9e0b8..00000000000 --- a/contrib/tools/cython/patches/pr6343.patch +++ /dev/null @@ -1,36 +0,0 @@ -From 2e774fca5ad55371b21ba2b531d218888319c151 Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Gonzalo=20Tornar=C3=ADa?= <tornaria@cmat.edu.uy> -Date: Sun, 18 Aug 2024 17:43:05 -0300 -Subject: [PATCH 1/2] Better fix for #6122 to avoid #6535 - -The change in #6124 introduces a regression with functions that are -implicit noexcept in a pxd file. ---- - Cython/Compiler/Nodes.py | 5 ++--- - 1 file changed, 2 insertions(+), 3 deletions(-) - -diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py -index d4737f7c373..b70d766ed4e 100644 ---- a/Cython/Compiler/Nodes.py -+++ b/Cython/Compiler/Nodes.py -@@ -710,10 +710,8 @@ def analyse(self, return_type, env, nonempty=0, directive_locals=None, visibilit - and not self.has_explicit_exc_clause - and self.exception_check - and visibility != 'extern'): -- # If function is already declared from pxd, the exception_check has already correct value. -- if not (self.declared_name() in env.entries and not in_pxd): -- self.exception_check = False - # implicit noexcept, with a warning -+ self.exception_check = False - warning(self.pos, - "Implicit noexcept declaration is deprecated." - " Function declaration should contain 'noexcept' keyword.", -@@ -3128,6 +3126,7 @@ def as_cfunction(self, cfunc=None, scope=None, overridable=True, returns=None, e - if scope is None: - scope = cfunc.scope - cfunc_type = cfunc.type -+ has_explicit_exc_clause=True - if len(self.args) != len(cfunc_type.args) or cfunc_type.has_varargs: - error(self.pos, "wrong number of arguments") - error(cfunc.pos, "previous declaration here") - diff --git a/contrib/tools/cython/ya.make b/contrib/tools/cython/ya.make index 38619e6d9b4..86802cf6b30 100644 --- a/contrib/tools/cython/ya.make +++ b/contrib/tools/cython/ya.make @@ -11,9 +11,9 @@ LICENSE_TEXTS(.yandex_meta/licenses.list.txt) SUBSCRIBER(g:python-contrib) -VERSION(3.0.11) +VERSION(3.0.12) -ORIGINAL_SOURCE(mirror://pypi/c/cython/cython-3.0.11.tar.gz) +ORIGINAL_SOURCE(mirror://pypi/c/cython/cython-3.0.12.tar.gz) NO_LINT() diff --git a/library/cpp/http/simple/http_client.cpp b/library/cpp/http/simple/http_client.cpp index 6fb04755391..87bbabc3f83 100644 --- a/library/cpp/http/simple/http_client.cpp +++ b/library/cpp/http/simple/http_client.cpp @@ -6,15 +6,25 @@ #include <util/string/cast.h> #include <util/string/join.h> #include <util/string/split.h> +#include <util/system/spinlock.h> +#include <library/cpp/cache/cache.h> + + +TSpinLock TKeepAliveHttpClient::ConnectionQuarantineMutex; +TQueue<THolder<NPrivate::THttpConnection>> TKeepAliveHttpClient::ConnectionQuarantine; TKeepAliveHttpClient::TKeepAliveHttpClient(const TString& host, ui32 port, TDuration socketTimeout, - TDuration connectTimeout) + TDuration connectTimeout, + bool useKeepAlive, + bool useConnectionPool) : Host(CutHttpPrefix(host)) , Port(port) , SocketTimeout(socketTimeout) , ConnectTimeout(connectTimeout) + , UseKeepAlive(useKeepAlive) + , UseConnectionPool(useConnectionPool) , IsHttps(host.StartsWith("https")) , IsClosingRequired(false) , HttpsVerification(TVerifyCert{Host}) @@ -159,13 +169,29 @@ bool TKeepAliveHttpClient::CreateNewConnectionIfNeeded() { ConnectTimeout, IsHttps, ClientCertificate, - HttpsVerification); + HttpsVerification, + UseKeepAlive); IsClosingRequired = false; return true; } return false; } +TKeepAliveHttpClient::~TKeepAliveHttpClient() { + if (UseConnectionPool) { + THolder<NPrivate::THttpConnection> oldConnection; + with_lock(ConnectionQuarantineMutex) { + while (ConnectionQuarantine.size() > 100) { + oldConnection = std::move(ConnectionQuarantine.front()); + + ConnectionQuarantine.pop(); + oldConnection.Reset(); + } + ConnectionQuarantine.push(std::move(Connection)); + } + } +} + THttpRequestException::THttpRequestException(int statusCode) : StatusCode(statusCode) { @@ -180,6 +206,8 @@ TSimpleHttpClient::TSimpleHttpClient(const TOptions& options) , Port(options.Port()) , SocketTimeout(options.SocketTimeout()) , ConnectTimeout(options.ConnectTimeout()) + , UseKeepAlive(options.UseKeepAlive()) + , UseConnectionPool(options.UseConnectionPool()) { } @@ -229,7 +257,8 @@ namespace NPrivate { TDuration connTimeout, bool isHttps, const TMaybe<TOpenSslClientIO::TOptions::TClientCert>& clientCert, - const TMaybe<TOpenSslClientIO::TOptions::TVerifyCert>& verifyCert) + const TMaybe<TOpenSslClientIO::TOptions::TVerifyCert>& verifyCert, + bool keepAlive) : Addr(Resolve(host, port)) , Socket(Connect(Addr, sockTimeout, connTimeout, host, port)) , SocketIn(Socket) @@ -249,8 +278,9 @@ namespace NPrivate { } else { HttpOut = MakeHolder<THttpOutput>(&SocketOut); } - - HttpOut->EnableKeepAlive(true); + if (keepAlive) { + HttpOut->EnableKeepAlive(true); + } } TNetworkAddress THttpConnection::Resolve(const TString& host, ui32 port) { @@ -292,7 +322,7 @@ TSimpleHttpClient::~TSimpleHttpClient() { } TKeepAliveHttpClient TSimpleHttpClient::CreateClient() const { - TKeepAliveHttpClient cl(Host, Port, SocketTimeout, ConnectTimeout); + TKeepAliveHttpClient cl(Host, Port, SocketTimeout, ConnectTimeout, UseKeepAlive, UseConnectionPool); if (!HttpsVerification) { cl.DisableVerificationForHttps(); diff --git a/library/cpp/http/simple/http_client.h b/library/cpp/http/simple/http_client.h index 3860862698d..87d3dc095af 100644 --- a/library/cpp/http/simple/http_client.h +++ b/library/cpp/http/simple/http_client.h @@ -8,6 +8,8 @@ #include <util/generic/strbuf.h> #include <util/generic/yexception.h> #include <util/network/socket.h> +#include <util/generic/queue.h> +#include <util/system/spinlock.h> #include <library/cpp/http/io/stream.h> #include <library/cpp/http/misc/httpcodes.h> @@ -50,7 +52,12 @@ public: TKeepAliveHttpClient(const TString& host, ui32 port, TDuration socketTimeout = TDuration::Seconds(5), - TDuration connectTimeout = TDuration::Seconds(30)); + TDuration connectTimeout = TDuration::Seconds(30), + bool useKeepAlive = true, + bool useConnectionPool = false); + + TKeepAliveHttpClient(TKeepAliveHttpClient&&) = default; + ~TKeepAliveHttpClient(); THttpCode DoGet(const TStringBuf relativeUrl, IOutputStream* output = nullptr, @@ -117,8 +124,13 @@ private: const ui32 Port; const TDuration SocketTimeout; const TDuration ConnectTimeout; + const bool UseKeepAlive; + const bool UseConnectionPool; const bool IsHttps; + static TSpinLock ConnectionQuarantineMutex; + static TQueue<THolder<NPrivate::THttpConnection>> ConnectionQuarantine; + THolder<NPrivate::THttpConnection> Connection; bool IsClosingRequired; TMaybe<TClientCert> ClientCertificate; @@ -158,6 +170,8 @@ protected: const ui32 Port; const TDuration SocketTimeout; const TDuration ConnectTimeout; + const bool UseKeepAlive = true; + const bool UseConnectionPool = false; bool HttpsVerification = false; public: @@ -215,7 +229,8 @@ namespace NPrivate { TDuration connTimeout, bool isHttps, const TMaybe<TOpenSslClientIO::TOptions::TClientCert>& clientCert, - const TMaybe<TOpenSslClientIO::TOptions::TVerifyCert>& verifyCert); + const TMaybe<TOpenSslClientIO::TOptions::TVerifyCert>& verifyCert, + bool keepAlive = true); bool IsOk() const { return IsNotSocketClosedByOtherSide(Socket); diff --git a/library/cpp/http/simple/http_client_options.h b/library/cpp/http/simple/http_client_options.h index 58849556a91..f237a3770a9 100644 --- a/library/cpp/http/simple/http_client_options.h +++ b/library/cpp/http/simple/http_client_options.h @@ -61,10 +61,30 @@ public: return MaxRedirectCount_; } + TSelf& UseKeepAlive(bool useKeepAlive) { + UseKeepAlive_ = useKeepAlive; + return *this; + } + + bool UseKeepAlive() const noexcept { + return UseKeepAlive_; + } + + TSelf& UseConnectionPool(bool useConnectionPool) { + UseConnectionPool_ = useConnectionPool; + return *this; + } + + bool UseConnectionPool() const noexcept { + return UseConnectionPool_; + } + private: TString Host_; ui16 Port_; TDuration SocketTimeout_ = TDuration::Seconds(5); TDuration ConnectTimeout_ = TDuration::Seconds(30); int MaxRedirectCount_ = INT_MAX; + bool UseKeepAlive_ = true; + bool UseConnectionPool_ = false; }; diff --git a/library/cpp/http/simple/ya.make b/library/cpp/http/simple/ya.make index 6a4e5775a4d..c645fb38442 100644 --- a/library/cpp/http/simple/ya.make +++ b/library/cpp/http/simple/ya.make @@ -1,6 +1,7 @@ LIBRARY() PEERDIR( + library/cpp/cache library/cpp/http/io library/cpp/openssl/io library/cpp/string_utils/url diff --git a/library/cpp/int128/int128.h b/library/cpp/int128/int128.h index 41f143e2959..47e24ff2b77 100644 --- a/library/cpp/int128/int128.h +++ b/library/cpp/int128/int128.h @@ -243,6 +243,8 @@ public: return ret; } + bool operator==(const TInteger128& a) const = default; + explicit constexpr operator bool() const noexcept { return Low_ || High_; } @@ -478,14 +480,6 @@ namespace std { }; } -constexpr bool operator==(const ui128 lhs, const ui128 rhs) noexcept { - return GetLow(lhs) == GetLow(rhs) && GetHigh(lhs) == GetHigh(rhs); -} - -constexpr bool operator==(const i128 lhs, const i128 rhs) noexcept { - return GetLow(lhs) == GetLow(rhs) && GetHigh(lhs) == GetHigh(rhs); -} - constexpr bool operator!=(const ui128 lhs, const ui128 rhs) noexcept { return !(lhs == rhs); } diff --git a/library/cpp/int128/ut/i128_find_ut.cpp b/library/cpp/int128/ut/i128_find_ut.cpp new file mode 100644 index 00000000000..afc19b6dbae --- /dev/null +++ b/library/cpp/int128/ut/i128_find_ut.cpp @@ -0,0 +1,16 @@ +#include <library/cpp/testing/unittest/registar.h> + +#include <library/cpp/int128/int128.h> + +#include <util/generic/cast.h> +#include <util/generic/vector.h> + +#include <type_traits> + +Y_UNIT_TEST_SUITE(Int128FindSuite) { + Y_UNIT_TEST(Int128Find) { + const ui128 value = 10; + TVector<ui128> list = {1, 2, 3, 4, 5, 11, 10}; + UNIT_ASSERT(Find(list, value) != list.end()); + } +} diff --git a/library/cpp/int128/ut/ya.make b/library/cpp/int128/ut/ya.make index 27b4b4dc884..2ee605429aa 100644 --- a/library/cpp/int128/ut/ya.make +++ b/library/cpp/int128/ut/ya.make @@ -12,6 +12,7 @@ SRCS( i128_and_intrinsic_identity_ut.cpp i128_comparison_ut.cpp i128_division_ut.cpp + i128_find_ut.cpp i128_type_traits_ut.cpp ui128_division_ut.cpp ) diff --git a/library/cpp/neh/https.cpp b/library/cpp/neh/https.cpp index 99db8a44cc2..e14dcf86679 100644 --- a/library/cpp/neh/https.cpp +++ b/library/cpp/neh/https.cpp @@ -13,7 +13,6 @@ #include <openssl/err.h> #include <openssl/x509v3.h> -#include <library/cpp/openssl/init/init.h> #include <library/cpp/openssl/method/io.h> #include <library/cpp/coroutine/listener/listen.h> #include <library/cpp/dns/cache.h> @@ -358,12 +357,6 @@ namespace NNeh { } } } - - struct TSSLInit { - inline TSSLInit() { - InitOpenSSL(); - } - } SSL_INIT; } static inline void PrepareSocket(SOCKET s) { diff --git a/library/cpp/openssl/crypto/rsa.cpp b/library/cpp/openssl/crypto/rsa.cpp index 4b1d6648268..c9d2e07b89d 100644 --- a/library/cpp/openssl/crypto/rsa.cpp +++ b/library/cpp/openssl/crypto/rsa.cpp @@ -1,7 +1,6 @@ #include "rsa.h" #include <library/cpp/openssl/big_integer/big_integer.h> -#include <library/cpp/openssl/init/init.h> #include <util/generic/yexception.h> #include <util/generic/buffer.h> @@ -12,14 +11,6 @@ using namespace NOpenSsl; using namespace NOpenSsl::NRsa; -namespace { - struct TInit { - inline TInit() { - InitOpenSSL(); - } - } INIT; -} - TPublicKey::TPublicKey(const TBigInteger& e, const TBigInteger& n) : Key_(RSA_new()) { diff --git a/library/cpp/openssl/init/init.cpp b/library/cpp/openssl/init/init.cpp index ae68ef08eaa..9a4e285ea35 100644 --- a/library/cpp/openssl/init/init.cpp +++ b/library/cpp/openssl/init/init.cpp @@ -1,66 +1,13 @@ -#include "init.h" - -#include <util/generic/singleton.h> -#include <util/generic/vector.h> -#include <util/generic/ptr.h> -#include <util/generic/buffer.h> - -#include <util/system/yassert.h> -#include <util/system/mutex.h> -#include <util/system/thread.h> - -#include <util/random/entropy.h> -#include <util/stream/input.h> - -#include <openssl/bio.h> -#include <openssl/ssl.h> -#include <openssl/err.h> -#include <openssl/rand.h> -#include <openssl/conf.h> #include <openssl/crypto.h> namespace { - struct TInitSsl { - struct TOpensslLocks { - inline TOpensslLocks() - : Mutexes(CRYPTO_num_locks()) - { - for (auto& mpref : Mutexes) { - mpref.Reset(new TMutex()); - } - } - - inline void LockOP(int mode, int n) { - auto& mutex = *Mutexes.at(n); - - if (mode & CRYPTO_LOCK) { - mutex.Acquire(); - } else { - mutex.Release(); - } - } - - TVector<TAutoPtr<TMutex>> Mutexes; - }; - - inline TInitSsl() { - OPENSSL_init_crypto(OPENSSL_INIT_NO_ATEXIT, nullptr); - } - - inline ~TInitSsl() { - OPENSSL_cleanup(); - } - - static void LockingFunction(int mode, int n, const char* /*file*/, int /*line*/) { - Singleton<TOpensslLocks>()->LockOP(mode, n); - } - - static unsigned long ThreadIdFunction() { - return TThread::CurrentThreadId(); - } - }; + // Initialize OpenSSL as early as possible + // in order to prevent any further initializations with different flags. + // + // Initialize it with OPENSSL_INIT_NO_ATEXIT thus omitting the cleanup routine at process exit + // (it looks like it does nothing when openssl is linked statically). + [[maybe_unused]] auto _ = OPENSSL_init_crypto(OPENSSL_INIT_ENGINE_ALL_BUILTIN | OPENSSL_INIT_NO_ATEXIT, nullptr); } void InitOpenSSL() { - (void)SingletonWithPriority<TInitSsl, 0>(); } diff --git a/library/cpp/openssl/init/ya.make b/library/cpp/openssl/init/ya.make index 1c39d273801..2d49965ca88 100644 --- a/library/cpp/openssl/init/ya.make +++ b/library/cpp/openssl/init/ya.make @@ -5,7 +5,7 @@ PEERDIR( ) SRCS( - init.cpp + GLOBAL init.cpp ) END() diff --git a/library/cpp/openssl/io/stream.cpp b/library/cpp/openssl/io/stream.cpp index 0b4be38c0e3..2666988728f 100644 --- a/library/cpp/openssl/io/stream.cpp +++ b/library/cpp/openssl/io/stream.cpp @@ -4,7 +4,6 @@ #include <util/generic/singleton.h> #include <util/generic/yexception.h> -#include <library/cpp/openssl/init/init.h> #include <library/cpp/openssl/method/io.h> #include <library/cpp/resource/resource.h> @@ -19,12 +18,6 @@ using TOptions = TOpenSslClientIO::TOptions; namespace { struct TSslIO; - struct TSslInitOnDemand { - inline TSslInitOnDemand() { - InitOpenSSL(); - } - }; - int GetLastSslError() noexcept { return ERR_peek_last_error(); } @@ -121,7 +114,7 @@ namespace { IOutputStream* Out; }; - struct TSslIO: public TSslInitOnDemand, public TOptions { + struct TSslIO: public TOptions { inline TSslIO(IInputStream* in, IOutputStream* out, const TOptions& opts) : TOptions(opts) , Io(in, out) diff --git a/library/cpp/tld/tlds-alpha-by-domain.txt b/library/cpp/tld/tlds-alpha-by-domain.txt index 040f4f01ac7..fc8f36a83ec 100644 --- a/library/cpp/tld/tlds-alpha-by-domain.txt +++ b/library/cpp/tld/tlds-alpha-by-domain.txt @@ -1,4 +1,4 @@ -# Version 2025051600, Last Updated Fri May 16 07:07:02 2025 UTC +# Version 2025052800, Last Updated Wed May 28 07:07:02 2025 UTC AAA AARP ABB @@ -912,7 +912,6 @@ POLITIE PORN POST PR -PRAMERICA PRAXI PRESS PRIME diff --git a/library/cpp/unified_agent_client/logger.h b/library/cpp/unified_agent_client/logger.h index 8130c2894c3..01c3e0ed747 100644 --- a/library/cpp/unified_agent_client/logger.h +++ b/library/cpp/unified_agent_client/logger.h @@ -35,6 +35,17 @@ YLOG(TLOG_CRIT, msg, Logger); \ _Exit(1); +#define YLOG_EMERG_F(fmt, ...) YLOG_EMERG(std::format(fmt, __VA_ARGS__)) +#define YLOG_ALERT_F(fmt, ...) YLOG_ALERT(std::format(fmt, __VA_ARGS__)) +#define YLOG_CRIT_F(fmt, ...) YLOG_CRIT(std::format(fmt, __VA_ARGS__)) +#define YLOG_ERR_F(fmt, ...) YLOG_ERR(std::format(fmt, __VA_ARGS__)) +#define YLOG_WARNING_F(fmt, ...) YLOG_WARNING(std::format(fmt, __VA_ARGS__)) +#define YLOG_NOTICE_F(fmt, ...) YLOG_NOTICE(std::format(fmt, __VA_ARGS__)) +#define YLOG_INFO_F(fmt, ...) YLOG_INFO(std::format(fmt, __VA_ARGS__)) +#define YLOG_DEBUG_F(fmt, ...) YLOG_DEBUG(std::format(fmt, __VA_ARGS__)) +#define YLOG_RESOURCES_F(fmt, ...) YLOG_RESOURCES(std::format(fmt, __VA_ARGS__)) +#define YLOG_FATAL_F(fmt, ...) YLOG_FATAL(std::format(fmt, __VA_ARGS__)) + namespace NUnifiedAgent { class TScopeLogger; diff --git a/library/cpp/uri/common.h b/library/cpp/uri/common.h index 23be2571fc5..7945340f089 100644 --- a/library/cpp/uri/common.h +++ b/library/cpp/uri/common.h @@ -366,7 +366,9 @@ namespace NUri { NewFeaturesRecommended = 0 | FeatureSchemeKnown | FeatureRemoteOnly | FeatureToLower | FeatureCheckHost | FeatureConvertHostIDN | FeatureFragmentToHashBang | FeatureEncodeSpace | FeatureEncodeCntrl | FeatureEncodeExtendedASCII | FeatureUpperEncoded | FeatureDecodeUnreserved | FeaturePathOperation | FeaturePathStripRootParent, // FeaturesRobot is deprecated, use NewFeaturesRecommended: ROBOTQUALITY-718 - FeaturesRobot = FeaturesRecommended + FeaturesRobot = FeaturesRecommended, + + FeaturesDefaultOrSchemeKnown = 0 | FeaturesDefault | FeatureSchemeKnown }; }; diff --git a/tools/cpp_style_checker/__main__.py b/tools/cpp_style_checker/__main__.py index 4ca1bc3a0f9..a3d303e8584 100644 --- a/tools/cpp_style_checker/__main__.py +++ b/tools/cpp_style_checker/__main__.py @@ -65,8 +65,13 @@ def check_file(clang_format_binary, style_config_json, filename): if styled_source == actual_source: return reporter.LintStatus.GOOD, "" else: - diff = make_diff(actual_source, styled_source) - return reporter.LintStatus.FAIL, diff + # FIXME(YA-2574) remove retry + styled_source = subprocess.check_output(command, input=actual_source) + if styled_source == actual_source: + return reporter.LintStatus.GOOD, "" + else: + diff = make_diff(actual_source, styled_source) + return reporter.LintStatus.FAIL, diff def make_diff(left, right): diff --git a/vendor/golang.org/x/crypto/pbkdf2/ya.make b/vendor/golang.org/x/crypto/pbkdf2/ya.make index 707da3616c7..faa80cfb49d 100644 --- a/vendor/golang.org/x/crypto/pbkdf2/ya.make +++ b/vendor/golang.org/x/crypto/pbkdf2/ya.make @@ -2,7 +2,7 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) -VERSION(v0.33.0) +VERSION(v0.36.0) SRCS( pbkdf2.go diff --git a/vendor/golang.org/x/crypto/scrypt/ya.make b/vendor/golang.org/x/crypto/scrypt/ya.make index 5ba22b0c2d1..6fc75e76a8a 100644 --- a/vendor/golang.org/x/crypto/scrypt/ya.make +++ b/vendor/golang.org/x/crypto/scrypt/ya.make @@ -2,7 +2,7 @@ GO_LIBRARY() LICENSE(BSD-3-Clause) -VERSION(v0.33.0) +VERSION(v0.36.0) SRCS( scrypt.go @@ -39,33 +39,33 @@ REGISTRY_ENDPOINT = os.environ.get("YA_REGISTRY_ENDPOINT", "https://devtools-reg PLATFORM_MAP = { "data": { "win32": { - "md5": "700385eaf8a059b387a08c8554c38bd0", + "md5": "a35cfa461e491148f28678d3003faacc", "urls": [ - f"{REGISTRY_ENDPOINT}/8689723252" + f"{REGISTRY_ENDPOINT}/8850028615" ] }, "darwin": { - "md5": "703aa992a2568a97ae0cd261bf9b58a6", + "md5": "71cdd2288787dc00033a993bad73cd9f", "urls": [ - f"{REGISTRY_ENDPOINT}/8689718827" + f"{REGISTRY_ENDPOINT}/8850027607" ] }, "darwin-arm64": { - "md5": "96b0ee82f13aa73908c2726b84925aea", + "md5": "101f95cee3a660ce04084e0541193a7e", "urls": [ - f"{REGISTRY_ENDPOINT}/8689714545" + f"{REGISTRY_ENDPOINT}/8850026111" ] }, "linux-aarch64": { - "md5": "8245182717411b9c75525edf913d4a8a", + "md5": "a474afa5f85e3fdbde05052b947e97aa", "urls": [ - f"{REGISTRY_ENDPOINT}/8689710653" + f"{REGISTRY_ENDPOINT}/8850025012" ] }, "linux": { - "md5": "499519b40142e3444cec4dba8254edee", + "md5": "bbb8672b210476e787700bca7287b3b5", "urls": [ - f"{REGISTRY_ENDPOINT}/8689727626" + f"{REGISTRY_ENDPOINT}/8850029468" ] } } @@ -18,6 +18,7 @@ tools_cache_master = true use_atd_revisions_info = true use_jstyle_server = true use_command_file_in_testtool = true +use_universal_fetcher_everywhere = false # ===== opensource only table params ===== diff --git a/ydb/ci/rightlib.txt b/ydb/ci/rightlib.txt index 7b9914b9656..2b19faa77b9 100644 --- a/ydb/ci/rightlib.txt +++ b/ydb/ci/rightlib.txt @@ -1 +1 @@ -bb919ff615bff1f8a16a321843cd843497ae83d1 +fdbc38349df2ee0ddc678fa2bffe84786f9639a3 diff --git a/ydb/core/backup/ya.make b/ydb/core/backup/ya.make index 956b0076d1f..efee8b43117 100644 --- a/ydb/core/backup/ya.make +++ b/ydb/core/backup/ya.make @@ -1,6 +1,5 @@ RECURSE( common - common/proto controller impl ) diff --git a/ydb/library/analytics/ya.make b/ydb/library/analytics/ya.make index 5c20a3f42c2..324f3fffc09 100644 --- a/ydb/library/analytics/ya.make +++ b/ydb/library/analytics/ya.make @@ -9,7 +9,3 @@ SRCS( ) END() - -RECURSE( - protos -) diff --git a/ydb/library/planner/share/ya.make b/ydb/library/planner/share/ya.make index ea6a5b3c9c6..9f0c97f2e80 100644 --- a/ydb/library/planner/share/ya.make +++ b/ydb/library/planner/share/ya.make @@ -20,6 +20,5 @@ SRCS( END() RECURSE( - protos ut ) diff --git a/ydb/library/shop/ya.make b/ydb/library/shop/ya.make index 81b5922b16a..9616691994f 100644 --- a/ydb/library/shop/ya.make +++ b/ydb/library/shop/ya.make @@ -16,7 +16,6 @@ PEERDIR( END() RECURSE( - protos sim_flowctl sim_shop ut diff --git a/yql/essentials/core/common_opt/yql_co_finalizers.cpp b/yql/essentials/core/common_opt/yql_co_finalizers.cpp index 0e616abd9c1..933b0bb9310 100644 --- a/yql/essentials/core/common_opt/yql_co_finalizers.cpp +++ b/yql/essentials/core/common_opt/yql_co_finalizers.cpp @@ -112,8 +112,8 @@ void FilterPushdownWithMultiusage(const TExprNode::TPtr& node, TNodeOnNodeOwnedM if (auto cond = parentFlatMap.Lambda().Body().Maybe<TCoConditionalValueBase>()) { const TCoArgument lambdaArg = parentFlatMap.Lambda().Args().Arg(0); auto pred = cond.Cast().Predicate(); - if (pred.Maybe<TCoLikely>() || - (pred.Maybe<TCoAnd>() && AnyOf(pred.Ref().ChildrenList(), [](const auto& p) { return p->IsCallable("Likely"); })) || + if (pred.Maybe<TCoNoPushBase>() || + (pred.Maybe<TCoAnd>() && AnyOf(pred.Ref().ChildrenList(), [](const auto& p) { return IsNoPush(*p); })) || !IsStrict(pred.Ptr()) || HasDependsOn(pred.Ptr(), lambdaArg.Ptr()) || IsDepended(parentFlatMap.Lambda().Ref(), *node)) diff --git a/yql/essentials/core/common_opt/yql_co_flow2.cpp b/yql/essentials/core/common_opt/yql_co_flow2.cpp index 1bd95745579..7099574b572 100644 --- a/yql/essentials/core/common_opt/yql_co_flow2.cpp +++ b/yql/essentials/core/common_opt/yql_co_flow2.cpp @@ -1684,7 +1684,7 @@ TExprBase FilterOverAggregate(const TCoFlatMapBase& node, TExprContext& ctx, TOp size_t separableComponents = 0; for (auto& p : andComponents) { TSet<TStringBuf> usedFields; - if (p->IsCallable("Likely") || + if (IsNoPush(*p) || HasDependsOn(p, arg.Ptr()) || !HaveFieldsSubset(p, arg.Ref(), usedFields, *optCtx.ParentsMap) || !AllOf(usedFields, [&](TStringBuf field) { return keyColumns.contains(field); }) || @@ -1701,7 +1701,7 @@ TExprBase FilterOverAggregate(const TCoFlatMapBase& node, TExprContext& ctx, TOp size_t maxKeyPredicates = 0; if (AllowComplexFiltersOverAggregatePushdown(optCtx)) { for (auto& p : restComponents) { - if (p->IsCallable("Likely")) { + if (IsNoPush(*p)) { continue; } const TNodeMap<ESubgraphType> marked = MarkSubgraphForAggregate(p, arg, keyColumns); @@ -1740,7 +1740,7 @@ TExprBase FilterOverAggregate(const TCoFlatMapBase& node, TExprContext& ctx, TOp calculator->DropCache(); } nonSeparableComponents += canPush; - p = ctx.WrapByCallableIf(canPush, "Likely", std::move(p)); + p = ctx.WrapByCallableIf(canPush, "NoPush", std::move(p)); } } diff --git a/yql/essentials/core/common_opt/yql_co_simple1.cpp b/yql/essentials/core/common_opt/yql_co_simple1.cpp index f109c4fb289..99534916211 100644 --- a/yql/essentials/core/common_opt/yql_co_simple1.cpp +++ b/yql/essentials/core/common_opt/yql_co_simple1.cpp @@ -659,7 +659,7 @@ TExprNode::TPtr RemoveOptionalReduceOverData(const TExprNode::TPtr& node, TExprC } TExprNode::TPtr PropagateCoalesceWithConstIntoLogicalOps(const TExprNode::TPtr& node, TExprContext& ctx) { - if (node->Head().IsCallable({"Likely", "AssumeStrict", "AssumeNonStrict"})) { + if (node->Head().IsCallable({"NoPush", "Likely", "AssumeStrict", "AssumeNonStrict"})) { const auto value = FromString<bool>(node->Child(1)->Head().Content()); if (!value) { YQL_CLOG(DEBUG, Core) << "PropagateCoalesceWithConst over " << node->Head().Content() << " (false)"; @@ -6950,8 +6950,8 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) { return node; }; - map["Likely"] = [](const TExprNode::TPtr& node, TExprContext& /*ctx*/, TOptimizeContext& /*optCtx*/) { - if (node->Head().IsCallable("Likely")) { + map["Likely"] = map["NoPush"] = [](const TExprNode::TPtr& node, TExprContext& /*ctx*/, TOptimizeContext& /*optCtx*/) { + if (IsNoPush(node->Head())) { YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content(); return node->HeadPtr(); } diff --git a/yql/essentials/core/common_opt/yql_co_simple2.cpp b/yql/essentials/core/common_opt/yql_co_simple2.cpp index 2adcdb6f22e..e7628c076cb 100644 --- a/yql/essentials/core/common_opt/yql_co_simple2.cpp +++ b/yql/essentials/core/common_opt/yql_co_simple2.cpp @@ -350,18 +350,18 @@ void DropDups(TExprNode::TListType& children) { } } -void StripLikely(TExprNodeList& args, TNodeOnNodeOwnedMap& likelyArgs) { +void StripNoPush(TExprNodeList& args, TNodeOnNodeOwnedMap& noPushArgs) { for (auto& arg : args) { - if (arg->IsCallable("Likely")) { - likelyArgs[arg->Child(0)] = arg; + if (IsNoPush(*arg)) { + noPushArgs[arg->Child(0)] = arg; arg = arg->HeadPtr(); } } } -void UnstripLikely(TExprNodeList& args, const TNodeOnNodeOwnedMap& likelyArgs) { +void UnstripNoPush(TExprNodeList& args, const TNodeOnNodeOwnedMap& noPushArgs) { for (auto& arg : args) { - if (auto it = likelyArgs.find(arg.Get()); it != likelyArgs.end()) { + if (auto it = noPushArgs.find(arg.Get()); it != noPushArgs.end()) { arg = it->second; } } @@ -369,11 +369,11 @@ void UnstripLikely(TExprNodeList& args, const TNodeOnNodeOwnedMap& likelyArgs) { TExprNode::TPtr OptimizeDups(const TExprNode::TPtr& node, TExprContext& ctx) { auto children = node->ChildrenList(); - TNodeOnNodeOwnedMap likelyArgs; - StripLikely(children, likelyArgs); + TNodeOnNodeOwnedMap noPushArgs; + StripNoPush(children, noPushArgs); DropDups(children); if (children.size() < node->ChildrenSize()) { - UnstripLikely(children, likelyArgs); + UnstripNoPush(children, noPushArgs); YQL_CLOG(DEBUG, Core) << node->Content() << " with " << node->ChildrenSize() - children.size() << " dups"; return 1U == children.size() ? children.front() : ctx.ChangeChildren(*node, std::move(children)); } @@ -517,7 +517,7 @@ bool AllOrNoneOr(const TExprNodeList& children) { } TExprNodeList GetOrAndChildren(TExprNode::TPtr node, bool visitOr) { - if (node->IsCallable("Likely")) { + if (IsNoPush(*node)) { node = node->HeadPtr(); } if (visitOr && node->IsCallable("Or") || !visitOr && node->IsCallable("And")) { @@ -573,8 +573,8 @@ TVector<TVector<size_t>> SplitToNonIntersectingGroups(const TExprNodeList& child TExprNode::TPtr ApplyAndAbsorption(const TExprNode::TPtr& node, TExprContext& ctx) { YQL_ENSURE(node->IsCallable("And")); TExprNodeList children = node->ChildrenList(); - TNodeOnNodeOwnedMap likelyPreds; - StripLikely(children, likelyPreds); + TNodeOnNodeOwnedMap noPushPreds; + StripNoPush(children, noPushPreds); if (AllOrNoneOr(children)) { return node; } @@ -608,7 +608,7 @@ TExprNode::TPtr ApplyAndAbsorption(const TExprNode::TPtr& node, TExprContext& ct newChildren.push_back(children[i]); } } - UnstripLikely(newChildren, likelyPreds); + UnstripNoPush(newChildren, noPushPreds); bool addJust = node->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional && AllOf(newChildren, [](const auto& node) { YQL_ENSURE(node->GetTypeAnn()); @@ -661,7 +661,7 @@ TExprNode::TPtr ApplyOrAbsorption(const TExprNode::TPtr& node, TExprContext& ctx for (auto& idx : andIndexes) { TExprNodeList andChildren = children[idx]->ChildrenList(); bool haveCommonFactor = AnyOf(andChildren, [&](TExprNode::TPtr child) { - if (child->IsCallable("Likely")) { + if (IsNoPush(*child)) { child = child->HeadPtr(); } TExprNodeList orList = GetOrChildren(child); @@ -714,13 +714,13 @@ TExprNode::TPtr ApplyOrDistributive(const TExprNode::TPtr& node, TExprContext& c } TExprNodeList commonPreds = children[group.front()]->ChildrenList(); - TNodeOnNodeOwnedMap likelyPreds; - StripLikely(commonPreds, likelyPreds); + TNodeOnNodeOwnedMap noPushPreds; + StripNoPush(commonPreds, noPushPreds); Sort(commonPreds, ptrComparator); for (size_t i = 1; i < group.size() && !commonPreds.empty(); ++i) { TExprNodeList curr = children[group[i]]->ChildrenList(); - StripLikely(curr, likelyPreds); + StripNoPush(curr, noPushPreds); Sort(curr, ptrComparator); TExprNodeList intersected; @@ -735,7 +735,7 @@ TExprNode::TPtr ApplyOrDistributive(const TExprNode::TPtr& node, TExprContext& c for (const auto& c : commonPreds) { commonSet.insert(c.Get()); } - UnstripLikely(commonPreds, likelyPreds); + UnstripNoPush(commonPreds, noPushPreds); // stabilize common predicate order Sort(commonPreds, [](const auto& l, const auto& r) { return CompareNodes(*l, *r) < 0; }); @@ -743,7 +743,7 @@ TExprNode::TPtr ApplyOrDistributive(const TExprNode::TPtr& node, TExprContext& c for (auto& idx : group) { auto childAnd = children[idx]; TExprNodeList preds = childAnd->ChildrenList(); - EraseIf(preds, [&](const TExprNode::TPtr& p) { return commonSet.contains(p->IsCallable("Likely") ? p->Child(0) : p.Get()); }); + EraseIf(preds, [&](const TExprNode::TPtr& p) { return commonSet.contains(IsNoPush(*p) ? p->Child(0) : p.Get()); }); if (!preds.empty()) { newGroup.emplace_back(ctx.ChangeChildren(*childAnd, std::move(preds))); } @@ -772,13 +772,13 @@ TExprNode::TPtr OptimizeOr(const TExprNode::TPtr& node, TExprContext& ctx, TOpti return opt; } - TNodeOnNodeOwnedMap likelyPreds; + TNodeOnNodeOwnedMap noPushPreds; TExprNodeList children = node->ChildrenList(); - StripLikely(children, likelyPreds); - if (!likelyPreds.empty()) { + StripNoPush(children, noPushPreds); + if (!noPushPreds.empty()) { // Likely(A) OR B -> Likely(A OR B) - YQL_CLOG(DEBUG, Core) << "Or with Likely argument"; - return ctx.NewCallable(node->Pos(), "Likely", { ctx.ChangeChildren(*node, std::move(children)) }); + YQL_CLOG(DEBUG, Core) << "Or with NoPush argument"; + return ctx.NewCallable(node->Pos(), "NoPush", { ctx.ChangeChildren(*node, std::move(children)) }); } if (IsExtractCommonPredicatesFromLogicalOpsEnabled(optCtx)) { diff --git a/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp b/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp index fb9dbb9a934..1ebe47096e2 100644 --- a/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp +++ b/yql/essentials/core/common_opt/yql_flatmap_over_join.cpp @@ -1524,7 +1524,7 @@ TExprBase FlatMapOverEquiJoin( const bool skipNulls = NeedEmitSkipNullMembers(types); for (const auto& andTerm : andTerms) { - if (andTerm->IsCallable("Likely")) { + if (IsNoPush(*andTerm)) { continue; } diff --git a/yql/essentials/core/expr_nodes/yql_expr_nodes.json b/yql/essentials/core/expr_nodes/yql_expr_nodes.json index 740fae8474f..c62b447217e 100644 --- a/yql/essentials/core/expr_nodes/yql_expr_nodes.json +++ b/yql/essentials/core/expr_nodes/yql_expr_nodes.json @@ -2680,12 +2680,23 @@ "ListBase": "TCoReplicationTarget" }, { - "Name" : "TCoLikely", + "Name" : "TCoNoPushBase", "Base" : "TCallable", - "Match": {"Type": "Callable", "Name": "Likely"}, + "Match": {"Type": "CallableBase"}, + "Builder": {"Generate": "None"}, "Children": [ {"Index": 0, "Name": "Predicate", "Type": "TExprBase"} ] + }, + { + "Name" : "TCoNoPush", + "Base" : "TCoNoPushBase", + "Match": {"Type": "Callable", "Name": "NoPush"} + }, + { + "Name" : "TCoLikely", + "Base" : "TCoNoPushBase", + "Match": {"Type": "Callable", "Name": "Likely"} } ] } diff --git a/yql/essentials/core/file_storage/file_storage.cpp b/yql/essentials/core/file_storage/file_storage.cpp index 734226f7e59..de24524ac30 100644 --- a/yql/essentials/core/file_storage/file_storage.cpp +++ b/yql/essentials/core/file_storage/file_storage.cpp @@ -8,6 +8,8 @@ #include <yql/essentials/core/file_storage/http_download/http_download.h> #include <yql/essentials/core/file_storage/defs/provider.h> +#include <yql/essentials/public/issue/yql_issue.h> + #include <yql/essentials/utils/fetch/fetch.h> #include <yql/essentials/utils/log/log.h> #include <yql/essentials/utils/log/context.h> @@ -153,7 +155,7 @@ public: } catch (const std::exception& e) { const TString msg = TStringBuilder() << "FileStorage: Failed to download file by URL \"" << urlStr << "\", details: " << e.what(); YQL_LOG(ERROR) << msg; - YQL_LOG_CTX_THROW yexception() << msg; + YQL_LOG_CTX_THROW TErrorException(UNEXPECTED_ERROR) << msg; } } diff --git a/yql/essentials/core/file_storage/ya.make b/yql/essentials/core/file_storage/ya.make index 2fa051cf90d..0bc16b99d45 100644 --- a/yql/essentials/core/file_storage/ya.make +++ b/yql/essentials/core/file_storage/ya.make @@ -23,6 +23,7 @@ PEERDIR( yql/essentials/core/file_storage/defs yql/essentials/core/file_storage/download yql/essentials/core/file_storage/http_download + yql/essentials/public/issue yql/essentials/utils yql/essentials/utils/log yql/essentials/utils/fetch diff --git a/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp b/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp index 666f1c6183d..2759510b09a 100644 --- a/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp +++ b/yql/essentials/core/peephole_opt/yql_opt_peephole_physical.cpp @@ -32,6 +32,8 @@ namespace { using namespace NNodes; +constexpr size_t WideLimit = 101; + using TPeepHoleOptimizerPtr = TExprNode::TPtr (*const)(const TExprNode::TPtr&, TExprContext&); using TPeepHoleOptimizerMap = std::unordered_map<std::string_view, TPeepHoleOptimizerPtr>; @@ -2533,7 +2535,7 @@ TExprNode::TPtr ExpandFlatMap(const TExprNode::TPtr& node, TExprContext& ctx) { } if (const auto kind = node->Head().GetTypeAnn()->GetKind(); (kind == ETypeAnnotationKind::Flow || kind == ETypeAnnotationKind::List) && - body.IsCallable("AsList") && body.ChildrenSize() > 1U) { + body.IsCallable("AsList") && body.ChildrenSize() > 1U && body.ChildrenSize() <= WideLimit) { constexpr auto multimap = Ordered ? "OrderedMultiMap" : "MultiMap"; YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content() << " as " << multimap << " of size " << body.ChildrenSize(); return ctx.NewCallable(node->Pos(), multimap, {node->HeadPtr(), ctx.DeepCopyLambda(lambda, body.ChildrenList())}); @@ -3495,13 +3497,16 @@ TExprNode::TPtr OptimizeMap(const TExprNode::TPtr& node, TExprContext& ctx) { } TExprNode::TPtr MakeWideTableSource(const TExprNode& tableSource, TExprContext& ctx, TVector<TString>* narrowMapColumns = nullptr) { - // TODO check wide limit if (tableSource.GetTypeAnn()->GetKind() != ETypeAnnotationKind::List) { return nullptr; } - YQL_CLOG(DEBUG, CorePeepHole) << "Generate WideTableSource"; auto structType = tableSource.GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>(); + if (structType->GetSize() > WideLimit) { + return nullptr; + } + + YQL_CLOG(DEBUG, CorePeepHole) << "Generate WideTableSource"; TVector<TString> columns; for (const auto& item : structType->GetItems()) { columns.push_back(TString(item->GetName())); @@ -5660,6 +5665,10 @@ TExprNode::TPtr OptimizeWideCombiner(const TExprNode::TPtr& node, TExprContext& if (needKeyFlatten.front()) { const auto flattenSize = *needKeyFlatten.front(); + if (flattenSize > WideLimit) { + return node; + } + YQL_CLOG(DEBUG, CorePeepHole) << "Flatten key by tuple for " << node->Content() << " from " << originalKeySize << " to " << flattenSize; auto children = node->ChildrenList(); @@ -5673,6 +5682,10 @@ TExprNode::TPtr OptimizeWideCombiner(const TExprNode::TPtr& node, TExprContext& if (needKeyFlatten.back()) { const auto flattenSize = *needKeyFlatten.back(); + if (flattenSize > WideLimit) { + return node; + } + YQL_CLOG(DEBUG, CorePeepHole) << "Flatten key by struct for " << node->Content() << " from " << originalKeySize << " to " << flattenSize; auto children = node->ChildrenList(); @@ -5694,6 +5707,10 @@ TExprNode::TPtr OptimizeWideCombiner(const TExprNode::TPtr& node, TExprContext& if (needStateFlatten.front()) { const auto flattenSize = *needStateFlatten.front(); + if (flattenSize > WideLimit) { + return node; + } + YQL_CLOG(DEBUG, CorePeepHole) << "Flatten state by tuple for " << node->Content() << " from " << originalStateSize << " to " << flattenSize; auto children = node->ChildrenList(); @@ -5707,6 +5724,10 @@ TExprNode::TPtr OptimizeWideCombiner(const TExprNode::TPtr& node, TExprContext& if (needStateFlatten.back()) { const auto flattenSize = *needStateFlatten.back(); + if (flattenSize > WideLimit) { + return node; + } + YQL_CLOG(DEBUG, CorePeepHole) << "Flatten state by struct for " << node->Content() << " from " << originalStateSize << " to " << flattenSize; auto children = node->ChildrenList(); @@ -6144,7 +6165,7 @@ bool CollectBlockRewrites(const TMultiExprType* multiInputType, bool keepInputCo TExprNode::TListType funcArgs; std::string_view arrowFunctionName; - const bool rewriteAsIs = node->IsCallable({"AssumeStrict", "AssumeNonStrict", "Likely"}); + const bool rewriteAsIs = node->IsCallable({"AssumeStrict", "AssumeNonStrict", "NoPush", "Likely"}); if (node->IsList() || rewriteAsIs || node->IsCallable({"DecimalMul", "DecimalDiv", "DecimalMod", "And", "Or", "Xor", "Not", "Coalesce", "Exists", "If", "Just", "AsStruct", "Member", "Nth", "ToPg", "FromPg", "PgResolvedCall", "PgResolvedOp"})) { diff --git a/yql/essentials/core/type_ann/type_ann_core.cpp b/yql/essentials/core/type_ann/type_ann_core.cpp index b6a5cbfbb11..d8c9267479c 100644 --- a/yql/essentials/core/type_ann/type_ann_core.cpp +++ b/yql/essentials/core/type_ann/type_ann_core.cpp @@ -3733,7 +3733,7 @@ namespace NTypeAnnImpl { return IGraphTransformer::TStatus::Ok; } - IGraphTransformer::TStatus LikelyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { + IGraphTransformer::TStatus NoPushWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) { if (!EnsureArgsCount(*input, 1, ctx.Expr)) { return IGraphTransformer::TStatus::Error; } @@ -12604,7 +12604,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot> Functions["Or"] = &LogicalWrapper<false>; Functions["Xor"] = &LogicalWrapper<true>; Functions["Not"] = &BoolOpt1Wrapper; - Functions["Likely"] = &LikelyWrapper; + Functions["NoPush"] = &NoPushWrapper; + Functions["Likely"] = &NoPushWrapper; Functions["Map"] = &MapWrapper; Functions["OrderedMap"] = &MapWrapper; Functions["MapNext"] = &MapNextWrapper; diff --git a/yql/essentials/core/yql_opt_utils.cpp b/yql/essentials/core/yql_opt_utils.cpp index 9a51b068010..0b956ea5de6 100644 --- a/yql/essentials/core/yql_opt_utils.cpp +++ b/yql/essentials/core/yql_opt_utils.cpp @@ -355,6 +355,10 @@ bool IsTablePropsDependent(const TExprNode& node) { return found; } +bool IsNoPush(const TExprNode& node) { + return node.IsCallable({"NoPush", "Likely"}); +} + TExprNode::TPtr KeepColumnOrder(const TExprNode::TPtr& node, const TExprNode& src, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) { auto columnOrder = typeCtx.LookupColumnOrder(src); if (!columnOrder) { diff --git a/yql/essentials/core/yql_opt_utils.h b/yql/essentials/core/yql_opt_utils.h index fd818dbd353..447d53737d1 100644 --- a/yql/essentials/core/yql_opt_utils.h +++ b/yql/essentials/core/yql_opt_utils.h @@ -26,6 +26,7 @@ bool IsRenameOrApplyFlatMapWithMapping(const NNodes::TCoFlatMapBase& node, TExpr bool IsPassthroughFlatMap(const NNodes::TCoFlatMapBase& flatmap, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember = false); bool IsPassthroughLambda(const NNodes::TCoLambda& lambda, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember = false); bool IsTablePropsDependent(const TExprNode& node); +bool IsNoPush(const TExprNode& node); bool HasOnlyOneJoinType(const TExprNode& joinTree, TStringBuf joinType); diff --git a/yql/essentials/core/yql_type_annotation.h b/yql/essentials/core/yql_type_annotation.h index d07d5afea84..ce70891e2e0 100644 --- a/yql/essentials/core/yql_type_annotation.h +++ b/yql/essentials/core/yql_type_annotation.h @@ -384,7 +384,7 @@ inline TString GetRandomKey<TGUID>() { struct TTypeAnnotationContext: public TThrRefBase { TSimpleSharedPtr<NDq::TOrderingsStateMachine> OrderingsFSM; - TLangVersion LangVer = UnknownLangVersion; + TLangVersion LangVer = MinLangVersion; THashMap<TString, TIntrusivePtr<TOptimizerStatistics::TColumnStatMap>> ColumnStatisticsByTableName; THashMap<ui64, std::shared_ptr<TOptimizerStatistics>> StatisticsMap; TIntrusivePtr<ITimeProvider> TimeProvider; diff --git a/yql/essentials/core/yql_udf_index.cpp b/yql/essentials/core/yql_udf_index.cpp index f861e9eca65..c0b62596b07 100644 --- a/yql/essentials/core/yql_udf_index.cpp +++ b/yql/essentials/core/yql_udf_index.cpp @@ -53,6 +53,9 @@ TVector<TResourceInfo::TPtr> ConvertResolveResultToResources(const TResolveResul newFunction.Messages.push_back(m); } + newFunction.MinLangVer = udf.GetMinLangVer(); + newFunction.MaxLangVer = udf.GetMaxLangVer(); + functionIndex[package].push_back(newFunction); } diff --git a/yql/essentials/core/yql_udf_index.h b/yql/essentials/core/yql_udf_index.h index 4f2f33404ca..315c21e4be6 100644 --- a/yql/essentials/core/yql_udf_index.h +++ b/yql/essentials/core/yql_udf_index.h @@ -20,6 +20,8 @@ struct TFunctionInfo { bool IsStrict = false; bool SupportsBlocks = false; TVector<TString> Messages; + TLangVersion MinLangVer = UnknownLangVersion; + TLangVersion MaxLangVer = UnknownLangVersion; }; // todo: specify whether path is frozen diff --git a/yql/essentials/docs/en/changelog/2025.03.md b/yql/essentials/docs/en/changelog/2025.03.md new file mode 100644 index 00000000000..e85da1f9c28 --- /dev/null +++ b/yql/essentials/docs/en/changelog/2025.03.md @@ -0,0 +1,3 @@ +## Changes in RE2 module {#re2-module} + +* Queries are guaranteed to fail when invalid regular expressions are passed diff --git a/yql/essentials/docs/en/changelog/toc_i.yaml b/yql/essentials/docs/en/changelog/toc_i.yaml index 968c0814ab5..37879aedee0 100644 --- a/yql/essentials/docs/en/changelog/toc_i.yaml +++ b/yql/essentials/docs/en/changelog/toc_i.yaml @@ -5,3 +5,5 @@ items: href: 2025.01.md - name: 2025.02 href: 2025.02.md +- name: 2025.03 + href: 2025.03.md diff --git a/yql/essentials/docs/en/udf/list/re2.md b/yql/essentials/docs/en/udf/list/re2.md index ab527b012a0..f83dd976c13 100644 --- a/yql/essentials/docs/en/udf/list/re2.md +++ b/yql/essentials/docs/en/udf/list/re2.md @@ -16,6 +16,13 @@ The Re2 module supports regular expressions based on [google::RE2](https://githu By default, the UTF-8 mode is enabled automatically if the regular expression is a valid UTF-8-encoded string, but is not a valid ASCII string. You can manually control the settings of the re2 library, if you pass the result of the `Re2::Options` function as the second argument to other module functions, next to the regular expression. +{% note info "Note" %} + +All regular expressions passed to functions must be valid. Otherwise, your query may fail. +Starting from version [2025.03](../../changelog/2025.03.md#re2-module), such a query will definitely fail. + +{% endnote %} + {% note warning %} Make sure to double all the backslashes in your regular expressions (if they are within a quoted string): standard string literals are treated as C-escaped strings in SQL. You can also format regular expressions as raw strings `@@regexp@@`: double slashes are not needed in this case. @@ -88,7 +95,7 @@ Notes on Re2::Options from the official [repository](https://github.com/google/r | CaseSensitive:Bool? | true | match is case-sensitive (regexp can override with (?i) unless in posix_syntax mode) | | DotNl:Bool? | false | let `.` match `\n` (default ) | | Literal:Bool? | false | interpret string as literal, not regexp | -| LogErrors:Bool? | true | log syntax and execution errors to ERROR | +| LogErrors:Bool? | true | this option is ignored | | LongestMatch:Bool? | false | search for longest match, not first match | | MaxMem:Uint64? | - | (see below) approx. max memory footprint of RE2 | | NeverCapture:Bool? | false | parse all parents as non-capturing | @@ -119,4 +126,3 @@ SELECT ``` In both cases, the word FOO will be found. Using the raw string @@regexp@@ lets you avoid double slashes. - diff --git a/yql/essentials/docs/ru/changelog/2025.03.md b/yql/essentials/docs/ru/changelog/2025.03.md new file mode 100644 index 00000000000..0a25c97b15b --- /dev/null +++ b/yql/essentials/docs/ru/changelog/2025.03.md @@ -0,0 +1,3 @@ +## Изменения в модуле RE2 {#re2-module} + +* Гарантируется падение запросов при передаче невалидных регулярных выражений diff --git a/yql/essentials/docs/ru/changelog/toc_i.yaml b/yql/essentials/docs/ru/changelog/toc_i.yaml index 9a1dac426db..bd294b79bbc 100644 --- a/yql/essentials/docs/ru/changelog/toc_i.yaml +++ b/yql/essentials/docs/ru/changelog/toc_i.yaml @@ -5,3 +5,5 @@ items: href: 2025.01.md - name: 2025.02 href: 2025.02.md +- name: 2025.03 + href: 2025.03.md diff --git a/yql/essentials/docs/ru/udf/list/math.md b/yql/essentials/docs/ru/udf/list/math.md index 467a8d1704c..6c5c811a3b0 100644 --- a/yql/essentials/docs/ru/udf/list/math.md +++ b/yql/essentials/docs/ru/udf/list/math.md @@ -148,15 +148,15 @@ SELECT Math::Rem(-1, 7); -- -1 #### Список функций -* `Math::RoundDownward() -> Tagged<Uint32, MathRoundingMode>` -- rounding towards negative infinity -* `Math::RoundToNearest() -> Tagged<Uint32, MathRoundingMode>` -- rounding towards nearest representable value -* `Math::RoundTowardZero() -> Tagged<Uint32, MathRoundingMode>` -- rounding towards zero -* `Math::RoundUpward() -> Tagged<Uint32, MathRoundingMode>` -- rounding towards positive infinity -* `Math::NearbyInt(AutoMap<Double>, Tagged<Uint32, MathRoundingMode>) -> Optional<Int64>` +* `Math::RoundDownward() -> Tagged<Uint32, MathRoundingMode>` — округление в сторону отрицательной бесконечности +* `Math::RoundToNearest() -> Tagged<Uint32, MathRoundingMode>` — округление в сторону ближайшего представимого значения +* `Math::RoundTowardZero() -> Tagged<Uint32, MathRoundingMode>` — округление в сторону нуля +* `Math::RoundUpward() -> Tagged<Uint32, MathRoundingMode>` — округление в сторону положительной бесконечности +* `Math::NearbyInt(AutoMap<Double>, Tagged<Uint32, MathRoundingMode>) -> Optional<Int64>` — округление до ближайшего целого значения -Функция `Math::NearbyInt` округляет первый аргумент до целого числа в соответсвии с режимом, заданным вторым аргументом. +Функция `Math::NearbyInt` округляет первый аргумент до целого числа в соответствии с режимом, заданным вторым аргументом. -Если результат выходит за пределы 64-битного целого числа, возращается NULL. +Если результат выходит за пределы 64-битного целого числа, возвращается NULL. #### Примеры diff --git a/yql/essentials/docs/ru/udf/list/re2.md b/yql/essentials/docs/ru/udf/list/re2.md index bece3053081..3c0906f30e1 100644 --- a/yql/essentials/docs/ru/udf/list/re2.md +++ b/yql/essentials/docs/ru/udf/list/re2.md @@ -16,6 +16,13 @@ Re2::Options([CaseSensitive:Bool?,DotNl:Bool?,Literal:Bool?,LogErrors:Bool?,Long По умолчанию UTF-8 режим включается автоматически, если регулярное выражение является валидной строкой в кодировке UTF-8, но не является валидной ASCII-строкой. Вручную настройками библиотеки re2 можно управлять с помощью передачи результата функции `Re2::Options` вторым аргументом другим функциям модуля, рядом с регулярным выражением. +{% note info "Примечание" %} + +Все регулярные выражения, переданные в функции, должны быть валидными. Иначе ваш запрос может упасть. +Начиная с версии [2025.03](../../changelog/2025.03.md#re2-module) такой запрос гарантированно завершится с ошибкой. + +{% endnote %} + {% note warning %} Все обратные слеши в регулярных выражениях (если они записаны в строке с кавычками) нужно удваивать, так как стандартные строковые литералы в SQL рассматриваются как С-escaped строки. Также можно записывать регулярное выражение в форме raw строки `@@regexp@@` — в этом случае удвоение слешей не требуется. @@ -88,7 +95,7 @@ SELECT | CaseSensitive:Bool? | true | match is case-sensitive (regexp can override with (?i) unless in posix_syntax mode) | | DotNl:Bool? | false | let `.` match `\n` (default ) | | Literal:Bool? | false | interpret string as literal, not regexp | -| LogErrors:Bool? | true | log syntax and execution errors to ERROR | +| LogErrors:Bool? | true | this option is ignored | | LongestMatch:Bool? | false | search for longest match, not first match | | MaxMem:Uint64? | - | (see below) approx. max memory footprint of RE2 | | NeverCapture:Bool? | false | parse all parens as non-capturing | diff --git a/yql/essentials/minikql/comp_nodes/mkql_udf.cpp b/yql/essentials/minikql/comp_nodes/mkql_udf.cpp index 0036091ad77..8230a759d6b 100644 --- a/yql/essentials/minikql/comp_nodes/mkql_udf.cpp +++ b/yql/essentials/minikql/comp_nodes/mkql_udf.cpp @@ -37,12 +37,14 @@ public: TString&& typeConfig, NUdf::TSourcePosition pos, const TCallableType* callableType, + const TCallableType* functionType, TType* userType) : TBaseComputation(mutables, EValueRepresentation::Boxed) , FunctionName(std::move(functionName)) , TypeConfig(std::move(typeConfig)) , Pos(pos) , CallableType(callableType) + , FunctionType(functionType) , UserType(userType) { this->Stateless = false; @@ -65,16 +67,55 @@ public: } NUdf::TUnboxedValue udf(NUdf::TUnboxedValuePod(funcInfo.Implementation.Release())); - TValidate<TValidatePolicy,TValidateMode>::WrapCallable(CallableType, udf, TStringBuilder() << "FunctionWithConfig<" << FunctionName << ">"); + TValidate<TValidatePolicy,TValidateMode>::WrapCallable(FunctionType, udf, TStringBuilder() << "FunctionWithConfig<" << FunctionName << ">"); + ExtendArgs(udf, CallableType, funcInfo.FunctionType); return udf.Release(); } private: + // xXX: This class implements the wrapper to properly handle + // the case when the signature of the emitted callable (i.e. + // callable type) requires less arguments than the actual + // function (i.e. function type). It wraps the unboxed value + // with the resolved UDF to introduce the bridge in the + // Run chain, preparing the valid argument vector for the + // chosen UDF implementation. + class TExtendedArgsWrapper: public NUdf::TBoxedValue { + public: + TExtendedArgsWrapper(NUdf::TUnboxedValue&& callable, size_t usedArgs, size_t requiredArgs) + : Callable_(callable) + , UsedArgs_(usedArgs) + , RequiredArgs_(requiredArgs) + {}; + + private: + NUdf::TUnboxedValue Run(const NUdf::IValueBuilder* valueBuilder, const NUdf::TUnboxedValuePod* args) const final { + NStackArray::TStackArray<NUdf::TUnboxedValue> values(ALLOC_ON_STACK(NUdf::TUnboxedValue, RequiredArgs_)); + for (size_t i = 0; i < UsedArgs_; i++) { + values[i] = args[i]; + } + return Callable_.Run(valueBuilder, values.data()); + } + + const NUdf::TUnboxedValue Callable_; + const size_t UsedArgs_; + const size_t RequiredArgs_; + }; + + void ExtendArgs(NUdf::TUnboxedValue& callable, const TCallableType* callableType, const TCallableType* functionType) const { + const auto callableArgc = callableType->GetArgumentsCount(); + const auto functionArgc = functionType->GetArgumentsCount(); + if (callableArgc < functionArgc) { + callable = NUdf::TUnboxedValuePod(new TExtendedArgsWrapper(std::move(callable), callableArgc, functionArgc)); + } + } + void RegisterDependencies() const final {} const TString FunctionName; const TString TypeConfig; const NUdf::TSourcePosition Pos; const TCallableType *const CallableType; + const TCallableType *const FunctionType; TType *const UserType; }; @@ -90,12 +131,13 @@ public: TString&& typeConfig, NUdf::TSourcePosition pos, const TCallableType* callableType, + const TCallableType* functionType, TType* userType, TString&& moduleIRUniqID, TString&& moduleIR, TString&& fuctioNameIR, NUdf::TUniquePtr<NUdf::IBoxedValue>&& impl) - : TSimpleUdfWrapper(mutables, std::move(functionName), std::move(typeConfig), pos, callableType, userType) + : TSimpleUdfWrapper(mutables, std::move(functionName), std::move(typeConfig), pos, callableType, functionType, userType) , ModuleIRUniqID(std::move(moduleIRUniqID)) , ModuleIR(std::move(moduleIR)) , IRFunctionName(std::move(fuctioNameIR)) @@ -138,7 +180,7 @@ public: NUdf::TSourcePosition pos, IComputationNode* runConfigNode, ui32 runConfigArgs, - const TCallableType* callableType, + const TCallableType* functionType, TType* userType) : TBaseComputation(mutables, EValueRepresentation::Boxed) , FunctionName(std::move(functionName)) @@ -146,7 +188,7 @@ public: , Pos(pos) , RunConfigNode(runConfigNode) , RunConfigArgs(runConfigArgs) - , CallableType(callableType) + , FunctionType(functionType) , UserType(userType) , UdfIndex(mutables.CurValueIndex++) { @@ -238,7 +280,7 @@ private: } void Wrap(NUdf::TUnboxedValue& callable) const { - TValidate<TValidatePolicy,TValidateMode>::WrapCallable(CallableType, callable, TStringBuilder() << "FunctionWithConfig<" << FunctionName << ">"); + TValidate<TValidatePolicy,TValidateMode>::WrapCallable(FunctionType, callable, TStringBuilder() << "FunctionWithConfig<" << FunctionName << ">"); } void RegisterDependencies() const final { @@ -250,7 +292,7 @@ private: const NUdf::TSourcePosition Pos; IComputationNode* const RunConfigNode; const ui32 RunConfigArgs; - const TCallableType* CallableType; + const TCallableType* FunctionType; TType* const UserType; const ui32 UdfIndex; }; @@ -317,6 +359,8 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont << status.GetError()).c_str()); } + const auto callableFuncType = AS_TYPE(TCallableType, funcInfo.FunctionType); + const auto callableNodeType = AS_TYPE(TCallableType, callable.GetType()->GetReturnType()); const auto runConfigFuncType = funcInfo.RunConfigType; const auto runConfigNodeType = runCfgNode.GetStaticType(); @@ -324,7 +368,8 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont // It's only legal, when the compiled UDF declares its // signature using run config at compilation phase, but then // omits it in favor to function currying at execution phase. - if (!runConfigFuncType->IsVoid()) { + // And vice versa for the forward compatibility. + if (!runConfigNodeType->IsVoid() && !runConfigFuncType->IsVoid()) { TString diff = TStringBuilder() << "run config type mismatch, expected: " << PrintNode((runConfigNodeType), true) @@ -337,17 +382,31 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont << TruncateTypeDiff(diff)).c_str()); } + const auto callableType = runConfigNodeType->IsVoid() + ? callableNodeType : callableFuncType; + const auto runConfigType = runConfigNodeType->IsVoid() + ? runConfigFuncType : runConfigNodeType; + // If so, check the following invariants: // * The first argument of the head function in the sequence // of the curried functions has to be the same as the // run config type. + // * All other arguments of the head function in the sequence + // of the curried function have to be optional. // * The type of the resulting callable has to be the same // as the function type. - const auto firstArgType = funcInfo.FunctionType->GetArgumentType(0); - if (!runConfigNodeType->IsSameType(*firstArgType)) { + if (callableType->GetArgumentsCount() - callableType->GetOptionalArgumentsCount() != 1U) { + UdfTerminate((TStringBuilder() << pos + << " Udf Function '" + << funcName + << "' wrapper has more than one required argument: " + << PrintNode(callableType)).c_str()); + } + const auto firstArgType = callableType->GetArgumentType(0); + if (!runConfigType->IsSameType(*firstArgType)) { TString diff = TStringBuilder() << "type mismatch, expected run config type: " - << PrintNode(runConfigNodeType, true) + << PrintNode(runConfigType, true) << ", actual: " << PrintNode(firstArgType, true); UdfTerminate((TStringBuilder() << pos @@ -356,14 +415,18 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont << "' " << TruncateTypeDiff(diff)).c_str()); } - const auto callableFuncType = funcInfo.FunctionType->GetReturnType(); - const auto callableNodeType = callable.GetType()->GetReturnType(); - if (!callableNodeType->IsSameType(*callableFuncType)) { + const auto closureFuncType = runConfigNodeType->IsVoid() + ? callableFuncType + : AS_TYPE(TCallableType, callableFuncType)->GetReturnType(); + const auto closureNodeType = runConfigNodeType->IsVoid() + ? AS_TYPE(TCallableType, callableNodeType)->GetReturnType() + : callableNodeType; + if (!closureNodeType->IsSameType(*closureFuncType)) { TString diff = TStringBuilder() << "type mismatch, expected return type: " - << PrintNode(callableNodeType, true) + << PrintNode(closureNodeType, true) << ", actual: " - << PrintNode(callableFuncType, true); + << PrintNode(closureFuncType, true); UdfTerminate((TStringBuilder() << pos << " Udf Function '" << funcName @@ -373,12 +436,14 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont const auto runConfigCompNode = LocateNode(ctx.NodeLocator, *runCfgNode.GetNode()); const auto runConfigArgs = funcInfo.FunctionType->GetArgumentsCount(); - return CreateUdfWrapper<false>(ctx, std::move(funcName), std::move(typeConfig), pos, runConfigCompNode, runConfigArgs, funcInfo.FunctionType, userType); + return runConfigNodeType->IsVoid() + ? CreateUdfWrapper<true>(ctx, std::move(funcName), std::move(typeConfig), pos, callableNodeType, callableFuncType, userType) + : CreateUdfWrapper<false>(ctx, std::move(funcName), std::move(typeConfig), pos, runConfigCompNode, runConfigArgs, callableFuncType, userType); } - if (!funcInfo.FunctionType->IsConvertableTo(*callable.GetType()->GetReturnType(), true)) { - TString diff = TStringBuilder() << "type mismatch, expected return type: " << PrintNode(callable.GetType()->GetReturnType(), true) << - ", actual:" << PrintNode(funcInfo.FunctionType, true); + if (!callableFuncType->IsConvertableTo(*callableNodeType, true)) { + TString diff = TStringBuilder() << "type mismatch, expected return type: " << PrintNode(callableNodeType, true) << + ", actual:" << PrintNode(callableFuncType, true); UdfTerminate((TStringBuilder() << pos << " UDF Function '" << funcName << "' " << TruncateTypeDiff(diff)).c_str()); } @@ -389,15 +454,15 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont if (runConfigFuncType->IsVoid()) { if (ctx.ValidateMode == NUdf::EValidateMode::None && funcInfo.ModuleIR && funcInfo.IRFunctionName) { return new TUdfRunCodegeneratorNode( - ctx.Mutables, std::move(funcName), std::move(typeConfig), pos, funcInfo.FunctionType, userType, + ctx.Mutables, std::move(funcName), std::move(typeConfig), pos, callableNodeType, callableFuncType, userType, std::move(funcInfo.ModuleIRUniqID), std::move(funcInfo.ModuleIR), std::move(funcInfo.IRFunctionName), std::move(funcInfo.Implementation) ); } - return CreateUdfWrapper<true>(ctx, std::move(funcName), std::move(typeConfig), pos, funcInfo.FunctionType, userType); + return CreateUdfWrapper<true>(ctx, std::move(funcName), std::move(typeConfig), pos, callableNodeType, callableFuncType, userType); } const auto runCfgCompNode = LocateNode(ctx.NodeLocator, *runCfgNode.GetNode()); - return CreateUdfWrapper<false>(ctx, std::move(funcName), std::move(typeConfig), pos, runCfgCompNode, 1U, funcInfo.FunctionType, userType); + return CreateUdfWrapper<false>(ctx, std::move(funcName), std::move(typeConfig), pos, runCfgCompNode, 1U, callableFuncType, userType); } IComputationNode* WrapScriptUdf(TCallable& callable, const TComputationNodeFactoryContext& ctx) { diff --git a/yql/essentials/minikql/comp_nodes/ut/mkql_udf_ut.cpp b/yql/essentials/minikql/comp_nodes/ut/mkql_udf_ut.cpp index f5b8b7892c1..265ad3bd059 100644 --- a/yql/essentials/minikql/comp_nodes/ut/mkql_udf_ut.cpp +++ b/yql/essentials/minikql/comp_nodes/ut/mkql_udf_ut.cpp @@ -5,6 +5,31 @@ namespace NKikimr { namespace NMiniKQL { +class TImpl : public NYql::NUdf::TBoxedValue { +public: + explicit TImpl(NYql::NUdf::TSourcePosition pos, + const std::string_view upvalue) + : Pos_(pos) + , Upvalue_(upvalue) + {} + + NYql::NUdf::TUnboxedValue Run(const NYql::NUdf::IValueBuilder* valueBuilder, + const NYql::NUdf::TUnboxedValuePod* args) + const override try { + TStringStream concat; + concat << Upvalue_ << " " << args[0].AsStringRef(); + return valueBuilder->NewString(NYql::NUdf::TStringRef(concat.Data(), + concat.Size())); + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } + + +private: + const NYql::NUdf::TSourcePosition Pos_; + const TString Upvalue_; +}; + // Class, implementing the closure with run config. class TRunConfig : public NYql::NUdf::TBoxedValue { public: @@ -35,6 +60,15 @@ public: return true; } + NYql::NUdf::TUnboxedValue Run(const NYql::NUdf::IValueBuilder*, + const NYql::NUdf::TUnboxedValuePod* args) + const final try { + const std::string_view upvalue(args[0].AsStringRef()); + return NYql::NUdf::TUnboxedValuePod(new TImpl(Pos_, upvalue)); + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } + private: const NYql::NUdf::TSourcePosition Pos_; }; @@ -81,31 +115,6 @@ public: } private: - class TImpl : public NYql::NUdf::TBoxedValue { - public: - explicit TImpl(NYql::NUdf::TSourcePosition pos, - const std::string_view upvalue) - : Pos_(pos) - , Upvalue_(upvalue) - {} - - NYql::NUdf::TUnboxedValue Run(const NYql::NUdf::IValueBuilder* valueBuilder, - const NYql::NUdf::TUnboxedValuePod* args) - const override try { - TStringStream concat; - concat << Upvalue_ << " " << args[0].AsStringRef(); - return valueBuilder->NewString(NYql::NUdf::TStringRef(concat.Data(), - concat.Size())); - } catch (const std::exception& e) { - UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); - } - - - private: - const NYql::NUdf::TSourcePosition Pos_; - const TString Upvalue_; - }; - const NYql::NUdf::TSourcePosition Pos_; }; @@ -118,6 +127,79 @@ private: SIMPLE_MODULE(TRunConfigUTModule, TRunConfig) SIMPLE_MODULE(TCurryingUTModule, TCurrying) + +SIMPLE_STRICT_UDF(TTest, char*(char*, char*, char*)) { + TStringStream concat; + concat << args[0].AsStringRef() << " " + << args[1].AsStringRef() << " " + << args[2].AsStringRef(); + return valueBuilder->NewString(NYql::NUdf::TStringRef(concat.Data(), + concat.Size())); +} + +template<bool Old> +class TNewTest : public NYql::NUdf::TBoxedValue { +public: + explicit TNewTest(NYql::NUdf::TSourcePosition pos) + : Pos_(pos) + {} + + static const NYql::NUdf::TStringRef& Name() { + static auto name = NYql::NUdf::TStringRef::Of("Test"); + return name; + } + + static bool DeclareSignature(const NYql::NUdf::TStringRef& name, + NYql::NUdf::TType*, + NYql::NUdf::IFunctionTypeInfoBuilder& builder, + bool typesOnly) + { + if (Name() != name) { + return false; + } + + if (Old && typesOnly) { + builder.SimpleSignature<char*(char*, char*, char*)>(); + return true; + } + + builder.SimpleSignature<char*(char*, char*, char*, NYql::NUdf::TOptional<char*>)>() + .OptionalArgs(1); + if (!typesOnly) { + builder.Implementation(new TNewTest(builder.GetSourcePosition())); + } + + return true; + } + + NYql::NUdf::TUnboxedValue Run(const NYql::NUdf::IValueBuilder* valueBuilder, + const NYql::NUdf::TUnboxedValuePod* args) + const override try { + TStringStream concat; + concat << args[0].AsStringRef() << " " + << args[1].AsStringRef() << " "; + if (args[3]) { + concat << args[3].AsStringRef() << " "; + } + concat << args[2].AsStringRef(); + return valueBuilder->NewString(NYql::NUdf::TStringRef(concat.Data(), + concat.Size())); + } catch (const std::exception& e) { + UdfTerminate((TStringBuilder() << Pos_ << " " << e.what()).data()); + } + +private: + const NYql::NUdf::TSourcePosition Pos_; +}; + +// XXX: "Old" UDF is declared via SIMPLE_UDF helper, so it has to +// use the *actual* function name as a class name. Furthermore, +// the UDF, declared by SIMPLE_UDF has to provide the same +// semantics as TNewTest<true>. +SIMPLE_MODULE(TOldUTModule, TTest) +SIMPLE_MODULE(TIncrementalUTModule, TNewTest<true>) +SIMPLE_MODULE(TNewUTModule, TNewTest<false>) + Y_UNIT_TEST_SUITE(TMiniKQLUdfTest) { Y_UNIT_TEST_LLVM(RunconfigToCurrying) { // Create the test setup, using TRunConfig implementation @@ -166,6 +248,238 @@ Y_UNIT_TEST_SUITE(TMiniKQLUdfTest) { UNIT_ASSERT_STRINGS_EQUAL(TStringBuf(result.AsStringRef()), "Canary is alive"); UNIT_ASSERT(!iterator.Next(result)); } + + Y_UNIT_TEST_LLVM(CurryingToRunconfig) { + // Create the test setup, using TCurrying implementation + // for TestModule.Test UDF. + TVector<TUdfModuleInfo> compileModules; + compileModules.emplace_back( + TUdfModuleInfo{"", "TestModule", new TCurryingUTModule()} + ); + TSetup<LLVM> compileSetup(GetTestFactory(), std::move(compileModules)); + TProgramBuilder& pb = *compileSetup.PgmBuilder; + + // Build the graph on the setup with TRunConfig implementation. + const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto upvalue = pb.NewDataLiteral<NUdf::EDataSlot::String>("Canary"); + const auto optional = pb.NewOptional(pb.NewDataLiteral(true)); + const auto value = pb.NewDataLiteral<NUdf::EDataSlot::String>("is alive"); + const auto userType = pb.NewTupleType({ + pb.NewTupleType({strType}), + pb.NewEmptyStructType(), + pb.NewEmptyTupleType()}); + const auto udf = pb.Udf("TestModule.Test", pb.NewVoid(), userType); + const auto closure = pb.Apply(udf, {upvalue, optional}); + + const auto list = pb.NewList(strType, {value}); + const auto pgmReturn = pb.Map(list, [&pb, closure](const TRuntimeNode item) { + return pb.Apply(closure, {item}); + }); + + // Create the test setup, using TRunConfig implementation + // for TestModule.Test UDF. + TVector<TUdfModuleInfo> runModules; + runModules.emplace_back( + TUdfModuleInfo{"", "TestModule", new TRunConfigUTModule()} + ); + TSetup<LLVM> runSetup(GetTestFactory(), std::move(runModules)); + // Move the graph from the one setup to another as a + // serialized bytecode sequence. + const auto bytecode = SerializeRuntimeNode(pgmReturn, *compileSetup.Env); + const auto root = DeserializeRuntimeNode(bytecode, *runSetup.Env); + + // Run the graph on the setup with TCurrying implementation. + const auto graph = runSetup.BuildGraph(root); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue result; + UNIT_ASSERT(iterator.Next(result)); + UNIT_ASSERT_STRINGS_EQUAL(TStringBuf(result.AsStringRef()), "Canary is alive"); + UNIT_ASSERT(!iterator.Next(result)); + } + + Y_UNIT_TEST_LLVM(OldToIncremental) { + // Create the test setup, using the old implementation for + // TestModule.Test UDF. + TVector<TUdfModuleInfo> compileModules; + compileModules.emplace_back( + TUdfModuleInfo{"", "TestModule", new TOldUTModule()} + ); + TSetup<LLVM> compileSetup(GetTestFactory(), std::move(compileModules)); + TProgramBuilder& pb = *compileSetup.PgmBuilder; + + // Build the graph, using the old setup. + const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto arg1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Canary"); + const auto arg2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("is"); + const auto arg3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("alive"); + + const auto udf = pb.Udf("TestModule.Test"); + const auto argsType = pb.NewTupleType({strType, strType, strType}); + const auto argList = pb.NewList(argsType, {pb.NewTuple({arg1, arg2, arg3})}); + const auto pgmReturn = pb.Map(argList, [&pb, udf](const TRuntimeNode args) { + return pb.Apply(udf, {pb.Nth(args, 0), pb.Nth(args, 1), pb.Nth(args, 2)}); + }); + + // Create the test setup, using the incremental + // implementation for TestModule.Test UDF. + TVector<TUdfModuleInfo> runModules; + runModules.emplace_back( + TUdfModuleInfo{"", "TestModule", new TIncrementalUTModule()} + ); + TSetup<LLVM> runSetup(GetTestFactory(), std::move(runModules)); + // Move the graph from the one setup to another as a + // serialized bytecode sequence. + const auto bytecode = SerializeRuntimeNode(pgmReturn, *compileSetup.Env); + const auto root = DeserializeRuntimeNode(bytecode, *runSetup.Env); + + // Run the graph, using the incremental setup. + const auto graph = runSetup.BuildGraph(root); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue result; + UNIT_ASSERT(iterator.Next(result)); + UNIT_ASSERT_STRINGS_EQUAL(TStringBuf(result.AsStringRef()), "Canary is alive"); + UNIT_ASSERT(!iterator.Next(result)); + } + + Y_UNIT_TEST_LLVM(IncrementalToOld) { + // Create the test setup, using the incremental + // implementation for TestModule.Test UDF. + TVector<TUdfModuleInfo> compileModules; + compileModules.emplace_back( + TUdfModuleInfo{"", "TestModule", new TIncrementalUTModule()} + ); + TSetup<LLVM> compileSetup(GetTestFactory(), std::move(compileModules)); + TProgramBuilder& pb = *compileSetup.PgmBuilder; + + // Build the graph, using the incremental setup. + const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto arg1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Canary"); + const auto arg2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("is"); + const auto arg3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("alive"); + + const auto udf = pb.Udf("TestModule.Test"); + const auto argsType = pb.NewTupleType({strType, strType, strType}); + const auto argList = pb.NewList(argsType, {pb.NewTuple({arg1, arg2, arg3})}); + const auto pgmReturn = pb.Map(argList, [&pb, udf](const TRuntimeNode args) { + return pb.Apply(udf, {pb.Nth(args, 0), pb.Nth(args, 1), pb.Nth(args, 2)}); + }); + + // Create the test setup, using the old implementation for + // TestModule.Test UDF. + TVector<TUdfModuleInfo> runModules; + runModules.emplace_back( + TUdfModuleInfo{"", "TestModule", new TOldUTModule()} + ); + TSetup<LLVM> runSetup(GetTestFactory(), std::move(runModules)); + // Move the graph from the one setup to another as a + // serialized bytecode sequence. + const auto bytecode = SerializeRuntimeNode(pgmReturn, *compileSetup.Env); + const auto root = DeserializeRuntimeNode(bytecode, *runSetup.Env); + + // Run the graph, using the old setup. + const auto graph = runSetup.BuildGraph(root); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue result; + UNIT_ASSERT(iterator.Next(result)); + UNIT_ASSERT_STRINGS_EQUAL(TStringBuf(result.AsStringRef()), "Canary is alive"); + UNIT_ASSERT(!iterator.Next(result)); + } + + Y_UNIT_TEST_LLVM(IncrementalToNew) { + // Create the test setup, using the incremental + // implementation for TestModule.Test UDF. + TVector<TUdfModuleInfo> compileModules; + compileModules.emplace_back( + TUdfModuleInfo{"", "TestModule", new TIncrementalUTModule()} + ); + TSetup<LLVM> compileSetup(GetTestFactory(), std::move(compileModules)); + TProgramBuilder& pb = *compileSetup.PgmBuilder; + + // Build the graph, using the incremental setup. + const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto arg1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Canary"); + const auto arg2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("is"); + const auto arg3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("alive"); + + const auto udf = pb.Udf("TestModule.Test"); + const auto argsType = pb.NewTupleType({strType, strType, strType}); + const auto argList = pb.NewList(argsType, {pb.NewTuple({arg1, arg2, arg3})}); + const auto pgmReturn = pb.Map(argList, [&pb, udf](const TRuntimeNode args) { + return pb.Apply(udf, {pb.Nth(args, 0), pb.Nth(args, 1), pb.Nth(args, 2)}); + }); + + // Create the test setup, using the new implementation for + // TestModule.Test UDF. + TVector<TUdfModuleInfo> runModules; + runModules.emplace_back( + TUdfModuleInfo{"", "TestModule", new TNewUTModule()} + ); + TSetup<LLVM> runSetup(GetTestFactory(), std::move(runModules)); + // Move the graph from the one setup to another as a + // serialized bytecode sequence. + const auto bytecode = SerializeRuntimeNode(pgmReturn, *compileSetup.Env); + const auto root = DeserializeRuntimeNode(bytecode, *runSetup.Env); + + // Run the graph, using the new setup. + const auto graph = runSetup.BuildGraph(root); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue result; + UNIT_ASSERT(iterator.Next(result)); + UNIT_ASSERT_STRINGS_EQUAL(TStringBuf(result.AsStringRef()), "Canary is alive"); + UNIT_ASSERT(!iterator.Next(result)); + } + + Y_UNIT_TEST_LLVM(NewToIncremental) { + // Create the test setup, using the new implementation for + // TestModule.Test UDF. + TVector<TUdfModuleInfo> compileModules; + compileModules.emplace_back( + TUdfModuleInfo{"", "TestModule", new TNewUTModule()} + ); + TSetup<LLVM> compileSetup(GetTestFactory(), std::move(compileModules)); + TProgramBuilder& pb = *compileSetup.PgmBuilder; + + // Build the graph, using the new setup. + const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id); + const auto optType = pb.NewOptionalType(strType); + const auto arg1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Canary"); + const auto arg2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("is"); + const auto arg3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("alive"); + const auto arg4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("still"); + const auto opt4 = pb.NewOptional(arg4); + + const auto udf = pb.Udf("TestModule.Test"); + const auto argsType = pb.NewTupleType({strType, strType, strType, optType}); + const auto argList = pb.NewList(argsType, {pb.NewTuple({arg1, arg2, arg3, opt4})}); + const auto pgmReturn = pb.Map(argList, [&pb, udf](const TRuntimeNode args) { + return pb.Apply(udf, {pb.Nth(args, 0), pb.Nth(args, 1), pb.Nth(args, 2), pb.Nth(args, 3)}); + }); + + // Create the test setup, using the incremental + // implementation for TestModule.Test UDF. + TVector<TUdfModuleInfo> runModules; + runModules.emplace_back( + TUdfModuleInfo{"", "TestModule", new TIncrementalUTModule()} + ); + TSetup<LLVM> runSetup(GetTestFactory(), std::move(runModules)); + // Move the graph from the one setup to another as a + // serialized bytecode sequence. + const auto bytecode = SerializeRuntimeNode(pgmReturn, *compileSetup.Env); + const auto root = DeserializeRuntimeNode(bytecode, *runSetup.Env); + + // Run the graph, using the incremental setup. + const auto graph = runSetup.BuildGraph(root); + const auto iterator = graph->GetValue().GetListIterator(); + + NUdf::TUnboxedValue result; + UNIT_ASSERT(iterator.Next(result)); + UNIT_ASSERT_STRINGS_EQUAL(TStringBuf(result.AsStringRef()), "Canary is still alive"); + UNIT_ASSERT(!iterator.Next(result)); + } } // Y_UNIT_TEST_SUITE } // namespace NMiniKQL diff --git a/yql/essentials/minikql/computation/mkql_computation_node.cpp b/yql/essentials/minikql/computation/mkql_computation_node.cpp index 7fa810fb8cd..6625d3a6eef 100644 --- a/yql/essentials/minikql/computation/mkql_computation_node.cpp +++ b/yql/essentials/minikql/computation/mkql_computation_node.cpp @@ -60,6 +60,7 @@ TComputationContext::TComputationContext(const THolderFactory& holderFactory, , CountersProvider(opts.CountersProvider) , SecureParamsProvider(opts.SecureParamsProvider) , LogProvider(opts.LogProvider) + , LangVer(opts.LangVer) { std::fill_n(MutableValues.get(), mutables.CurValueIndex, NUdf::TUnboxedValue(NUdf::TUnboxedValuePod::Invalid())); diff --git a/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp b/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp index 848284e2b41..733bc5a3ad5 100644 --- a/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp +++ b/yql/essentials/providers/common/mkql/yql_provider_mkql.cpp @@ -3016,7 +3016,7 @@ TMkqlCommonCallableCompiler::TShared::TShared() { return MkqlBuildExpr(node.Head(), ctx); }); - AddCallable({ "AssumeStrict", "AssumeNonStrict", "Likely" }, [](const TExprNode& node, TMkqlBuildContext& ctx) { + AddCallable({ "AssumeStrict", "AssumeNonStrict", "NoPush", "Likely" }, [](const TExprNode& node, TMkqlBuildContext& ctx) { return MkqlBuildExpr(node.Head(), ctx); }); diff --git a/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp b/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp index 3892de463e7..45844281c4f 100644 --- a/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp +++ b/yql/essentials/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp @@ -205,6 +205,8 @@ private: function.IsStrict = info.IsStrict; function.SupportsBlocks = info.SupportsBlocks; function.Messages = info.Messages; + function.MinLangVer = info.MinLangVer; + function.MaxLangVer = info.MaxLangVer; return true; } diff --git a/yql/essentials/sql/settings/translation_settings.h b/yql/essentials/sql/settings/translation_settings.h index 053e09acbec..99f0fc149fc 100644 --- a/yql/essentials/sql/settings/translation_settings.h +++ b/yql/essentials/sql/settings/translation_settings.h @@ -77,7 +77,7 @@ namespace NSQLTranslation { TTranslationSettings(); google::protobuf::Arena* Arena = nullptr; - NYql::TLangVersion LangVer = NYql::UnknownLangVersion; + NYql::TLangVersion LangVer = NYql::MinLangVersion; THashMap<TString, TString> ClusterMapping; TString PathPrefix; // keys (cluster name) should be normalized diff --git a/yql/essentials/sql/v1/SQLv1.g.in b/yql/essentials/sql/v1/SQLv1.g.in index 6ba91358316..ed04c337507 100644 --- a/yql/essentials/sql/v1/SQLv1.g.in +++ b/yql/essentials/sql/v1/SQLv1.g.in @@ -371,7 +371,7 @@ select_unparenthesized_stmt: select_kind_partial (select_op select_kind_parenthe select_kind_parenthesis: select_kind_partial | LPAREN select_kind_partial RPAREN; -select_op: UNION (ALL)? | INTERSECT | EXCEPT; +select_op: (UNION | INTERSECT | EXCEPT) (DISTINCT | ALL)?; select_kind_partial: select_kind (LIMIT expr ((OFFSET | COMMA) expr)?)? diff --git a/yql/essentials/sql/v1/SQLv1Antlr4.g.in b/yql/essentials/sql/v1/SQLv1Antlr4.g.in index 66be65e2aee..6bfdf3820f7 100644 --- a/yql/essentials/sql/v1/SQLv1Antlr4.g.in +++ b/yql/essentials/sql/v1/SQLv1Antlr4.g.in @@ -370,7 +370,7 @@ select_unparenthesized_stmt: select_kind_partial (select_op select_kind_parenthe select_kind_parenthesis: select_kind_partial | LPAREN select_kind_partial RPAREN; -select_op: UNION (ALL)? | INTERSECT | EXCEPT; +select_op: (UNION | INTERSECT | EXCEPT) (DISTINCT | ALL)?; select_kind_partial: select_kind (LIMIT expr ((OFFSET | COMMA) expr)?)? diff --git a/yql/essentials/sql/v1/complete/analysis/global/evaluate.cpp b/yql/essentials/sql/v1/complete/analysis/global/evaluate.cpp new file mode 100644 index 00000000000..b938eaf0631 --- /dev/null +++ b/yql/essentials/sql/v1/complete/analysis/global/evaluate.cpp @@ -0,0 +1,52 @@ +#include "evaluate.h" + +namespace NSQLComplete { + + namespace { + + class TVisitor: public SQLv1Antlr4BaseVisitor { + public: + explicit TVisitor(const TEnvironment* env) + : Env_(env) + { + } + + std::any visitBind_parameter(SQLv1::Bind_parameterContext* ctx) override { + std::string id = GetId(ctx); + if (const NYT::TNode* node = Env_->Parameters.FindPtr(id)) { + return *node; + } + return defaultResult(); + } + + std::any defaultResult() override { + return NYT::TNode(); + } + + private: + std::string GetId(SQLv1::Bind_parameterContext* ctx) const { + if (auto* x = ctx->an_id_or_type()) { + return x->getText(); + } else if (auto* x = ctx->TOKEN_TRUE()) { + return x->getText(); + } else if (auto* x = ctx->TOKEN_FALSE()) { + return x->getText(); + } else { + Y_ABORT("You should change implementation according grammar changes"); + } + } + + const TEnvironment* Env_; + }; + + NYT::TNode EvaluateG(antlr4::ParserRuleContext* ctx, const TEnvironment& env) { + return std::any_cast<NYT::TNode>(TVisitor(&env).visit(ctx)); + } + + } // namespace + + NYT::TNode Evaluate(SQLv1::Bind_parameterContext* ctx, const TEnvironment& env) { + return EvaluateG(ctx, env); + } + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/analysis/global/evaluate.h b/yql/essentials/sql/v1/complete/analysis/global/evaluate.h new file mode 100644 index 00000000000..03cbcdc798b --- /dev/null +++ b/yql/essentials/sql/v1/complete/analysis/global/evaluate.h @@ -0,0 +1,11 @@ +#pragma once + +#include "parse_tree.h" + +#include <yql/essentials/sql/v1/complete/core/environment.h> + +namespace NSQLComplete { + + NYT::TNode Evaluate(SQLv1::Bind_parameterContext* ctx, const TEnvironment& env); + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/analysis/global/global.cpp b/yql/essentials/sql/v1/complete/analysis/global/global.cpp index a216b03de1a..9acb3310fac 100644 --- a/yql/essentials/sql/v1/complete/analysis/global/global.cpp +++ b/yql/essentials/sql/v1/complete/analysis/global/global.cpp @@ -42,13 +42,13 @@ namespace NSQLComplete { Parser_.setErrorHandler(std::make_shared<TErrorStrategy>()); } - TGlobalContext Analyze(TCompletionInput input) override { + TGlobalContext Analyze(TCompletionInput input, TEnvironment env) override { SQLv1::Sql_queryContext* sqlQuery = Parse(input.Text); Y_ENSURE(sqlQuery); TGlobalContext ctx; - ctx.Use = FindUseStatement(sqlQuery, &Tokens_, input.CursorPosition); + ctx.Use = FindUseStatement(sqlQuery, &Tokens_, input.CursorPosition, env); return ctx; } @@ -70,9 +70,9 @@ namespace NSQLComplete { class TGlobalAnalysis: public IGlobalAnalysis { public: - TGlobalContext Analyze(TCompletionInput input) override { + TGlobalContext Analyze(TCompletionInput input, TEnvironment env) override { const bool isAnsiLexer = IsAnsiQuery(TString(input.Text)); - return GetSpecialized(isAnsiLexer).Analyze(std::move(input)); + return GetSpecialized(isAnsiLexer).Analyze(std::move(input), std::move(env)); } private: diff --git a/yql/essentials/sql/v1/complete/analysis/global/global.h b/yql/essentials/sql/v1/complete/analysis/global/global.h index a5249bc3a48..97c55738e45 100644 --- a/yql/essentials/sql/v1/complete/analysis/global/global.h +++ b/yql/essentials/sql/v1/complete/analysis/global/global.h @@ -1,6 +1,7 @@ #pragma once #include <yql/essentials/sql/v1/complete/core/input.h> +#include <yql/essentials/sql/v1/complete/core/environment.h> #include <util/generic/ptr.h> #include <util/generic/maybe.h> @@ -22,7 +23,7 @@ namespace NSQLComplete { using TPtr = THolder<IGlobalAnalysis>; virtual ~IGlobalAnalysis() = default; - virtual TGlobalContext Analyze(TCompletionInput input) = 0; + virtual TGlobalContext Analyze(TCompletionInput input, TEnvironment env) = 0; }; IGlobalAnalysis::TPtr MakeGlobalAnalysis(); diff --git a/yql/essentials/sql/v1/complete/analysis/global/use.cpp b/yql/essentials/sql/v1/complete/analysis/global/use.cpp index da7dc6a5751..2fd7fab6614 100644 --- a/yql/essentials/sql/v1/complete/analysis/global/use.cpp +++ b/yql/essentials/sql/v1/complete/analysis/global/use.cpp @@ -1,14 +1,20 @@ #include "use.h" +#include "evaluate.h" + namespace NSQLComplete { namespace { class TVisitor: public SQLv1Antlr4BaseVisitor { public: - TVisitor(antlr4::TokenStream* tokens, size_t cursorPosition) + TVisitor( + antlr4::TokenStream* tokens, + size_t cursorPosition, + const TEnvironment* env) : Tokens_(tokens) , CursorPosition_(cursorPosition) + , Env_(env) { } @@ -33,7 +39,9 @@ namespace NSQLComplete { } if (SQLv1::Pure_column_or_namedContext* ctx = expr->pure_column_or_named()) { - cluster = ctx->getText(); + if (auto id = GetId(ctx)) { + cluster = std::move(*id); + } } if (cluster.empty()) { @@ -76,8 +84,27 @@ namespace NSQLComplete { return antlr4::misc::Interval(CursorPosition_, CursorPosition_); } + TMaybe<TString> GetId(SQLv1::Pure_column_or_namedContext* ctx) const { + if (auto* x = ctx->bind_parameter()) { + return GetId(x); + } else if (auto* x = ctx->an_id()) { + return x->getText(); + } else { + Y_ABORT("You should change implementation according grammar changes"); + } + } + + TMaybe<TString> GetId(SQLv1::Bind_parameterContext* ctx) const { + NYT::TNode node = Evaluate(ctx, *Env_); + if (!node.HasValue() || !node.IsString()) { + return Nothing(); + } + return node.AsString(); + } + antlr4::TokenStream* Tokens_; size_t CursorPosition_; + const TEnvironment* Env_; }; } // namespace @@ -85,8 +112,9 @@ namespace NSQLComplete { TMaybe<TUseContext> FindUseStatement( SQLv1::Sql_queryContext* ctx, antlr4::TokenStream* tokens, - size_t cursorPosition) { - std::any result = TVisitor(tokens, cursorPosition).visit(ctx); + size_t cursorPosition, + const TEnvironment& env) { + std::any result = TVisitor(tokens, cursorPosition, &env).visit(ctx); if (!result.has_value()) { return Nothing(); } diff --git a/yql/essentials/sql/v1/complete/analysis/global/use.h b/yql/essentials/sql/v1/complete/analysis/global/use.h index 54f3557fd62..0cdb9b15469 100644 --- a/yql/essentials/sql/v1/complete/analysis/global/use.h +++ b/yql/essentials/sql/v1/complete/analysis/global/use.h @@ -12,6 +12,7 @@ namespace NSQLComplete { TMaybe<TUseContext> FindUseStatement( SQLv1::Sql_queryContext* ctx, antlr4::TokenStream* tokens, - size_t cursorPosition); + size_t cursorPosition, + const TEnvironment& env); } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/analysis/global/ya.make b/yql/essentials/sql/v1/complete/analysis/global/ya.make index a28d99f94c2..a8aa1eb7214 100644 --- a/yql/essentials/sql/v1/complete/analysis/global/ya.make +++ b/yql/essentials/sql/v1/complete/analysis/global/ya.make @@ -1,6 +1,7 @@ LIBRARY() SRCS( + evaluate.cpp global.cpp use.cpp ) diff --git a/yql/essentials/sql/v1/complete/antlr4/c3i.h b/yql/essentials/sql/v1/complete/antlr4/c3i.h index ed69f3cf29b..89201e12562 100644 --- a/yql/essentials/sql/v1/complete/antlr4/c3i.h +++ b/yql/essentials/sql/v1/complete/antlr4/c3i.h @@ -36,6 +36,7 @@ namespace NSQLComplete { struct TConfig { std::unordered_set<TTokenId> IgnoredTokens; std::unordered_set<TRuleId> PreferredRules; + std::unordered_set<TRuleId> IgnoredRules; }; virtual TC3Candidates Complete(TStringBuf text, size_t caretTokenIndex) = 0; diff --git a/yql/essentials/sql/v1/complete/antlr4/c3t.h b/yql/essentials/sql/v1/complete/antlr4/c3t.h index 1f71553e04f..2b49068ac8f 100644 --- a/yql/essentials/sql/v1/complete/antlr4/c3t.h +++ b/yql/essentials/sql/v1/complete/antlr4/c3t.h @@ -25,12 +25,17 @@ namespace NSQLComplete { , Tokens_(&Lexer_) , Parser_(&Tokens_) , CompletionCore_(&Parser_) + , IgnoredRules_(std::move(config.IgnoredRules)) { Lexer_.removeErrorListeners(); Parser_.removeErrorListeners(); CompletionCore_.ignoredTokens = std::move(config.IgnoredTokens); CompletionCore_.preferredRules = std::move(config.PreferredRules); + + for (TRuleId rule : IgnoredRules_) { + CompletionCore_.preferredRules.emplace(rule); + } } TC3Candidates Complete(TStringBuf text, size_t caretTokenIndex) override { @@ -47,23 +52,36 @@ namespace NSQLComplete { Tokens_.fill(); } - static TC3Candidates Converted(c3::CandidatesCollection candidates) { + TC3Candidates Converted(c3::CandidatesCollection candidates) const { TC3Candidates converted; + for (auto& [token, following] : candidates.tokens) { converted.Tokens.emplace_back(token, std::move(following)); } + for (auto& [rule, data] : candidates.rules) { + if (IsIgnored(rule, data.ruleList)) { + continue; + } + converted.Rules.emplace_back(rule, std::move(data.ruleList)); converted.Rules.back().ParserCallStack.emplace_back(rule); } + return converted; } + bool IsIgnored(TRuleId head, const std::vector<TRuleId> tail) const { + return IgnoredRules_.contains(head) || + AnyOf(tail, [this](TRuleId r) { return IgnoredRules_.contains(r); }); + } + antlr4::ANTLRInputStream Chars_; G::TLexer Lexer_; antlr4::BufferedTokenStream Tokens_; G::TParser Parser_; c3::CodeCompletionCore CompletionCore_; + std::unordered_set<TRuleId> IgnoredRules_; }; } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/core/environment.cpp b/yql/essentials/sql/v1/complete/core/environment.cpp new file mode 100644 index 00000000000..89f179533be --- /dev/null +++ b/yql/essentials/sql/v1/complete/core/environment.cpp @@ -0,0 +1 @@ +#include "environment.h" diff --git a/yql/essentials/sql/v1/complete/core/environment.h b/yql/essentials/sql/v1/complete/core/environment.h new file mode 100644 index 00000000000..0f0299a9353 --- /dev/null +++ b/yql/essentials/sql/v1/complete/core/environment.h @@ -0,0 +1,16 @@ +#pragma once + +#include <library/cpp/yson/node/node.h> + +#include <util/generic/string.h> +#include <util/generic/hash.h> + +namespace NSQLComplete { + + struct TEnvironment { + // Given `{ "$x": "{ "Data": "foo" }" }`, + // it will contain `{ "$x": "foo" }` + THashMap<TString, NYT::TNode> Parameters; + }; + +} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/core/ya.make b/yql/essentials/sql/v1/complete/core/ya.make index 8bc457f8f95..599810301ca 100644 --- a/yql/essentials/sql/v1/complete/core/ya.make +++ b/yql/essentials/sql/v1/complete/core/ya.make @@ -1,7 +1,12 @@ LIBRARY() SRCS( + environment.cpp input.cpp ) +PEERDIR( + library/cpp/yson/node +) + END() diff --git a/yql/essentials/sql/v1/complete/name/object/dispatch/schema.cpp b/yql/essentials/sql/v1/complete/name/object/dispatch/schema.cpp deleted file mode 100644 index f6d79b280a0..00000000000 --- a/yql/essentials/sql/v1/complete/name/object/dispatch/schema.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include "schema.h" - -namespace NSQLComplete { - - namespace { - - class TSchema: public ISchema { - public: - explicit TSchema(THashMap<TString, ISchema::TPtr> mapping) - : Mapping_(std::move(mapping)) - { - } - - NThreading::TFuture<TListResponse> List(const TListRequest& request) const override { - auto iter = Mapping_.find(request.Cluster); - if (iter == std::end(Mapping_)) { - yexception e; - e << "unknown cluster '" << request.Cluster << "'"; - std::exception_ptr p = std::make_exception_ptr(e); - return NThreading::MakeErrorFuture<TListResponse>(p); - } - - return iter->second->List(request); - } - - private: - THashMap<TString, ISchema::TPtr> Mapping_; - }; - - } // namespace - - ISchema::TPtr MakeDispatchSchema(THashMap<TString, ISchema::TPtr> mapping) { - return new TSchema(std::move(mapping)); - } - -} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/dispatch/schema.h b/yql/essentials/sql/v1/complete/name/object/dispatch/schema.h deleted file mode 100644 index 517a3ad0af7..00000000000 --- a/yql/essentials/sql/v1/complete/name/object/dispatch/schema.h +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -#include <yql/essentials/sql/v1/complete/name/object/schema.h> - -namespace NSQLComplete { - - ISchema::TPtr MakeDispatchSchema(THashMap<TString, ISchema::TPtr> mapping); - -} // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/dispatch/ya.make b/yql/essentials/sql/v1/complete/name/object/dispatch/ya.make deleted file mode 100644 index 071bf5dff7d..00000000000 --- a/yql/essentials/sql/v1/complete/name/object/dispatch/ya.make +++ /dev/null @@ -1,11 +0,0 @@ -LIBRARY() - -SRCS( - schema.cpp -) - -PEERDIR( - yql/essentials/sql/v1/complete/name/object -) - -END() diff --git a/yql/essentials/sql/v1/complete/name/object/simple/schema_ut.cpp b/yql/essentials/sql/v1/complete/name/object/simple/schema_ut.cpp index 954ecc4da75..7fdd5d66b21 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/schema_ut.cpp +++ b/yql/essentials/sql/v1/complete/name/object/simple/schema_ut.cpp @@ -9,16 +9,16 @@ using namespace NSQLComplete; Y_UNIT_TEST_SUITE(StaticSchemaTests) { ISchema::TPtr MakeStaticSchemaUT() { - THashMap<TString, TVector<TFolderEntry>> fs = { - {"/", {{"Folder", "local"}, - {"Folder", "test"}, - {"Folder", "prod"}}}, - {"/local/", {{"Table", "example"}, - {"Table", "account"}, - {"Table", "abacaba"}}}, - {"/test/", {{"Folder", "service"}, - {"Table", "meta"}}}, - {"/test/service/", {{"Table", "example"}}}, + THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> fs = { + {"", {{"/", {{"Folder", "local"}, + {"Folder", "test"}, + {"Folder", "prod"}}}, + {"/local/", {{"Table", "example"}, + {"Table", "account"}, + {"Table", "abacaba"}}}, + {"/test/", {{"Folder", "service"}, + {"Table", "meta"}}}, + {"/test/service/", {{"Table", "example"}}}}}, }; return MakeSimpleSchema( MakeStaticSimpleSchema(std::move(fs))); diff --git a/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp index 99d946a0d26..3482da8a332 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp +++ b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.cpp @@ -6,12 +6,14 @@ namespace NSQLComplete { class TSimpleSchema: public ISimpleSchema { public: - explicit TSimpleSchema(THashMap<TString, TVector<TFolderEntry>> data) + explicit TSimpleSchema(THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> data) : Data_(std::move(data)) { - for (const auto& [k, _] : Data_) { - Y_ENSURE(k.StartsWith("/"), k << " must start with the '/'"); - Y_ENSURE(k.EndsWith("/"), k << " must end with the '/'"); + for (const auto& [_, tables] : Data_) { + for (const auto& [k, _] : tables) { + Y_ENSURE(k.StartsWith("/"), k << " must start with the '/'"); + Y_ENSURE(k.EndsWith("/"), k << " must end with the '/'"); + } } } @@ -26,25 +28,31 @@ namespace NSQLComplete { return {head, tail}; } - NThreading::TFuture<TVector<TFolderEntry>> List(TString folder) const override { + NThreading::TFuture<TVector<TFolderEntry>> List(TString cluster, TString folder) const override { if (!folder.StartsWith('/')) { folder.prepend('/'); } TVector<TFolderEntry> entries; - if (const auto* data = Data_.FindPtr(folder)) { - entries = *data; + + const THashMap<TString, TVector<TFolderEntry>>* tables = nullptr; + const TVector<TFolderEntry>* items = nullptr; + if ((tables = Data_.FindPtr(cluster)) && + (items = tables->FindPtr(folder))) { + entries = *items; } + return NThreading::MakeFuture(std::move(entries)); } private: - THashMap<TString, TVector<TFolderEntry>> Data_; + THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> Data_; }; } // namespace - ISimpleSchema::TPtr MakeStaticSimpleSchema(THashMap<TString, TVector<TFolderEntry>> fs) { + ISimpleSchema::TPtr MakeStaticSimpleSchema( + THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> fs) { return new TSimpleSchema(std::move(fs)); } diff --git a/yql/essentials/sql/v1/complete/name/object/simple/static/schema.h b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.h index f04c89f0b23..009b433ee4a 100644 --- a/yql/essentials/sql/v1/complete/name/object/simple/static/schema.h +++ b/yql/essentials/sql/v1/complete/name/object/simple/static/schema.h @@ -5,6 +5,6 @@ namespace NSQLComplete { ISimpleSchema::TPtr MakeStaticSimpleSchema( - THashMap<TString, TVector<TFolderEntry>> fs); + THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> fs); } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/name/object/ya.make b/yql/essentials/sql/v1/complete/name/object/ya.make index 2561c018292..dd5e5046315 100644 --- a/yql/essentials/sql/v1/complete/name/object/ya.make +++ b/yql/essentials/sql/v1/complete/name/object/ya.make @@ -11,6 +11,5 @@ PEERDIR( END() RECURSE( - dispatch simple ) diff --git a/yql/essentials/sql/v1/complete/name/service/schema/name_service.cpp b/yql/essentials/sql/v1/complete/name/service/schema/name_service.cpp index de8e8db65ac..b21ad828317 100644 --- a/yql/essentials/sql/v1/complete/name/service/schema/name_service.cpp +++ b/yql/essentials/sql/v1/complete/name/service/schema/name_service.cpp @@ -32,12 +32,7 @@ namespace NSQLComplete { } static TString ClusterName(const TObjectNameConstraints& constraints) { - TString name = constraints.Cluster; - if (!constraints.Provider.empty()) { - name.prepend(":"); - name.prepend(constraints.Provider); - } - return name; + return constraints.Cluster; } static TListFilter ToListFilter(const TNameConstraints& constraints) { diff --git a/yql/essentials/sql/v1/complete/sql_complete.cpp b/yql/essentials/sql/v1/complete/sql_complete.cpp index 00d346f0770..807786512e4 100644 --- a/yql/essentials/sql/v1/complete/sql_complete.cpp +++ b/yql/essentials/sql/v1/complete/sql_complete.cpp @@ -1,5 +1,6 @@ #include "sql_complete.h" +#include <yql/essentials/sql/v1/complete/syntax/grammar.h> #include <yql/essentials/sql/v1/complete/text/word.h> #include <yql/essentials/sql/v1/complete/name/service/static/name_service.h> #include <yql/essentials/sql/v1/complete/syntax/local.h> @@ -18,17 +19,17 @@ namespace NSQLComplete { INameService::TPtr names, ISqlCompletionEngine::TConfiguration configuration) : Configuration_(std::move(configuration)) - , SyntaxAnalysis_(MakeLocalSyntaxAnalysis(lexer)) + , SyntaxAnalysis_(MakeLocalSyntaxAnalysis(lexer, Configuration_.IgnoredRules)) , GlobalAnalysis_(MakeGlobalAnalysis()) , Names_(std::move(names)) { } TCompletion Complete(TCompletionInput input) override { - return CompleteAsync(std::move(input)).ExtractValueSync(); + return CompleteAsync(std::move(input), {}).ExtractValueSync(); } - virtual NThreading::TFuture<TCompletion> CompleteAsync(TCompletionInput input) override { + virtual NThreading::TFuture<TCompletion> CompleteAsync(TCompletionInput input, TEnvironment env) override { if ( input.CursorPosition < input.Text.length() && IsUTF8ContinuationByte(input.Text.at(input.CursorPosition)) || @@ -41,7 +42,7 @@ namespace NSQLComplete { TLocalSyntaxContext context = SyntaxAnalysis_->Analyze(input); auto keywords = context.Keywords; - TGlobalContext global = GlobalAnalysis_->Analyze(input); + TGlobalContext global = GlobalAnalysis_->Analyze(input, std::move(env)); TNameRequest request = NameRequestFrom(input, context, global); if (request.IsEmpty()) { @@ -118,7 +119,7 @@ namespace NSQLComplete { if (context.Cluster) { TClusterName::TConstraints constraints; - constraints.Namespace = context.Cluster->Provider; + constraints.Namespace = ""; // TODO(YQL-19747): filter by provider request.Constraints.Cluster = std::move(constraints); } @@ -185,14 +186,14 @@ namespace NSQLComplete { if constexpr (std::is_base_of_v<TFolderName, T>) { name.Indentifier.append('/'); if (!context.Object->IsQuoted) { - name.Indentifier = Quoted(std::move(name.Indentifier)); + name.Indentifier.prepend('`'); } return {ECandidateKind::FolderName, std::move(name.Indentifier)}; } if constexpr (std::is_base_of_v<TTableName, T>) { if (!context.Object->IsQuoted) { - name.Indentifier = Quoted(std::move(name.Indentifier)); + name.Indentifier.prepend('`'); } return {ECandidateKind::TableName, std::move(name.Indentifier)}; } @@ -213,6 +214,45 @@ namespace NSQLComplete { INameService::TPtr Names_; }; + ISqlCompletionEngine::TConfiguration MakeConfiguration(THashSet<TString> allowedStmts) { + allowedStmts.emplace("sql_stmt"); + + ISqlCompletionEngine::TConfiguration config; + for (const std::string& name : GetSqlGrammar().GetAllRules()) { + if (name.ends_with("_stmt") && !allowedStmts.contains(name)) { + config.IgnoredRules.emplace(name); + } + } + return config; + } + + ISqlCompletionEngine::TConfiguration MakeYDBConfiguration() { + return { + .IgnoredRules = {}, + }; + } + + ISqlCompletionEngine::TConfiguration MakeYQLConfiguration() { + return MakeConfiguration(/* allowedStmts = */ { + "lambda_stmt", + "pragma_stmt", + "select_stmt", + "named_nodes_stmt", + "drop_table_stmt", + "use_stmt", + "into_table_stmt", + "commit_stmt", + "declare_stmt", + "import_stmt", + "export_stmt", + "do_stmt", + "define_action_or_subquery_stmt", + "if_stmt", + "for_stmt", + "values_stmt", + }); + } + ISqlCompletionEngine::TPtr MakeSqlCompletionEngine( TLexerSupplier lexer, INameService::TPtr names, diff --git a/yql/essentials/sql/v1/complete/sql_complete.h b/yql/essentials/sql/v1/complete/sql_complete.h index 1bc2c0ecf4e..e5dffcb8d27 100644 --- a/yql/essentials/sql/v1/complete/sql_complete.h +++ b/yql/essentials/sql/v1/complete/sql_complete.h @@ -1,6 +1,7 @@ #pragma once #include <yql/essentials/sql/v1/complete/core/input.h> +#include <yql/essentials/sql/v1/complete/core/environment.h> #include <yql/essentials/sql/v1/complete/name/service/name_service.h> #include <yql/essentials/sql/v1/lexer/lexer.h> @@ -8,6 +9,7 @@ #include <util/generic/string.h> #include <util/generic/vector.h> +#include <util/generic/hash_set.h> namespace NSQLComplete { @@ -31,6 +33,7 @@ namespace NSQLComplete { struct TCandidate { ECandidateKind Kind; TString Content; + size_t CursorShift = 0; friend bool operator==(const TCandidate& lhs, const TCandidate& rhs) = default; }; @@ -46,15 +49,21 @@ namespace NSQLComplete { struct TConfiguration { size_t Limit = 256; + THashSet<TString> IgnoredRules; }; virtual ~ISqlCompletionEngine() = default; virtual TCompletion Complete(TCompletionInput input) = 0; // TODO(YQL-19747): migrate YDB CLI to CompleteAsync - virtual NThreading::TFuture<TCompletion> CompleteAsync(TCompletionInput input) = 0; + virtual NThreading::TFuture<TCompletion> + CompleteAsync(TCompletionInput input, TEnvironment env = {}) = 0; }; using TLexerSupplier = std::function<NSQLTranslation::ILexer::TPtr(bool ansi)>; + ISqlCompletionEngine::TConfiguration MakeYDBConfiguration(); + + ISqlCompletionEngine::TConfiguration MakeYQLConfiguration(); + ISqlCompletionEngine::TPtr MakeSqlCompletionEngine( TLexerSupplier lexer, INameService::TPtr names, diff --git a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp index d3a39d1c503..7460924b48f 100644 --- a/yql/essentials/sql/v1/complete/sql_complete_ut.cpp +++ b/yql/essentials/sql/v1/complete/sql_complete_ut.cpp @@ -1,7 +1,7 @@ #include "sql_complete.h" +#include <yql/essentials/sql/v1/complete/syntax/grammar.h> #include <yql/essentials/sql/v1/complete/name/cluster/static/discovery.h> -#include <yql/essentials/sql/v1/complete/name/object/dispatch/schema.h> #include <yql/essentials/sql/v1/complete/name/object/simple/schema.h> #include <yql/essentials/sql/v1/complete/name/object/simple/static/schema.h> #include <yql/essentials/sql/v1/complete/name/service/ranking/frequency.h> @@ -16,6 +16,8 @@ #include <yql/essentials/sql/v1/lexer/antlr4_pure_ansi/lexer.h> #include <library/cpp/testing/unittest/registar.h> +#include <library/cpp/iterator/iterate_keys.h> +#include <library/cpp/iterator/functools.h> #include <util/charset/utf8.h> @@ -78,7 +80,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { }, }; - THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> fss = { + THashMap<TString, THashMap<TString, TVector<TFolderEntry>>> fs = { {"", {{"/", {{"Folder", "local"}, {"Folder", "test"}, {"Folder", "prod"}, @@ -94,40 +96,29 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { {{"/", {{"Table", "people"}, {"Folder", "yql"}}}, {"/yql/", {{"Table", "tutorial"}}}}}, - {"yt:saurus", + {"saurus", {{"/", {{"Table", "maxim"}}}}}, }; - TVector<TString> clusters; - for (const auto& [cluster, _] : fss) { - clusters.emplace_back(cluster); - } - EraseIf(clusters, [](const auto& s) { return s.empty(); }); + auto clustersIt = NFuncTools::Filter( + [](const auto& x) { return !x.empty(); }, IterateKeys(fs)); + TVector<TString> clusters(begin(clustersIt), end(clustersIt)); TFrequencyData frequency; - IRanking::TPtr ranking = MakeDefaultRanking(frequency); - - THashMap<TString, ISchema::TPtr> schemasByCluster; - for (auto& [cluster, fs] : fss) { - schemasByCluster[std::move(cluster)] = - MakeSimpleSchema( - MakeStaticSimpleSchema(std::move(fs))); - } - TVector<INameService::TPtr> children = { MakeStaticNameService(std::move(names), frequency), - MakeSchemaNameService(MakeDispatchSchema(std::move(schemasByCluster))), + MakeSchemaNameService(MakeSimpleSchema(MakeStaticSimpleSchema(std::move(fs)))), MakeClusterNameService(MakeStaticClusterDiscovery(std::move(clusters))), }; - - INameService::TPtr service = MakeUnionNameService(std::move(children), ranking); + INameService::TPtr service = MakeUnionNameService( + std::move(children), MakeDefaultRanking(frequency)); return MakeSqlCompletionEngine(std::move(lexer), std::move(service)); } - TVector<TCandidate> Complete(ISqlCompletionEngine::TPtr& engine, TString sharped) { - return engine->CompleteAsync(SharpedInput(sharped)).GetValueSync().Candidates; + TVector<TCandidate> Complete(ISqlCompletionEngine::TPtr& engine, TString sharped, TEnvironment env = {}) { + return engine->CompleteAsync(SharpedInput(sharped), std::move(env)).GetValueSync().Candidates; } TVector<TCandidate> CompleteTop(size_t limit, ISqlCompletionEngine::TPtr& engine, TString sharped) { @@ -189,12 +180,47 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { Y_UNIT_TEST(Use) { TVector<TCandidate> expected = { {ClusterName, "example"}, - {ClusterName, "yt:saurus"}, + {ClusterName, "saurus"}, }; auto engine = MakeSqlCompletionEngineUT(); UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "USE "), expected); } + Y_UNIT_TEST(UseClusterResultion) { + auto engine = MakeSqlCompletionEngineUT(); + { + TVector<TCandidate> expected = { + {TableName, "`maxim"}, + {ClusterName, "example"}, + {ClusterName, "saurus"}, + {Keyword, "ANY"}, + }; + UNIT_ASSERT_VALUES_EQUAL( + Complete( + engine, + "USE yt:$cluster_name; SELECT * FROM ", + {.Parameters = {{"cluster_name", "saurus"}}}), + expected); + } + { + TVector<TCandidate> expected = { + {FolderName, "`.sys/"}, + {FolderName, "`local/"}, + {FolderName, "`prod/"}, + {FolderName, "`test/"}, + {ClusterName, "example"}, + {ClusterName, "saurus"}, + {Keyword, "ANY"}, + }; + UNIT_ASSERT_VALUES_EQUAL( + Complete( + engine, + "USE yt:$cluster_name; SELECT * FROM ", + {.Parameters = {}}), + expected); + } + } + Y_UNIT_TEST(Alter) { TVector<TCandidate> expected = { {Keyword, "ASYNC REPLICATION"}, @@ -243,12 +269,12 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { auto engine = MakeSqlCompletionEngineUT(); { TVector<TCandidate> expected = { - {FolderName, "`.sys/`"}, - {FolderName, "`local/`"}, - {FolderName, "`prod/`"}, - {FolderName, "`test/`"}, + {FolderName, "`.sys/"}, + {FolderName, "`local/"}, + {FolderName, "`prod/"}, + {FolderName, "`test/"}, {ClusterName, "example"}, - {ClusterName, "yt:saurus"}, + {ClusterName, "saurus"}, {Keyword, "IF NOT EXISTS"}, }; UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "CREATE TABLE #"), expected); @@ -292,12 +318,12 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { Y_UNIT_TEST(DropObject) { TVector<TCandidate> expected = { - {FolderName, "`.sys/`"}, - {FolderName, "`local/`"}, - {FolderName, "`prod/`"}, - {FolderName, "`test/`"}, + {FolderName, "`.sys/"}, + {FolderName, "`local/"}, + {FolderName, "`prod/"}, + {FolderName, "`test/"}, {ClusterName, "example"}, - {ClusterName, "yt:saurus"}, + {ClusterName, "saurus"}, {Keyword, "IF EXISTS"}, }; auto engine = MakeSqlCompletionEngineUT(); @@ -488,12 +514,12 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { auto engine = MakeSqlCompletionEngineUT(); { TVector<TCandidate> expected = { - {FolderName, "`.sys/`"}, - {FolderName, "`local/`"}, - {FolderName, "`prod/`"}, - {FolderName, "`test/`"}, + {FolderName, "`.sys/"}, + {FolderName, "`local/"}, + {FolderName, "`prod/"}, + {FolderName, "`test/"}, {ClusterName, "example"}, - {ClusterName, "yt:saurus"}, + {ClusterName, "saurus"}, {Keyword, "ANY"}, }; UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM "), expected); @@ -501,7 +527,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { { TString input = "SELECT * FROM pr"; TVector<TCandidate> expected = { - {FolderName, "`prod/`"}, + {FolderName, "`prod/"}, }; TCompletion actual = engine->Complete(SharpedInput(input)); UNIT_ASSERT_VALUES_EQUAL(actual.Candidates, expected); @@ -593,12 +619,7 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { auto engine = MakeSqlCompletionEngineUT(); { TVector<TCandidate> expected = { - {ClusterName, "yt:saurus"}, - }; - UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM yt#"), expected); - } - { - TVector<TCandidate> expected = { + {ClusterName, "example"}, {ClusterName, "saurus"}, }; UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM yt:"), expected); @@ -611,13 +632,13 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { } { TVector<TCandidate> expected = { - {TableName, "`maxim`"}, + {TableName, "`maxim"}, }; UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "SELECT * FROM yt:saurus."), expected); } { TVector<TCandidate> expected = { - {TableName, "`people`"}, + {TableName, "`people"}, }; UNIT_ASSERT_VALUES_EQUAL(CompleteTop(1, engine, "SELECT * FROM example."), expected); } @@ -633,34 +654,34 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { auto engine = MakeSqlCompletionEngineUT(); { TVector<TCandidate> expected = { - {TableName, "`maxim`"}, + {TableName, "`maxim"}, {ClusterName, "example"}, - {ClusterName, "yt:saurus"}, + {ClusterName, "saurus"}, {Keyword, "ANY"}, }; UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "USE yt:saurus; SELECT * FROM "), expected); } { TVector<TCandidate> expected = { - {TableName, "`people`"}, - {FolderName, "`yql/`"}, + {TableName, "`people"}, + {FolderName, "`yql/"}, }; UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "USE yt:saurus; SELECT * FROM example."), expected); } { TVector<TCandidate> expected = { - {TableName, "`maxim`"}, + {TableName, "`maxim"}, {ClusterName, "example"}, - {ClusterName, "yt:saurus"}, + {ClusterName, "saurus"}, {Keyword, "ANY"}, }; UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "USE example; USE yt:saurus; SELECT * FROM "), expected); } { TVector<TCandidate> expected = { - {TableName, "`maxim`"}, + {TableName, "`maxim"}, {ClusterName, "example"}, - {ClusterName, "yt:saurus"}, + {ClusterName, "saurus"}, {Keyword, "ANY"}, }; UNIT_ASSERT_VALUES_EQUAL(Complete(engine, R"( @@ -673,10 +694,10 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { } { TVector<TCandidate> expected = { - {TableName, "`people`"}, - {FolderName, "`yql/`"}, + {TableName, "`people"}, + {FolderName, "`yql/"}, {ClusterName, "example"}, - {ClusterName, "yt:saurus"}, + {ClusterName, "saurus"}, {Keyword, "ANY"}, }; UNIT_ASSERT_VALUES_EQUAL(Complete(engine, R"( @@ -745,12 +766,12 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { auto engine = MakeSqlCompletionEngineUT(); { TVector<TCandidate> expected = { - {FolderName, "`.sys/`"}, - {FolderName, "`local/`"}, - {FolderName, "`prod/`"}, - {FolderName, "`test/`"}, + {FolderName, "`.sys/"}, + {FolderName, "`local/"}, + {FolderName, "`prod/"}, + {FolderName, "`test/"}, {ClusterName, "example"}, - {ClusterName, "yt:saurus"}, + {ClusterName, "saurus"}, }; UNIT_ASSERT_VALUES_EQUAL(Complete(engine, "UPSERT INTO "), expected); } @@ -980,12 +1001,17 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { } Y_UNIT_TEST(Tabbing) { - TString query = - "SELECT \n" - " 123467, \"Hello, {name}! 编码\"}, \n" - " (1 + (5 * 1 / 0)), MIN(identifier), \n" - " Bool(field), Math::Sin(var) \n" - "FROM `local/test/space/table` JOIN test;"; + TString query = R"( +USE example; + +SELECT + 123467, \"Hello, {name}! 编码\"}, + (1 + (5 * 1 / 0)), MIN(identifier), + Bool(field), Math::Sin(var) +FROM `local/test/space/table` +JOIN yt:$cluster_name.test; +)"; + query += query + ";"; query += query + ";"; @@ -1176,4 +1202,30 @@ Y_UNIT_TEST_SUITE(SqlCompleteTests) { } } + Y_UNIT_TEST(IgnoredRules) { + auto lexer = MakePureLexerSupplier(); + + TNameSet names; + TFrequencyData frequency; + auto service = MakeStaticNameService(names, MakeDefaultRanking(frequency)); + + { + auto engine = MakeSqlCompletionEngine(lexer, service); + UNIT_ASSERT_UNEQUAL(Complete(engine, {"UPDA"}).size(), 0); + UNIT_ASSERT_UNEQUAL(Complete(engine, {"DELE"}).size(), 0); + UNIT_ASSERT_UNEQUAL(Complete(engine, {"ROLL"}).size(), 0); + UNIT_ASSERT_UNEQUAL(Complete(engine, {"INSE"}).size(), 0); + UNIT_ASSERT_UNEQUAL(Complete(engine, {"SELE"}).size(), 0); + } + + auto config = MakeYQLConfiguration(); + auto engine = MakeSqlCompletionEngine(lexer, std::move(service), config); + + UNIT_ASSERT_EQUAL(Complete(engine, {"UPDA"}).size(), 0); + UNIT_ASSERT_EQUAL(Complete(engine, {"DELE"}).size(), 0); + UNIT_ASSERT_EQUAL(Complete(engine, {"ROLL"}).size(), 0); + UNIT_ASSERT_UNEQUAL(Complete(engine, {"INSE"}).size(), 0); + UNIT_ASSERT_UNEQUAL(Complete(engine, {"SELE"}).size(), 0); + } + } // Y_UNIT_TEST_SUITE(SqlCompleteTests) diff --git a/yql/essentials/sql/v1/complete/syntax/grammar.cpp b/yql/essentials/sql/v1/complete/syntax/grammar.cpp index c080fae5ae4..790f272db86 100644 --- a/yql/essentials/sql/v1/complete/syntax/grammar.cpp +++ b/yql/essentials/sql/v1/complete/syntax/grammar.cpp @@ -34,6 +34,18 @@ namespace NSQLComplete { return Parser_->getRuleNames().at(rule); } + TRuleId GetRuleId(std::string_view symbolized) const override { + TRuleId index = Parser_->getRuleIndex(std::string(symbolized)); + if (index == INVALID_INDEX) { + ythrow yexception() << "Rule \"" << symbolized << "\" not found"; + } + return index; + } + + const std::vector<std::string>& GetAllRules() const override { + return Parser_->getRuleNames(); + } + private: static THolder<antlr4::Parser> MakeDummyParser() { return MakeHolder<NALADefaultAntlr4::SQLv1Antlr4Parser>(nullptr); diff --git a/yql/essentials/sql/v1/complete/syntax/grammar.h b/yql/essentials/sql/v1/complete/syntax/grammar.h index b128129e95f..4a259493dbc 100644 --- a/yql/essentials/sql/v1/complete/syntax/grammar.h +++ b/yql/essentials/sql/v1/complete/syntax/grammar.h @@ -6,6 +6,7 @@ #include <unordered_set> #include <string> +#include <string_view> #ifdef TOKEN_QUERY // Conflict with the winnt.h #undef TOKEN_QUERY @@ -21,6 +22,8 @@ namespace NSQLComplete { public: virtual const antlr4::dfa::Vocabulary& GetVocabulary() const = 0; virtual const std::string& SymbolizedRule(TRuleId rule) const = 0; + virtual TRuleId GetRuleId(std::string_view symbolized) const = 0; + virtual const std::vector<std::string>& GetAllRules() const = 0; virtual const std::unordered_set<TTokenId>& GetAllTokens() const = 0; virtual const std::unordered_set<TTokenId>& GetKeywordTokens() const = 0; virtual const std::unordered_set<TTokenId>& GetPunctuationTokens() const = 0; diff --git a/yql/essentials/sql/v1/complete/syntax/local.cpp b/yql/essentials/sql/v1/complete/syntax/local.cpp index a9803feae85..772f5b78dd5 100644 --- a/yql/essentials/sql/v1/complete/syntax/local.cpp +++ b/yql/essentials/sql/v1/complete/syntax/local.cpp @@ -49,10 +49,11 @@ namespace NSQLComplete { TDefaultYQLGrammar>; public: - explicit TSpecializedLocalSyntaxAnalysis(TLexerSupplier lexer) + explicit TSpecializedLocalSyntaxAnalysis( + TLexerSupplier lexer, const THashSet<TString>& IgnoredRules) : Grammar_(&GetSqlGrammar()) , Lexer_(lexer(/* ansi = */ IsAnsiLexer)) - , C3_(ComputeC3Config()) + , C3_(ComputeC3Config(IgnoredRules)) { } @@ -105,10 +106,11 @@ namespace NSQLComplete { } private: - IC3Engine::TConfig ComputeC3Config() const { + IC3Engine::TConfig ComputeC3Config(const THashSet<TString>& IgnoredRules) const { return { .IgnoredTokens = ComputeIgnoredTokens(), .PreferredRules = ComputePreferredRules(), + .IgnoredRules = ComputeIgnoredRules(IgnoredRules), }; } @@ -127,6 +129,15 @@ namespace NSQLComplete { return GetC3PreferredRules(); } + std::unordered_set<TRuleId> ComputeIgnoredRules(const THashSet<TString>& IgnoredRules) const { + std::unordered_set<TRuleId> ignored; + ignored.reserve(IgnoredRules.size()); + for (const auto& ruleName : IgnoredRules) { + ignored.emplace(Grammar_->GetRuleId(ruleName)); + } + return ignored; + } + TC3Candidates C3Complete(TCompletionInput statement, const TCursorTokenContext& context) { auto enclosing = context.Enclosing(); @@ -311,9 +322,10 @@ namespace NSQLComplete { class TLocalSyntaxAnalysis: public ILocalSyntaxAnalysis { public: - explicit TLocalSyntaxAnalysis(TLexerSupplier lexer) - : DefaultEngine_(lexer) - , AnsiEngine_(lexer) + explicit TLocalSyntaxAnalysis( + TLexerSupplier lexer, const THashSet<TString>& IgnoredRules) + : DefaultEngine_(lexer, IgnoredRules) + , AnsiEngine_(lexer, IgnoredRules) { } @@ -335,8 +347,9 @@ namespace NSQLComplete { TSpecializedLocalSyntaxAnalysis</* IsAnsiLexer = */ true> AnsiEngine_; }; - ILocalSyntaxAnalysis::TPtr MakeLocalSyntaxAnalysis(TLexerSupplier lexer) { - return MakeHolder<TLocalSyntaxAnalysis>(lexer); + ILocalSyntaxAnalysis::TPtr MakeLocalSyntaxAnalysis( + TLexerSupplier lexer, const THashSet<TString>& IgnoredRules) { + return MakeHolder<TLocalSyntaxAnalysis>(lexer, IgnoredRules); } } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/syntax/local.h b/yql/essentials/sql/v1/complete/syntax/local.h index 635485d2b7c..0017afa0684 100644 --- a/yql/essentials/sql/v1/complete/syntax/local.h +++ b/yql/essentials/sql/v1/complete/syntax/local.h @@ -66,6 +66,7 @@ namespace NSQLComplete { virtual ~ILocalSyntaxAnalysis() = default; }; - ILocalSyntaxAnalysis::TPtr MakeLocalSyntaxAnalysis(TLexerSupplier lexer); + ILocalSyntaxAnalysis::TPtr MakeLocalSyntaxAnalysis( + TLexerSupplier lexer, const THashSet<TString>& IgnoredRules); } // namespace NSQLComplete diff --git a/yql/essentials/sql/v1/complete/ut/ya.make b/yql/essentials/sql/v1/complete/ut/ya.make index c978e6e6048..d53220ff3b2 100644 --- a/yql/essentials/sql/v1/complete/ut/ya.make +++ b/yql/essentials/sql/v1/complete/ut/ya.make @@ -8,7 +8,6 @@ PEERDIR( yql/essentials/sql/v1/lexer/antlr4_pure yql/essentials/sql/v1/lexer/antlr4_pure_ansi yql/essentials/sql/v1/complete/name/cluster/static - yql/essentials/sql/v1/complete/name/object/dispatch yql/essentials/sql/v1/complete/name/object/simple yql/essentials/sql/v1/complete/name/object/simple/static yql/essentials/sql/v1/complete/name/service/cluster diff --git a/yql/essentials/sql/v1/format/sql_format_ut.h b/yql/essentials/sql/v1/format/sql_format_ut.h index 8bb2af27939..0a9726aa64d 100644 --- a/yql/essentials/sql/v1/format/sql_format_ut.h +++ b/yql/essentials/sql/v1/format/sql_format_ut.h @@ -1534,6 +1534,8 @@ Y_UNIT_TEST(Union) { "SELECT\n\t1\nUNION ALL\nSELECT\n\t2\nUNION\nSELECT\n\t3\nUNION ALL\nSELECT\n\t4\nUNION\nSELECT\n\t5\n;\n"}, {"select 1 union all (select 2)", "SELECT\n\t1\nUNION ALL\n(\n\tSELECT\n\t\t2\n);\n"}, + {"select 1 union distinct select 2 union select 3 union distinct select 4 union select 5", + "SELECT\n\t1\nUNION DISTINCT\nSELECT\n\t2\nUNION\nSELECT\n\t3\nUNION DISTINCT\nSELECT\n\t4\nUNION\nSELECT\n\t5\n;\n"}, }; TSetup setup; diff --git a/yql/essentials/sql/v1/sql_select.cpp b/yql/essentials/sql/v1/sql_select.cpp index 52dd399a990..8e11721c84f 100644 --- a/yql/essentials/sql/v1/sql_select.cpp +++ b/yql/essentials/sql/v1/sql_select.cpp @@ -1388,18 +1388,28 @@ TSourcePtr TSqlSelect::Build(const TRule& node, TPosition pos, TSelectKindResult outermostSettings.Label = next.Settings.Label; } - switch (b.GetRule_select_op1().Alt_case()) { - case TRule_select_op::kAltSelectOp1: - break; - case TRule_select_op::kAltSelectOp2: - case TRule_select_op::kAltSelectOp3: - Ctx.Error() << "INTERSECT and EXCEPT are not implemented yet"; - return nullptr; - case TRule_select_op::ALT_NOT_SET: - Y_ABORT("You should change implementation according to grammar changes"); + auto selectOp = b.GetRule_select_op1(); + const TString token = ToLowerUTF8(Token(selectOp.GetToken1())); + if (token == "union") { + // nothing + } else if (token == "intersect" || token == "except") { + Ctx.Error() << "INTERSECT and EXCEPT are not implemented yet"; + return nullptr; + } else { + Y_ABORT("You should change implementation according to grammar changes. Invalid token: %s", token.c_str()); } - const bool quantifier = b.GetRule_select_op1().GetAlt_select_op1().HasBlock2(); + bool quantifier = false; + if (selectOp.HasBlock2()) { + const TString token = ToLowerUTF8(Token(selectOp.GetBlock2().GetToken1())); + if (token == "all") { + quantifier = true; + } else if (token == "distinct") { + // nothing + } else { + Y_ABORT("You should change implementation according to grammar changes. Invalid token: %s", token.c_str()); + } + } if (!second && quantifier != currentQuantifier) { auto source = BuildUnion(pos, std::move(sources), currentQuantifier, {}); diff --git a/yql/essentials/sql/v1/sql_ut_common.h b/yql/essentials/sql/v1/sql_ut_common.h index 15ef58b8d86..cead4e2799e 100644 --- a/yql/essentials/sql/v1/sql_ut_common.h +++ b/yql/essentials/sql/v1/sql_ut_common.h @@ -1714,6 +1714,15 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Union"]); } + Y_UNIT_TEST(UnionDistinctTest) { + NYql::TAstParseResult res = SqlToYql("SELECT key FROM plato.Input UNION DISTINCT select subkey FROM plato.Input;"); + UNIT_ASSERT(res.Root); + + TWordCountHive elementStat = {{TString("Union"), 0}}; + VerifyProgram(res, elementStat, {}); + UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Union"]); + } + Y_UNIT_TEST(UnionAggregationTest) { NYql::TAstParseResult res = SqlToYql(R"( PRAGMA DisableEmitUnionMerge; diff --git a/yql/essentials/tests/common/test_framework/yql_utils.py b/yql/essentials/tests/common/test_framework/yql_utils.py index 214f963393f..9ecbe4a1cf7 100644 --- a/yql/essentials/tests/common/test_framework/yql_utils.py +++ b/yql/essentials/tests/common/test_framework/yql_utils.py @@ -503,6 +503,14 @@ def get_langver(cfg): return None +def get_envs(cfg): + envs = dict() + for item in cfg: + if item[0] == 'env': + envs[item[1]] = item[2] + return envs + + def is_skip_forceblocks(cfg): for item in cfg: if item[0] == 'skip_forceblocks': diff --git a/yql/essentials/tests/common/udf_test/test.py b/yql/essentials/tests/common/udf_test/test.py index 359987b2604..6936bde408e 100644 --- a/yql/essentials/tests/common/udf_test/test.py +++ b/yql/essentials/tests/common/udf_test/test.py @@ -78,6 +78,9 @@ def test(case): if yql_utils.get_param('TARGET_PLATFORM') and xfail: pytest.skip('xfail is not supported on non-default target platform') langver = yql_utils.get_langver(cfg) + envs = yql_utils.get_envs(cfg) + if not langver: + langver = "unknown" # no default version, because UDFs may have different release cycles extra_env = dict(os.environ) @@ -85,6 +88,7 @@ def test(case): extra_env["YQL_ARCADIA_BINARY_PATH"] = os.path.expandvars(yatest.common.build_path('.')) extra_env["YQL_ARCADIA_SOURCE_PATH"] = os.path.expandvars(yatest.common.source_path('.')) extra_env["Y_NO_AVX_IN_DOT_PRODUCT"] = "1" + extra_env.update(envs) # this breaks tests using V0 syntax if "YA_TEST_RUNNER" in extra_env: diff --git a/yql/essentials/tests/sql/minirun/part6/canondata/result.json b/yql/essentials/tests/sql/minirun/part6/canondata/result.json index 74fb98a7cc1..b31a0804b64 100644 --- a/yql/essentials/tests/sql/minirun/part6/canondata/result.json +++ b/yql/essentials/tests/sql/minirun/part6/canondata/result.json @@ -1449,6 +1449,20 @@ "uri": "https://{canondata_backend}/995452/27fc70b9589f65bcb911832f0d505bea9e66db7f/resource.tar.gz#test.test_udf-named_args--Results_/results.txt" } ], + "test.test[union-union_distinct-default.txt-Debug]": [ + { + "checksum": "4e68e4a9a59a12794bcae33aa5d639fa", + "size": 297, + "uri": "https://{canondata_backend}/1937492/e480305ecff3ccfe7cdc193e8c9dcc11c255e2c8/resource.tar.gz#test.test_union-union_distinct-default.txt-Debug_/opt.yql" + } + ], + "test.test[union-union_distinct-default.txt-Results]": [ + { + "checksum": "c5237daf2d6c78619bf209e9a39b7a12", + "size": 976, + "uri": "https://{canondata_backend}/1937492/e480305ecff3ccfe7cdc193e8c9dcc11c255e2c8/resource.tar.gz#test.test_union-union_distinct-default.txt-Results_/results.txt" + } + ], "test.test[window-rank/nulls_legacy-default.txt-Debug]": [ { "checksum": "fe3da55ac0b8ab732b49a5d8760d80db", diff --git a/yql/essentials/tests/sql/sql2yql/canondata/result.json b/yql/essentials/tests/sql/sql2yql/canondata/result.json index 920aafb870e..d13181fa778 100644 --- a/yql/essentials/tests/sql/sql2yql/canondata/result.json +++ b/yql/essentials/tests/sql/sql2yql/canondata/result.json @@ -7286,6 +7286,13 @@ "uri": "https://{canondata_backend}/1775319/f1fa0c55bf9f13cff57cf1c990c2330caed8eb1b/resource.tar.gz#test_sql2yql.test_union-union_column_extention_/sql.yql" } ], + "test_sql2yql.test[union-union_distinct]": [ + { + "checksum": "fc0bf00619910dc8863d1e4982c07065", + "size": 1676, + "uri": "https://{canondata_backend}/1925842/1b6f19c4d4916f84ec5d150703910a73500e178a/resource.tar.gz#test_sql2yql.test_union-union_distinct_/sql.yql" + } + ], "test_sql2yql.test[union-union_mix]": [ { "checksum": "a4681f5145adcca6d4a4af7c5e164d73", @@ -11253,6 +11260,11 @@ "uri": "file://test_sql_format.test_union-union_column_extention_/formatted.sql" } ], + "test_sql_format.test[union-union_distinct]": [ + { + "uri": "file://test_sql_format.test_union-union_distinct_/formatted.sql" + } + ], "test_sql_format.test[union-union_mix]": [ { "uri": "file://test_sql_format.test_union-union_mix_/formatted.sql" diff --git a/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_union-union_distinct_/formatted.sql b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_union-union_distinct_/formatted.sql new file mode 100644 index 00000000000..c3d0605b489 --- /dev/null +++ b/yql/essentials/tests/sql/sql2yql/canondata/test_sql_format.test_union-union_distinct_/formatted.sql @@ -0,0 +1,8 @@ +SELECT + 1 AS x, + 2 AS y +UNION DISTINCT +SELECT + 1 AS x, + 2 AS y +; diff --git a/yql/essentials/tests/sql/suites/union/union_distinct.sql b/yql/essentials/tests/sql/suites/union/union_distinct.sql new file mode 100644 index 00000000000..595d54c7284 --- /dev/null +++ b/yql/essentials/tests/sql/suites/union/union_distinct.sql @@ -0,0 +1,7 @@ +SELECT + 1 as x, + 2 as y +UNION DISTINCT +SELECT + 1 as x, + 2 as y; diff --git a/yql/essentials/tools/udf_resolver/discover.cpp b/yql/essentials/tools/udf_resolver/discover.cpp index 530e7d96e98..240cd174b46 100644 --- a/yql/essentials/tools/udf_resolver/discover.cpp +++ b/yql/essentials/tools/udf_resolver/discover.cpp @@ -86,6 +86,8 @@ NYql::TResolveResult DoDiscover(const NYql::TResolve& inMsg, IMutableFunctionReg udfRes->SetSupportsBlocks(funcInfo.SupportsBlocks); udfRes->SetIsStrict(funcInfo.IsStrict); + udfRes->SetMinLangVer(funcInfo.MinLangVer); + udfRes->SetMaxLangVer(funcInfo.MaxLangVer); } } } diff --git a/yql/essentials/tools/udf_resolver/udf_resolver.cpp b/yql/essentials/tools/udf_resolver/udf_resolver.cpp index 03116858f7e..9db127d0864 100644 --- a/yql/essentials/tools/udf_resolver/udf_resolver.cpp +++ b/yql/essentials/tools/udf_resolver/udf_resolver.cpp @@ -194,6 +194,8 @@ void ResolveUDFs() { udfRes->SetSupportsBlocks(funcInfo.SupportsBlocks); udfRes->SetIsStrict(funcInfo.IsStrict); + udfRes->SetMinLangVer(funcInfo.MinLangVer); + udfRes->SetMaxLangVer(funcInfo.MaxLangVer); } catch (yexception& e) { udfRes->SetError(TStringBuilder() << "Internal error was found when udf metadata is loading for function: " << udf.GetName() diff --git a/yql/essentials/tools/yql_complete/yql_complete b/yql/essentials/tools/yql_complete/yql_complete deleted file mode 100644 index 7fc1116ee5d..00000000000 --- a/yql/essentials/tools/yql_complete/yql_complete +++ /dev/null @@ -1 +0,0 @@ -/home/vityaman/.ya/build/symres/bbe5c007c4bcc83d4396e13689e6b39b/yql_complete
\ No newline at end of file diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp index ef74c02a1d3..d4a7121599c 100644 --- a/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp +++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.cpp @@ -443,7 +443,9 @@ void TFacadeRunOptions::Parse(int argc, const char *argv[]) { opts.AddLongOption("langver", "Set current language version").Optional().RequiredArgument("VER") .Handler1T<TString>([this](const TString& str) { - if (!ParseLangVersion(str, LangVer)) { + if (str == "unknown") { + LangVer = UnknownLangVersion; + } else if (!ParseLangVersion(str, LangVer)) { throw yexception() << "Failed to parse language version: " << str; } }); diff --git a/yql/essentials/tools/yql_facade_run/yql_facade_run.h b/yql/essentials/tools/yql_facade_run/yql_facade_run.h index a8661861073..1e6965cf888 100644 --- a/yql/essentials/tools/yql_facade_run/yql_facade_run.h +++ b/yql/essentials/tools/yql_facade_run/yql_facade_run.h @@ -70,7 +70,7 @@ public: ~TFacadeRunOptions(); EProgramType ProgramType = EProgramType::SExpr; - TLangVersion LangVer = UnknownLangVersion; + TLangVersion LangVer = MinLangVersion; TLangVersion MaxLangVer = GetMaxLangVersion(); NYson::EYsonFormat ResultsFormat = NYson::EYsonFormat::Text; ERunMode Mode = ERunMode::Run; diff --git a/yql/essentials/udfs/common/re2/re2_udf.cpp b/yql/essentials/udfs/common/re2/re2_udf.cpp index 965d24fe368..55c110b3458 100644 --- a/yql/essentials/udfs/common/re2/re2_udf.cpp +++ b/yql/essentials/udfs/common/re2/re2_udf.cpp @@ -1,8 +1,11 @@ +#include <yql/essentials/public/langver/yql_langver.h> #include <yql/essentials/public/udf/udf_helpers.h> +#include <yql/essentials/public/udf/udf_type_ops.h> #include <yql/essentials/public/udf/udf_value_builder.h> #include <contrib/libs/re2/re2/re2.h> +#include <util/system/env.h> #include <util/charset/utf8.h> #include <util/string/cast.h> @@ -36,7 +39,40 @@ namespace { xx(WordBoundary, 11, bool, false, set_word_boundary, Id) \ xx(OneLine, 12, bool, false, set_one_line, Id) - enum EOptionsField : ui32 { + ui64 GetFailProbability() { + auto envResult = TryGetEnv("YQL_RE2_REGEXP_PROBABILITY_FAIL"); + if (!envResult) { + return 0; + } + ui64 result; + bool isValid = TryIntFromString<10, ui64>(envResult->data(), envResult->size(), result); + Y_ENSURE(isValid, TStringBuilder() << "Error while parsing YQL_RE2_REGEXP_PROBABILITY_FAIL. Actual value is: " << *envResult); + return result; + } + + bool ShouldFailOnInvalidRegexp(const std::string_view regexp, NYql::TLangVersion currentLangVersion) { + if (currentLangVersion >= NYql::MakeLangVersion(2025, 3)) { + return true; + } + THashType hash = GetStringHash(regexp) % 100; + ui64 failProbability = GetFailProbability(); + return hash < failProbability; + } + + RE2::Options CreateDefaultOptions(){ + RE2::Options options; +#define FIELD_HANDLE(name, index, type, defVal, setter, conv) options.setter(conv(defVal)); + OPTIONS_MAP(FIELD_HANDLE) +#undef FIELD_HANDLE + options.set_log_errors(false); + return options; + } + + TString FormatRegexpError(const RE2& Regexp) { + return TStringBuilder() << "Regexp compilation failed. Regexp: \"" << Regexp.pattern() << "\". Original error is: \"" << Regexp.error() << "\""; + } + + enum EOptionsField: ui32 { OPTIONS_MAP(ENUM_VALUE_GEN) Count }; @@ -69,11 +105,13 @@ namespace { EMode mode, const TOptionsSchema& optionsSchema, TSourcePosition pos, + NYql::TLangVersion currentlangVersion, const TRegexpGroups& regexpGroups = TRegexpGroups()) : Mode(mode) , OptionsSchema(optionsSchema) , Pos_(pos) , RegexpGroups(regexpGroups) + , CurrentLangVersion(currentlangVersion) { } @@ -89,13 +127,15 @@ namespace { Mode, posix, OptionsSchema, - Pos_)); + Pos_, + CurrentLangVersion)); } EMode Mode; const TOptionsSchema OptionsSchema; TSourcePosition Pos_; const TRegexpGroups RegexpGroups; + NYql::TLangVersion CurrentLangVersion; }; static const TStringRef& Name(EMode mode) { @@ -130,18 +170,19 @@ namespace { EMode mode, bool posix, const TOptionsSchema& optionsSchema, - TSourcePosition pos) + TSourcePosition pos, + NYql::TLangVersion currentLangVersion) : RegexpGroups(regexpGroups) , Mode(mode) , Captured() , OptionsSchema(optionsSchema) , Pos_(pos) - { + , CurrentLangVersion(currentLangVersion) { try { auto patternValue = runConfig.GetElement(0); auto optionsValue = runConfig.GetElement(1); const std::string_view pattern(patternValue.AsStringRef()); - RE2::Options options; + RE2::Options options = CreateDefaultOptions(); options.set_posix_syntax(posix); bool needUtf8 = (UTF8Detect(pattern) == UTF8); @@ -154,10 +195,15 @@ namespace { #define FIELD_HANDLE(name, index, type, defVal, setter, conv) options.setter(conv(optionsValue.GetElement(OptionsSchema.Indices[index]).Get<type>())); OPTIONS_MAP(FIELD_HANDLE) #undef FIELD_HANDLE + options.set_log_errors(false); } Regexp = std::make_unique<RE2>(StringPiece(pattern.data(), pattern.size()), options); + if (!Regexp->ok() && ShouldFailOnInvalidRegexp(pattern, CurrentLangVersion)) { + throw yexception() << FormatRegexpError(*Regexp); + } + if (mode == EMode::CAPTURE) { Captured = std::make_unique<StringPiece[]>(Regexp->NumberOfCapturingGroups() + 1); } @@ -252,6 +298,7 @@ namespace { std::unique_ptr<StringPiece[]> Captured; const TOptionsSchema OptionsSchema; TSourcePosition Pos_; + NYql::TLangVersion CurrentLangVersion; TUnboxedValue BuildEmptyStruct(const IValueBuilder* valueBuilder) const { TUnboxedValue* items = nullptr; @@ -451,13 +498,18 @@ namespace { if (!typesOnly) { const auto mode = isMatch ? TRe2Udf::EMode::MATCH : TRe2Udf::EMode::GREP; - builder.Implementation(new TRe2Udf::TFactory<posix>(mode, optionsSchema, builder.GetSourcePosition())); + builder.Implementation(new TRe2Udf::TFactory<posix>(mode, optionsSchema, builder.GetSourcePosition(), builder.GetCurrentLangVer())); } } else if (isCapture) { TRegexpGroups groups; auto optionalStringType = builder.Optional()->Item<char*>().Build(); auto structBuilder = builder.Struct(); - RE2 regexp(StringPiece(typeConfig.Data(), typeConfig.Size())); + RE2::Options options = CreateDefaultOptions(); + RE2 regexp(StringPiece(typeConfig.Data(), typeConfig.Size()), options); + if (!regexp.ok()) { + builder.SetError(FormatRegexpError(regexp)); + return; + } const auto& groupNames = regexp.CapturingGroupNames(); int groupCount = regexp.NumberOfCapturingGroups(); if (groupCount >= 0) { @@ -487,35 +539,32 @@ namespace { if (!typesOnly) { builder.Implementation( - new TRe2Udf::TFactory<posix>(TRe2Udf::EMode::CAPTURE, optionsSchema, builder.GetSourcePosition(), groups)); + new TRe2Udf::TFactory<posix>(TRe2Udf::EMode::CAPTURE, optionsSchema, builder.GetSourcePosition(), builder.GetCurrentLangVer(), groups)); } } else { - if (regexp.ok()) { - builder.SetError("Regexp contains no capturing groups"); - } else { - builder.SetError(regexp.error()); - } + Y_ENSURE(regexp.ok()); + builder.SetError("Regexp contains no capturing groups"); } } else if (isReplace) { builder.SimpleSignature<TOptional<char*>(TOptional<char*>, char*)>() .RunConfig(MakeRunConfigType(builder, optOptionsStructType)); if (!typesOnly) { - builder.Implementation(new TRe2Udf::TFactory<posix>(TRe2Udf::EMode::REPLACE, optionsSchema, builder.GetSourcePosition())); + builder.Implementation(new TRe2Udf::TFactory<posix>(TRe2Udf::EMode::REPLACE, optionsSchema, builder.GetSourcePosition(), builder.GetCurrentLangVer())); } } else if (isCount) { builder.SimpleSignature<ui32(TOptional<char*>)>() .RunConfig(MakeRunConfigType(builder, optOptionsStructType)); if (!typesOnly) { - builder.Implementation(new TRe2Udf::TFactory<posix>(TRe2Udf::EMode::COUNT, optionsSchema, builder.GetSourcePosition())); + builder.Implementation(new TRe2Udf::TFactory<posix>(TRe2Udf::EMode::COUNT, optionsSchema, builder.GetSourcePosition(), builder.GetCurrentLangVer())); } } else if (isFindAndConsume) { builder.SimpleSignature<TListType<char*>(TOptional<char*>)>() .RunConfig(MakeRunConfigType(builder, optOptionsStructType)); if (!typesOnly) { - builder.Implementation(new TRe2Udf::TFactory<posix>(TRe2Udf::EMode::FIND_AND_CONSUME, optionsSchema, builder.GetSourcePosition())); + builder.Implementation(new TRe2Udf::TFactory<posix>(TRe2Udf::EMode::FIND_AND_CONSUME, optionsSchema, builder.GetSourcePosition(), builder.GetCurrentLangVer())); } } else if (!( TEscape::DeclareSignature(name, userType, builder, typesOnly) || diff --git a/yql/essentials/udfs/common/re2/test/canondata/result.json b/yql/essentials/udfs/common/re2/test/canondata/result.json index b9a16f32d66..75db00ebaf8 100644 --- a/yql/essentials/udfs/common/re2/test/canondata/result.json +++ b/yql/essentials/udfs/common/re2/test/canondata/result.json @@ -24,6 +24,26 @@ "uri": "file://test.test_FindAndConsumeEmpty_/results.txt" } ], + "test.test[InvalidCaptureRegexFail]": [ + { + "uri": "file://test.test_InvalidCaptureRegexFail_/extracted" + } + ], + "test.test[InvalidRegexFail_2025.02]": [ + { + "uri": "file://test.test_InvalidRegexFail_2025.02_/extracted" + } + ], + "test.test[InvalidRegexFail_2025.03]": [ + { + "uri": "file://test.test_InvalidRegexFail_2025.03_/extracted" + } + ], + "test.test[InvalidRegexSuccess_2025.02]": [ + { + "uri": "file://test.test_InvalidRegexSuccess_2025.02_/results.txt" + } + ], "test.test[LikeEscape]": [ { "uri": "file://test.test_LikeEscape_/results.txt" diff --git a/yql/essentials/udfs/common/re2/test/canondata/test.test_InvalidCaptureRegexFail_/extracted b/yql/essentials/udfs/common/re2/test/canondata/test.test_InvalidCaptureRegexFail_/extracted new file mode 100644 index 00000000000..f4d1c329f33 --- /dev/null +++ b/yql/essentials/udfs/common/re2/test/canondata/test.test_InvalidCaptureRegexFail_/extracted @@ -0,0 +1,14 @@ +<tmp_path>/program.sql:<main>: Error: Type annotation + + <tmp_path>/program.sql:<main>:8:1: Error: At function: RemovePrefixMembers, At function: Unordered, At function: PersistableRepr, At function: OrderedSqlProject, At tuple, At function: SqlProjectItem, At lambda + select $invalidCaptureRegexp("abc"); + ^ + <tmp_path>/program.sql:<main>:8:8: Error: At function: Apply + select $invalidCaptureRegexp("abc"); + ^ + <tmp_path>/program.sql:<main>:4:30: Error: At function: Udf, At Re2.Capture + $invalidCaptureRegexp = Re2::Capture("["); + ^ + <tmp_path>/program.sql:<main>:4:30: Error: Failed to find UDF function: Re2.Capture, reason: Error: Module: Re2, function: Capture, error: Regexp compilation failed. Regexp: "[". Original error is: "missing ]: [" + $invalidCaptureRegexp = Re2::Capture("["); + ^
\ No newline at end of file diff --git a/yql/essentials/udfs/common/re2/test/canondata/test.test_InvalidRegexFail_2025.02_/extracted b/yql/essentials/udfs/common/re2/test/canondata/test.test_InvalidRegexFail_2025.02_/extracted new file mode 100644 index 00000000000..f1fda30e02a --- /dev/null +++ b/yql/essentials/udfs/common/re2/test/canondata/test.test_InvalidRegexFail_2025.02_/extracted @@ -0,0 +1,8 @@ +<tmp_path>/program.sql:<main>: Error: Execution + + <tmp_path>/program.sql:<main>:8:1: Error: Execution of node: Result + SELECT $invalidRe("abaa"); + ^ + <tmp_path>/program.sql:<main>:4:19: Error: Regexp compilation failed. Regexp: "[". Original error is: "missing ]: [" + $invalidRe = Re2::FindAndConsume("["); + ^
\ No newline at end of file diff --git a/yql/essentials/udfs/common/re2/test/canondata/test.test_InvalidRegexFail_2025.03_/extracted b/yql/essentials/udfs/common/re2/test/canondata/test.test_InvalidRegexFail_2025.03_/extracted new file mode 100644 index 00000000000..f1fda30e02a --- /dev/null +++ b/yql/essentials/udfs/common/re2/test/canondata/test.test_InvalidRegexFail_2025.03_/extracted @@ -0,0 +1,8 @@ +<tmp_path>/program.sql:<main>: Error: Execution + + <tmp_path>/program.sql:<main>:8:1: Error: Execution of node: Result + SELECT $invalidRe("abaa"); + ^ + <tmp_path>/program.sql:<main>:4:19: Error: Regexp compilation failed. Regexp: "[". Original error is: "missing ]: [" + $invalidRe = Re2::FindAndConsume("["); + ^
\ No newline at end of file diff --git a/yql/essentials/udfs/common/re2/test/canondata/test.test_InvalidRegexSuccess_2025.02_/results.txt b/yql/essentials/udfs/common/re2/test/canondata/test.test_InvalidRegexSuccess_2025.02_/results.txt new file mode 100644 index 00000000000..c37c844a342 --- /dev/null +++ b/yql/essentials/udfs/common/re2/test/canondata/test.test_InvalidRegexSuccess_2025.02_/results.txt @@ -0,0 +1,31 @@ +[ + { + "Write" = [ + { + "Type" = [ + "ListType"; + [ + "StructType"; + [ + [ + "column0"; + [ + "ListType"; + [ + "DataType"; + "String" + ] + ] + ] + ] + ] + ]; + "Data" = [ + [ + [] + ] + ] + } + ] + } +]
\ No newline at end of file diff --git a/yql/essentials/udfs/common/re2/test/cases/InvalidCaptureRegexFail.cfg b/yql/essentials/udfs/common/re2/test/cases/InvalidCaptureRegexFail.cfg new file mode 100644 index 00000000000..0704b3634e0 --- /dev/null +++ b/yql/essentials/udfs/common/re2/test/cases/InvalidCaptureRegexFail.cfg @@ -0,0 +1,2 @@ +xfail +env YQL_RE2_REGEXP_PROBABILITY_FAIL 0 // Test that failure does not depend on this variable. diff --git a/yql/essentials/udfs/common/re2/test/cases/InvalidCaptureRegexFail.sql b/yql/essentials/udfs/common/re2/test/cases/InvalidCaptureRegexFail.sql new file mode 100644 index 00000000000..09b17677508 --- /dev/null +++ b/yql/essentials/udfs/common/re2/test/cases/InvalidCaptureRegexFail.sql @@ -0,0 +1,4 @@ +/* syntax version 1 */ +$invalidCaptureRegexp = Re2::Capture("["); + +select $invalidCaptureRegexp("abc"); diff --git a/yql/essentials/udfs/common/re2/test/cases/InvalidRegexFail_2025.02.cfg b/yql/essentials/udfs/common/re2/test/cases/InvalidRegexFail_2025.02.cfg new file mode 100644 index 00000000000..c0ef97affef --- /dev/null +++ b/yql/essentials/udfs/common/re2/test/cases/InvalidRegexFail_2025.02.cfg @@ -0,0 +1,3 @@ +xfail +env YQL_RE2_REGEXP_PROBABILITY_FAIL 100 +langver 2025.02 diff --git a/yql/essentials/udfs/common/re2/test/cases/InvalidRegexFail_2025.02.sql b/yql/essentials/udfs/common/re2/test/cases/InvalidRegexFail_2025.02.sql new file mode 100644 index 00000000000..087dcab2efd --- /dev/null +++ b/yql/essentials/udfs/common/re2/test/cases/InvalidRegexFail_2025.02.sql @@ -0,0 +1,4 @@ +/* syntax version 1 */ +$invalidRe = Re2::FindAndConsume("["); + +SELECT $invalidRe("abaa"); diff --git a/yql/essentials/udfs/common/re2/test/cases/InvalidRegexFail_2025.03.cfg b/yql/essentials/udfs/common/re2/test/cases/InvalidRegexFail_2025.03.cfg new file mode 100644 index 00000000000..e37bb6b41a2 --- /dev/null +++ b/yql/essentials/udfs/common/re2/test/cases/InvalidRegexFail_2025.03.cfg @@ -0,0 +1,2 @@ +langver 2025.03 +xfail diff --git a/yql/essentials/udfs/common/re2/test/cases/InvalidRegexFail_2025.03.sql b/yql/essentials/udfs/common/re2/test/cases/InvalidRegexFail_2025.03.sql new file mode 100644 index 00000000000..087dcab2efd --- /dev/null +++ b/yql/essentials/udfs/common/re2/test/cases/InvalidRegexFail_2025.03.sql @@ -0,0 +1,4 @@ +/* syntax version 1 */ +$invalidRe = Re2::FindAndConsume("["); + +SELECT $invalidRe("abaa"); diff --git a/yql/essentials/udfs/common/re2/test/cases/InvalidRegexSuccess_2025.02.cfg b/yql/essentials/udfs/common/re2/test/cases/InvalidRegexSuccess_2025.02.cfg new file mode 100644 index 00000000000..367bc6a9ec0 --- /dev/null +++ b/yql/essentials/udfs/common/re2/test/cases/InvalidRegexSuccess_2025.02.cfg @@ -0,0 +1 @@ +langver 2025.02 diff --git a/yql/essentials/udfs/common/re2/test/cases/InvalidRegexSuccess_2025.02.sql b/yql/essentials/udfs/common/re2/test/cases/InvalidRegexSuccess_2025.02.sql new file mode 100644 index 00000000000..087dcab2efd --- /dev/null +++ b/yql/essentials/udfs/common/re2/test/cases/InvalidRegexSuccess_2025.02.sql @@ -0,0 +1,4 @@ +/* syntax version 1 */ +$invalidRe = Re2::FindAndConsume("["); + +SELECT $invalidRe("abaa"); diff --git a/yql/essentials/udfs/common/re2/ya.make b/yql/essentials/udfs/common/re2/ya.make index 7e554133486..ca8be7370ba 100644 --- a/yql/essentials/udfs/common/re2/ya.make +++ b/yql/essentials/udfs/common/re2/ya.make @@ -2,7 +2,7 @@ YQL_UDF_CONTRIB(re2_udf) YQL_ABI_VERSION( 2 - 28 + 43 0 ) diff --git a/yql/essentials/udfs/common/string/test/canondata/result.json b/yql/essentials/udfs/common/string/test/canondata/result.json index aff883c405e..0feffac589c 100644 --- a/yql/essentials/udfs/common/string/test/canondata/result.json +++ b/yql/essentials/udfs/common/string/test/canondata/result.json @@ -14,6 +14,16 @@ "uri": "file://test.test_AsciiCmpIgnoreCase_2025_02_/extracted" } ], + "test.test[AsciiContainsIgnoreCase_2025_01]": [ + { + "uri": "file://test.test_AsciiContainsIgnoreCase_2025_01_/extracted" + } + ], + "test.test[AsciiContainsIgnoreCase_2025_01_scan]": [ + { + "uri": "file://test.test_AsciiContainsIgnoreCase_2025_01_scan_/extracted" + } + ], "test.test[Base32Decode]": [ { "uri": "file://test.test_Base32Decode_/results.txt" diff --git a/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiContainsIgnoreCase_2025_01_/extracted b/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiContainsIgnoreCase_2025_01_/extracted new file mode 100644 index 00000000000..6c66d69cd7f --- /dev/null +++ b/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiContainsIgnoreCase_2025_01_/extracted @@ -0,0 +1,50 @@ +<tmp_path>/program.sql:<main>: Error: Type annotation + + <tmp_path>/program.sql:<main>:2:1: Error: At function: RemovePrefixMembers, At function: Unordered, At function: PersistableRepr, At function: OrderedSqlProject, At tuple + SELECT + ^ + <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda + SELECT + ^ + <tmp_path>/program.sql:<main>:6:13: Error: At function: Apply, At function: Udf + String::AsciiContainsIgnoreCase(value, "AS") AS iccontains, + ^ + <tmp_path>/program.sql:<main>:6:13: Error: UDF 'String.AsciiContainsIgnoreCase' is not available before version 2025.02 + String::AsciiContainsIgnoreCase(value, "AS") AS iccontains, + ^ + <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda + SELECT + ^ + <tmp_path>/program.sql:<main>:8:13: Error: At function: Apply, At function: Udf + String::AsciiContainsIgnoreCase(value, "") AS icempty, + ^ + <tmp_path>/program.sql:<main>:8:13: Error: UDF 'String.AsciiContainsIgnoreCase' is not available before version 2025.02 + String::AsciiContainsIgnoreCase(value, "") AS icempty, + ^ + <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda + SELECT + ^ + <tmp_path>/program.sql:<main>:10:13: Error: At function: Apply, At function: Udf + String::AsciiStartsWithIgnoreCase(value, "AS") AS icstarts, + ^ + <tmp_path>/program.sql:<main>:10:13: Error: UDF 'String.AsciiStartsWithIgnoreCase' is not available before version 2025.02 + String::AsciiStartsWithIgnoreCase(value, "AS") AS icstarts, + ^ + <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda + SELECT + ^ + <tmp_path>/program.sql:<main>:12:13: Error: At function: Apply, At function: Udf + String::AsciiEndsWithIgnoreCase(value, "AS") AS icends, + ^ + <tmp_path>/program.sql:<main>:12:13: Error: UDF 'String.AsciiEndsWithIgnoreCase' is not available before version 2025.02 + String::AsciiEndsWithIgnoreCase(value, "AS") AS icends, + ^ + <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda + SELECT + ^ + <tmp_path>/program.sql:<main>:14:13: Error: At function: Apply, At function: Udf + String::AsciiEqualsIgnoreCase(value, "FDSA") AS icequals, + ^ + <tmp_path>/program.sql:<main>:14:13: Error: UDF 'String.AsciiEqualsIgnoreCase' is not available before version 2025.02 + String::AsciiEqualsIgnoreCase(value, "FDSA") AS icequals, + ^
\ No newline at end of file diff --git a/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiContainsIgnoreCase_2025_01_scan_/extracted b/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiContainsIgnoreCase_2025_01_scan_/extracted new file mode 100644 index 00000000000..6c66d69cd7f --- /dev/null +++ b/yql/essentials/udfs/common/string/test/canondata/test.test_AsciiContainsIgnoreCase_2025_01_scan_/extracted @@ -0,0 +1,50 @@ +<tmp_path>/program.sql:<main>: Error: Type annotation + + <tmp_path>/program.sql:<main>:2:1: Error: At function: RemovePrefixMembers, At function: Unordered, At function: PersistableRepr, At function: OrderedSqlProject, At tuple + SELECT + ^ + <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda + SELECT + ^ + <tmp_path>/program.sql:<main>:6:13: Error: At function: Apply, At function: Udf + String::AsciiContainsIgnoreCase(value, "AS") AS iccontains, + ^ + <tmp_path>/program.sql:<main>:6:13: Error: UDF 'String.AsciiContainsIgnoreCase' is not available before version 2025.02 + String::AsciiContainsIgnoreCase(value, "AS") AS iccontains, + ^ + <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda + SELECT + ^ + <tmp_path>/program.sql:<main>:8:13: Error: At function: Apply, At function: Udf + String::AsciiContainsIgnoreCase(value, "") AS icempty, + ^ + <tmp_path>/program.sql:<main>:8:13: Error: UDF 'String.AsciiContainsIgnoreCase' is not available before version 2025.02 + String::AsciiContainsIgnoreCase(value, "") AS icempty, + ^ + <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda + SELECT + ^ + <tmp_path>/program.sql:<main>:10:13: Error: At function: Apply, At function: Udf + String::AsciiStartsWithIgnoreCase(value, "AS") AS icstarts, + ^ + <tmp_path>/program.sql:<main>:10:13: Error: UDF 'String.AsciiStartsWithIgnoreCase' is not available before version 2025.02 + String::AsciiStartsWithIgnoreCase(value, "AS") AS icstarts, + ^ + <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda + SELECT + ^ + <tmp_path>/program.sql:<main>:12:13: Error: At function: Apply, At function: Udf + String::AsciiEndsWithIgnoreCase(value, "AS") AS icends, + ^ + <tmp_path>/program.sql:<main>:12:13: Error: UDF 'String.AsciiEndsWithIgnoreCase' is not available before version 2025.02 + String::AsciiEndsWithIgnoreCase(value, "AS") AS icends, + ^ + <tmp_path>/program.sql:<main>:2:1: Error: At function: SqlProjectItem, At lambda + SELECT + ^ + <tmp_path>/program.sql:<main>:14:13: Error: At function: Apply, At function: Udf + String::AsciiEqualsIgnoreCase(value, "FDSA") AS icequals, + ^ + <tmp_path>/program.sql:<main>:14:13: Error: UDF 'String.AsciiEqualsIgnoreCase' is not available before version 2025.02 + String::AsciiEqualsIgnoreCase(value, "FDSA") AS icequals, + ^
\ No newline at end of file diff --git a/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01.cfg b/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01.cfg new file mode 100644 index 00000000000..63fef4f7b7c --- /dev/null +++ b/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01.cfg @@ -0,0 +1,3 @@ +xfail +langver 2025.01 +in plato.Input default.in diff --git a/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01.sql b/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01.sql new file mode 100644 index 00000000000..db2ebad20e0 --- /dev/null +++ b/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01.sql @@ -0,0 +1,9 @@ +SELECT + value, + String::AsciiContainsIgnoreCase(value, "AS") AS iccontains, + String::AsciiContainsIgnoreCase(value, "") AS icempty, + String::AsciiStartsWithIgnoreCase(value, "AS") AS icstarts, + String::AsciiEndsWithIgnoreCase(value, "AS") AS icends, + String::AsciiEqualsIgnoreCase(value, "FDSA") AS icequals, +FROM Input; + diff --git a/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01_scan.cfg b/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01_scan.cfg new file mode 100644 index 00000000000..ea25436b9bb --- /dev/null +++ b/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01_scan.cfg @@ -0,0 +1,4 @@ +xfail +langver 2025.01 +scan_udfs +in plato.Input default.in diff --git a/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01_scan.sql b/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01_scan.sql new file mode 100644 index 00000000000..db2ebad20e0 --- /dev/null +++ b/yql/essentials/udfs/common/string/test/cases/AsciiContainsIgnoreCase_2025_01_scan.sql @@ -0,0 +1,9 @@ +SELECT + value, + String::AsciiContainsIgnoreCase(value, "AS") AS iccontains, + String::AsciiContainsIgnoreCase(value, "") AS icempty, + String::AsciiStartsWithIgnoreCase(value, "AS") AS icstarts, + String::AsciiEndsWithIgnoreCase(value, "AS") AS icends, + String::AsciiEqualsIgnoreCase(value, "FDSA") AS icequals, +FROM Input; + diff --git a/yql/essentials/udfs/common/string/test/cases/BlockReverseBits.cfg b/yql/essentials/udfs/common/string/test/cases/BlockReverseBits.cfg new file mode 100644 index 00000000000..12ab49f0ffc --- /dev/null +++ b/yql/essentials/udfs/common/string/test/cases/BlockReverseBits.cfg @@ -0,0 +1,2 @@ +langver 2025.02 +in plato.Input BlockReverseBits.in diff --git a/yql/essentials/udfs/common/string/test/cases/BlockReverseBytes.cfg b/yql/essentials/udfs/common/string/test/cases/BlockReverseBytes.cfg new file mode 100644 index 00000000000..86f08753875 --- /dev/null +++ b/yql/essentials/udfs/common/string/test/cases/BlockReverseBytes.cfg @@ -0,0 +1,2 @@ +langver 2025.02 +in plato.Input BlockReverseBytes.in diff --git a/yql/essentials/udfs/common/string/test/cases/ReverseBits.cfg b/yql/essentials/udfs/common/string/test/cases/ReverseBits.cfg new file mode 100644 index 00000000000..a431901cef4 --- /dev/null +++ b/yql/essentials/udfs/common/string/test/cases/ReverseBits.cfg @@ -0,0 +1,2 @@ +langver 2025.02 +in plato.Input ReverseBits.in diff --git a/yql/essentials/udfs/common/string/test/cases/ReverseBytes.cfg b/yql/essentials/udfs/common/string/test/cases/ReverseBytes.cfg new file mode 100644 index 00000000000..f4b5316f29c --- /dev/null +++ b/yql/essentials/udfs/common/string/test/cases/ReverseBytes.cfg @@ -0,0 +1,2 @@ +langver 2025.02 +in plato.Input ReverseBytes.in diff --git a/yt/cpp/mapreduce/client/client.cpp b/yt/cpp/mapreduce/client/client.cpp index 8977a29b106..067750cd5c4 100644 --- a/yt/cpp/mapreduce/client/client.cpp +++ b/yt/cpp/mapreduce/client/client.cpp @@ -1613,6 +1613,7 @@ TClientContext CreateClientContext( context.TvmOnly = options.TvmOnly_; context.ProxyAddress = options.ProxyAddress_; context.UseProxyUnixDomainSocket = options.UseProxyUnixDomainSocket_; + context.MultiproxyTargetCluster = options.MultiproxyTargetCluster_; if (options.UseTLS_) { context.UseTLS = *options.UseTLS_; diff --git a/yt/cpp/mapreduce/http/context.h b/yt/cpp/mapreduce/http/context.h index 85bed9a030a..d5f07ee2283 100644 --- a/yt/cpp/mapreduce/http/context.h +++ b/yt/cpp/mapreduce/http/context.h @@ -24,6 +24,7 @@ struct TClientContext TMaybe<TString> ProxyAddress; TMaybe<TString> RpcProxyRole; bool UseProxyUnixDomainSocket = false; + TMaybe<TString> MultiproxyTargetCluster; }; bool operator==(const TClientContext& lhs, const TClientContext& rhs); diff --git a/yt/cpp/mapreduce/interface/client_method_options.h b/yt/cpp/mapreduce/interface/client_method_options.h index d43020a9e13..10c200715df 100644 --- a/yt/cpp/mapreduce/interface/client_method_options.h +++ b/yt/cpp/mapreduce/interface/client_method_options.h @@ -1136,6 +1136,13 @@ struct TCreateClientOptions /// @brief Use unix domain socket for connection. /// Typically you will need this option when the RPC proxy is enabled within the job proxy. FLUENT_FIELD_DEFAULT(bool, UseProxyUnixDomainSocket, false); + + /// @brief Defines which cluster should handle incoming RPC requests. + /// + /// @note Multiproxy mode must be activated on the server side. + /// This mode allows a proxy from one cluster to forward requests to another. + /// Availability and specific usage might vary based on server configuration. + FLUENT_FIELD_OPTION(TString, MultiproxyTargetCluster); }; /// diff --git a/yt/yql/providers/yt/common/yql_configuration.h b/yt/yql/providers/yt/common/yql_configuration.h index ad54c9b376f..d323b48efe1 100644 --- a/yt/yql/providers/yt/common/yql_configuration.h +++ b/yt/yql/providers/yt/common/yql_configuration.h @@ -16,6 +16,8 @@ constexpr size_t YQL_JOB_CODEC_BLOCK_SIZE = 1_MB; constexpr size_t YQL_JOB_CODEC_MEM = YQL_JOB_CODEC_BLOCK_COUNT * YQL_JOB_CODEC_BLOCK_SIZE + (30_MB); +constexpr size_t YQL_ARROW_MEMORY_POOL_RESERVE = 640_MB; + constexpr ui64 DEFAULT_TOP_SORT_LIMIT = 1000ULL; constexpr ui16 DEFAULT_WIDE_FLOW_LIMIT = 101U; diff --git a/yt/yql/providers/yt/common/yql_yt_settings.cpp b/yt/yql/providers/yt/common/yql_yt_settings.cpp index 763793790f5..ab90b5f3cf1 100644 --- a/yt/yql/providers/yt/common/yql_yt_settings.cpp +++ b/yt/yql/providers/yt/common/yql_yt_settings.cpp @@ -471,6 +471,7 @@ TYtConfiguration::TYtConfiguration(TTypeAnnotationContext& typeCtx) REGISTER_SETTING(*this, UsePartitionsByKeysForFinalAgg); REGISTER_SETTING(*this, ForceJobSizeAdjuster); REGISTER_SETTING(*this, EnforceJobUtc); + REGISTER_SETTING(*this, _EnforceRegexpProbabilityFail); REGISTER_SETTING(*this, UseRPCReaderInDQ); REGISTER_SETTING(*this, DQRPCReaderInflight).Lower(1); REGISTER_SETTING(*this, DQRPCReaderTimeout); diff --git a/yt/yql/providers/yt/common/yql_yt_settings.h b/yt/yql/providers/yt/common/yql_yt_settings.h index 5cde259c974..2505a22f034 100644 --- a/yt/yql/providers/yt/common/yql_yt_settings.h +++ b/yt/yql/providers/yt/common/yql_yt_settings.h @@ -241,6 +241,7 @@ public: NCommon::TConfSetting<bool, Dynamic> _EnableYtPartitioning; NCommon::TConfSetting<bool, Dynamic> ForceJobSizeAdjuster; NCommon::TConfSetting<bool, Dynamic> EnforceJobUtc; + NCommon::TConfSetting<ui64, Static> _EnforceRegexpProbabilityFail; NCommon::TConfSetting<bool, Dynamic> UseRPCReaderInDQ; NCommon::TConfSetting<size_t, Dynamic> DQRPCReaderInflight; NCommon::TConfSetting<TDuration, Dynamic> DQRPCReaderTimeout; diff --git a/yt/yql/providers/yt/fmr/coordinator/impl/default_coordinator_settings.yson b/yt/yql/providers/yt/fmr/coordinator/impl/default_coordinator_settings.yson index 203b3d1ce66..ef865b7f5f7 100644 --- a/yt/yql/providers/yt/fmr/coordinator/impl/default_coordinator_settings.yson +++ b/yt/yql/providers/yt/fmr/coordinator/impl/default_coordinator_settings.yson @@ -23,4 +23,14 @@ "max_row_weight" = 16777216; }; }; -}
\ No newline at end of file + "partition" = { + "yt_table" = { + "max_data_weight_per_part" = 104857600; + "max_parts" = 100; + }; + "fmr_table" = { + "max_data_weight_per_part" = 104857600; + "max_parts" = 100; + }; + } +} diff --git a/yt/yql/providers/yt/fmr/coordinator/impl/ut/ya.make b/yt/yql/providers/yt/fmr/coordinator/impl/ut/ya.make index 2f67f6d05ef..179b3e230f0 100644 --- a/yt/yql/providers/yt/fmr/coordinator/impl/ut/ya.make +++ b/yt/yql/providers/yt/fmr/coordinator/impl/ut/ya.make @@ -2,10 +2,12 @@ UNITTEST() SRCS( yql_yt_coordinator_ut.cpp + yql_yt_partitioner_ut.cpp ) PEERDIR( yt/yql/providers/yt/fmr/coordinator/impl + yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file yt/yql/providers/yt/fmr/job_factory/impl yt/yql/providers/yt/fmr/worker/impl ) diff --git a/yt/yql/providers/yt/fmr/coordinator/impl/ut/yql_yt_coordinator_ut.cpp b/yt/yql/providers/yt/fmr/coordinator/impl/ut/yql_yt_coordinator_ut.cpp index 0de9cb8fda0..ae3505fdd7b 100644 --- a/yt/yql/providers/yt/fmr/coordinator/impl/ut/yql_yt_coordinator_ut.cpp +++ b/yt/yql/providers/yt/fmr/coordinator/impl/ut/yql_yt_coordinator_ut.cpp @@ -7,6 +7,7 @@ #include <yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.h> #include <yt/yql/providers/yt/fmr/job_factory/impl/yql_yt_job_factory_impl.h> #include <yt/yql/providers/yt/fmr/worker/impl/yql_yt_worker_impl.h> +#include <yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/yql_yt_file_coordinator_service.h> namespace NYql::NFmr { @@ -45,10 +46,12 @@ private: TDownloadOperationParams downloadOperationParams{ - .Input = TYtTableRef{"Path","Cluster"}, + .Input = TYtTableRef{.Path = "Path", .Cluster = "Cluster", .FilePath = "File_path"}, .Output = TFmrTableRef{{"TestCluster", "TestPath"}} }; +// TODO - создать общий файл на все тесты, наполнить его чем-то + TStartOperationRequest CreateOperationRequest(ETaskType taskType = ETaskType::Download, TOperationParams operationParams = downloadOperationParams) { return TStartOperationRequest{ .TaskType = taskType, @@ -83,13 +86,13 @@ auto defaultTaskFunction = [] (TTask::TPtr /*task*/, std::shared_ptr<std::atomic Y_UNIT_TEST_SUITE(FmrCoordinatorTests) { Y_UNIT_TEST(StartOperation) { - auto coordinator = MakeFmrCoordinator(); + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); auto startOperationResponse = coordinator->StartOperation(CreateOperationRequest()).GetValueSync(); auto status = startOperationResponse.Status; UNIT_ASSERT_VALUES_EQUAL(status, EOperationStatus::Accepted); } Y_UNIT_TEST(RetryAcceptedOperation) { - auto coordinator = MakeFmrCoordinator(); + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); auto downloadRequest = CreateOperationRequest(); auto firstResponse = coordinator->StartOperation(downloadRequest).GetValueSync(); auto firstOperationId = firstResponse.OperationId; @@ -102,13 +105,13 @@ Y_UNIT_TEST_SUITE(FmrCoordinatorTests) { } Y_UNIT_TEST(DeleteNonexistentOperation) { - auto coordinator = MakeFmrCoordinator(); + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); auto deleteOperationResponse = coordinator->DeleteOperation({"delete_operation_id"}).GetValueSync(); EOperationStatus status = deleteOperationResponse.Status; UNIT_ASSERT_VALUES_EQUAL(status, EOperationStatus::NotFound); } Y_UNIT_TEST(DeleteOperationBeforeSendToWorker) { - auto coordinator = MakeFmrCoordinator(); + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); auto startOperationResponse = coordinator->StartOperation(CreateOperationRequest()).GetValueSync(); TString operationId = startOperationResponse.OperationId; auto deleteOperationResponse = coordinator->DeleteOperation({operationId}).GetValueSync(); @@ -116,13 +119,13 @@ Y_UNIT_TEST_SUITE(FmrCoordinatorTests) { UNIT_ASSERT_VALUES_EQUAL(status, EOperationStatus::Aborted); } Y_UNIT_TEST(GetNonexistentOperation) { - auto coordinator = MakeFmrCoordinator(); + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); auto getOperationResponse = coordinator->GetOperation({"get_operation_id"}).GetValueSync(); EOperationStatus status = getOperationResponse.Status; UNIT_ASSERT_VALUES_EQUAL(status, EOperationStatus::NotFound); } Y_UNIT_TEST(GetAcceptedOperationStatus) { - auto coordinator = MakeFmrCoordinator(); + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); auto startOperationResponse = coordinator->StartOperation(CreateOperationRequest()).GetValueSync(); TString operationId = startOperationResponse.OperationId; auto getOperationResponse = coordinator->GetOperation({operationId}).GetValueSync(); @@ -130,7 +133,7 @@ Y_UNIT_TEST_SUITE(FmrCoordinatorTests) { UNIT_ASSERT_VALUES_EQUAL(status, EOperationStatus::Accepted); } Y_UNIT_TEST(GetRunningOperationStatus) { - auto coordinator = MakeFmrCoordinator(); + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); auto startOperationResponse = coordinator->StartOperation(CreateOperationRequest()).GetValueSync(); TString operationId = startOperationResponse.OperationId; @@ -145,7 +148,7 @@ Y_UNIT_TEST_SUITE(FmrCoordinatorTests) { UNIT_ASSERT_VALUES_EQUAL(status, EOperationStatus::InProgress); } Y_UNIT_TEST(GetCompletedOperationStatuses) { - auto coordinator = MakeFmrCoordinator(); + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); auto startOperationRequests = CreateSeveralOperationRequests(); std::vector<TString> operationIds; for (auto& request: startOperationRequests) { @@ -165,34 +168,27 @@ Y_UNIT_TEST_SUITE(FmrCoordinatorTests) { } } Y_UNIT_TEST(GetCompletedAndFailedOperationStatuses) { - auto coordinator = MakeFmrCoordinator(); + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); auto downloadOperationRequests = CreateSeveralOperationRequests(); std::vector<TString> downloadOperationIds; for (auto& request: downloadOperationRequests) { auto startOperationResponse = coordinator->StartOperation(request).GetValueSync(); downloadOperationIds.emplace_back(startOperationResponse.OperationId); } - auto uploadOperationRequest = CreateOperationRequest(ETaskType::Upload, TUploadOperationParams{ - {{"Cluster", "Path"}}, - {} + auto badDownloadRequest = CreateOperationRequest(ETaskType::Download, TDownloadOperationParams{ + .Input = TYtTableRef{.Path = "bad_path", .Cluster = "bad_cluster", .FilePath = "bad_file_path"}, + .Output = TFmrTableRef{{"bad_cluster", "bad_path"}} }); - auto uploadOperationResponse = coordinator->StartOperation(uploadOperationRequest).GetValueSync(); - auto uploadOperationId = uploadOperationResponse.OperationId; + auto badDownloadOperationResponse = coordinator->StartOperation(badDownloadRequest).GetValueSync(); + auto badDownloadOperationId = badDownloadOperationResponse.OperationId; auto func = [&] (TTask::TPtr task, std::shared_ptr<std::atomic<bool>> cancelFlag) { while (! cancelFlag->load()) { Sleep(TDuration::Seconds(1)); - ETaskStatus taskStatus = std::visit([] (auto&& taskParams) { - using T = std::decay_t<decltype(taskParams)>; - if constexpr (std::is_same_v<T, TUploadTaskParams>) { - return ETaskStatus::Failed; - } - return ETaskStatus::Completed; - }, task->TaskParams); - if (taskStatus == ETaskStatus::Failed) { + TDownloadTaskParams downloadTaskParams = std::get<TDownloadTaskParams>(task->TaskParams); + if (downloadTaskParams.Output.TableId.Contains("bad_path")) { return TJobResult{.TaskStatus = ETaskStatus::Failed, .Stats = TStatistics()}; } - Sleep(TDuration::Seconds(1)); return TJobResult{.TaskStatus = ETaskStatus::Completed, .Stats = TStatistics()}; } return TJobResult{.TaskStatus = ETaskStatus::Failed, .Stats = TStatistics()}; @@ -210,12 +206,12 @@ Y_UNIT_TEST_SUITE(FmrCoordinatorTests) { EOperationStatus status = getDownloadOperationResponse.Status; UNIT_ASSERT_VALUES_EQUAL(status, EOperationStatus::Completed); } - auto getUploadOperationResponse = coordinator->GetOperation({uploadOperationId}).GetValueSync(); - EOperationStatus uploadStatus = getUploadOperationResponse.Status; - UNIT_ASSERT_VALUES_EQUAL(uploadStatus, EOperationStatus::Failed); + auto getBadDownloadOperationResponse = coordinator->GetOperation({badDownloadOperationId}).GetValueSync(); + EOperationStatus badDownloadStatus = getBadDownloadOperationResponse.Status; + UNIT_ASSERT_VALUES_EQUAL(badDownloadStatus, EOperationStatus::Failed); } Y_UNIT_TEST(RetryRunningOperation) { - auto coordinator = MakeFmrCoordinator(); + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); auto downloadRequest = CreateOperationRequest(); auto startOperationResponse = coordinator->StartOperation(downloadRequest).GetValueSync(); TString firstOperationId = startOperationResponse.OperationId; @@ -236,7 +232,7 @@ Y_UNIT_TEST_SUITE(FmrCoordinatorTests) { Y_UNIT_TEST(RetryRunningOperationAfterIdempotencyKeyClear) { auto coordinatorSettings = TFmrCoordinatorSettings(); coordinatorSettings.IdempotencyKeyStoreTime = TDuration::Seconds(1); - auto coordinator = MakeFmrCoordinator(coordinatorSettings); + auto coordinator = MakeFmrCoordinator(coordinatorSettings, MakeFileYtCoordinatorService()); TFmrJobFactorySettings settings{.NumThreads = 3, .Function = defaultTaskFunction}; auto factory = MakeFmrJobFactory(settings); @@ -260,7 +256,7 @@ Y_UNIT_TEST_SUITE(FmrCoordinatorTests) { UNIT_ASSERT_VALUES_EQUAL(secondOperationStatus, EOperationStatus::Accepted); } Y_UNIT_TEST(CancelTasksAfterVolatileIdReload) { - auto coordinator = MakeFmrCoordinator(); + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); auto func = [&] (TTask::TPtr /*task*/, std::shared_ptr<std::atomic<bool>> cancelFlag) { int numIterations = 0; while (!cancelFlag->load()) { @@ -292,7 +288,7 @@ Y_UNIT_TEST_SUITE(FmrCoordinatorTests) { UNIT_ASSERT_NO_DIFF(*error.OperationId, operationId); } Y_UNIT_TEST(HandleJobErrors) { - auto coordinator = MakeFmrCoordinator(); + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); auto startOperationResponse = coordinator->StartOperation(CreateOperationRequest()).GetValueSync(); TString operationId = startOperationResponse.OperationId; @@ -323,13 +319,21 @@ Y_UNIT_TEST_SUITE(FmrCoordinatorTests) { } Y_UNIT_TEST(GetFmrTableInfo) { - auto coordinator = MakeFmrCoordinator(); - TTableStats tableStats = {.Chunks = 1, .Rows = 2, .DataWeight = 3}; - TString tableId = "test_table"; - TFmrTableOutputRef fmrTableOutputRef{.TableId = tableId, .PartId = "test_part_id"}; - std::unordered_map<TFmrTableOutputRef, TTableStats> outputTables{{fmrTableOutputRef, tableStats}}; - auto func = [&] (TTask::TPtr /*task*/, std::shared_ptr<std::atomic<bool>> cancelFlag) { + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); + ui64 totalChunkCount = 10, chunkRowCount = 1, chunkDataWeight = 2; + TString tableId = "TestCluster.TestPath"; // corresponds to CreateOperationRequest() + auto func = [&] (TTask::TPtr task, std::shared_ptr<std::atomic<bool>> cancelFlag) { while (!cancelFlag->load()) { + Sleep(TDuration::Seconds(1)); + TDownloadTaskParams downloadTaskParams = std::get<TDownloadTaskParams>(task->TaskParams); + TString partId = downloadTaskParams.Output.PartId; + TFmrTableOutputRef fmrTableOutputRef{.TableId = tableId, .PartId = partId}; + TTableChunkStats tableChunkStats{ + .PartId = partId, + .PartIdChunkStats = std::vector<TChunkStats>(totalChunkCount, TChunkStats{.Rows = chunkRowCount, .DataWeight = chunkDataWeight}) + }; + std::unordered_map<TFmrTableOutputRef, TTableChunkStats> outputTables{{fmrTableOutputRef, tableChunkStats}}; + return TJobResult{.TaskStatus = ETaskStatus::Completed, .Stats = TStatistics{ .OutputTables = outputTables }}; @@ -346,9 +350,9 @@ Y_UNIT_TEST_SUITE(FmrCoordinatorTests) { Sleep(TDuration::Seconds(3)); auto response = coordinator->GetFmrTableInfo({tableId}).GetValueSync(); worker->Stop(); - UNIT_ASSERT_VALUES_EQUAL(response.TableStats.Chunks, tableStats.Chunks); - UNIT_ASSERT_VALUES_EQUAL(response.TableStats.Rows, tableStats.Rows); - UNIT_ASSERT_VALUES_EQUAL(response.TableStats.DataWeight, tableStats.DataWeight); + UNIT_ASSERT_VALUES_EQUAL(response.TableStats.Chunks, totalChunkCount); + UNIT_ASSERT_VALUES_EQUAL(response.TableStats.Rows, totalChunkCount * chunkRowCount); + UNIT_ASSERT_VALUES_EQUAL(response.TableStats.DataWeight, totalChunkCount * chunkDataWeight); } } diff --git a/yt/yql/providers/yt/fmr/coordinator/impl/ut/yql_yt_partitioner_ut.cpp b/yt/yql/providers/yt/fmr/coordinator/impl/ut/yql_yt_partitioner_ut.cpp new file mode 100644 index 00000000000..87d491afd60 --- /dev/null +++ b/yt/yql/providers/yt/fmr/coordinator/impl/ut/yql_yt_partitioner_ut.cpp @@ -0,0 +1,220 @@ +#include <library/cpp/testing/unittest/registar.h> +#include <yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_partitioner.h> + +namespace NYql::NFmr { + +const TString FirstPartId = "test_part_id_0", SecondPartId = "test_part_id_1"; + +std::unordered_map<TFmrTableId, std::vector<TString>> GetTestPartIdsForTable(const TFmrTableId& fmrId) { + return std::unordered_map<TFmrTableId, std::vector<TString>>{{fmrId, std::vector<TString>{FirstPartId, SecondPartId}}}; +} + +std::unordered_map<TString, std::vector<TChunkStats>> GetTestPartIdStats() { + const std::vector<TChunkStats> firstPartitionChunkStats{ + TChunkStats{.DataWeight = 30}, + TChunkStats{.DataWeight = 30}, + TChunkStats{.DataWeight = 10}, + TChunkStats{.DataWeight = 20}, + }; + const std::vector<TChunkStats> secondPartitionChunkStats{ + TChunkStats{.DataWeight = 40}, + TChunkStats{.DataWeight = 15}, + }; + + return std::unordered_map<TString, std::vector<TChunkStats>>{ + {FirstPartId, firstPartitionChunkStats}, + {SecondPartId, secondPartitionChunkStats} + }; +} + +const TString FirstTablePartId = "first_table_part_id", SecondTablePartId = "sec_table_part_id", ThirdTablePartId = "third_table_part_id"; + +std::unordered_map<TFmrTableId, std::vector<TString>> GetTestPartIdsForMultipleTables(std::vector<TFmrTableId>& fmrTableIds) { + UNIT_ASSERT_VALUES_EQUAL(fmrTableIds.size(), 3); + return std::unordered_map<TFmrTableId, std::vector<TString>>{ + {fmrTableIds[0], std::vector<TString>{FirstTablePartId}}, + {fmrTableIds[1], std::vector<TString>{SecondTablePartId}}, + {fmrTableIds[2], std::vector<TString>{ThirdTablePartId}} + }; +} + +std::unordered_map<TString, std::vector<TChunkStats>> GetTestPartIdStatsForMultipleTables() { + const std::vector<TChunkStats> firstPartitionChunkStats{ + TChunkStats{.DataWeight = 40}, + TChunkStats{.DataWeight = 20}, + }; + const std::vector<TChunkStats> secondPartitionChunkStats{ + TChunkStats{.DataWeight = 40}, + TChunkStats{.DataWeight = 15}, + TChunkStats{.DataWeight = 5}, + }; + + const std::vector<TChunkStats> thirdPartitionChunkStats{ + TChunkStats{.DataWeight = 20}, + TChunkStats{.DataWeight = 30}, + TChunkStats{.DataWeight = 60}, + }; + + return std::unordered_map<TString, std::vector<TChunkStats>>{ + {FirstTablePartId, firstPartitionChunkStats}, + {SecondTablePartId, secondPartitionChunkStats}, + {ThirdTablePartId, thirdPartitionChunkStats} + }; +} + +std::vector<std::vector<TFmrTableInputRef>> ChangeGottenTasksFormat(const std::vector<TTaskTableInputRef>& inputTasks) { + // needed for testing so we can check resulting vectors for equality. + std::vector<std::vector<TFmrTableInputRef>> resultTasks; + for (auto& task: inputTasks) { + std::vector<TFmrTableInputRef> curTask; + std::transform(task.Inputs.begin(),task.Inputs.end(), std::back_inserter(curTask), [](const TTaskTableRef& tablePart){ + return std::get<TFmrTableInputRef>(tablePart); + }); + resultTasks.emplace_back(curTask); + } + return resultTasks; +} + +Y_UNIT_TEST_SUITE(PartitionerTests) { + Y_UNIT_TEST(PartitionFmrTable) { + auto fmrTableId = TFmrTableId("test_cluster", "test_path"); + TFmrTableRef fmrTable = TFmrTableRef{fmrTableId}; + + auto partIdsForTables = GetTestPartIdsForTable(fmrTableId); + auto partIdStats = GetTestPartIdStats(); + TFmrPartitionerSettings settings{.MaxDataWeightPerPart = 50, .MaxParts = 100}; + TFmrPartitioner partitioner(partIdsForTables, partIdStats, settings); + + auto [gottenTasks, status] = partitioner.PartitionFmrTablesIntoTasks({fmrTable}); + UNIT_ASSERT_VALUES_EQUAL(status, true); + + std::vector<std::vector<TFmrTableInputRef>> expectedTasks = { + {TFmrTableInputRef{ + .TableId = fmrTableId.Id, + .TableRanges = { + TTableRange{.PartId = FirstPartId, .MinChunk = 0, .MaxChunk = 1} + } + }}, + {TFmrTableInputRef{ + .TableId = fmrTableId.Id, + .TableRanges = { + TTableRange{.PartId = FirstPartId, .MinChunk = 1, .MaxChunk = 3} + } + }}, + {TFmrTableInputRef{ + .TableId = fmrTableId.Id, + .TableRanges = { + TTableRange{.PartId = SecondPartId, .MinChunk = 0, .MaxChunk = 1} + } + }}, + {TFmrTableInputRef{ + .TableId = fmrTableId.Id, + .TableRanges = { + TTableRange{.PartId = FirstPartId, .MinChunk = 3, .MaxChunk = 4}, + TTableRange{.PartId = SecondPartId, .MinChunk = 1, .MaxChunk = 2}, + } + }}, + }; + UNIT_ASSERT_VALUES_EQUAL(ChangeGottenTasksFormat(gottenTasks), expectedTasks); + } + Y_UNIT_TEST(MaxPartsNumExceeded) { + auto fmrTableId = TFmrTableId("test_cluster", "test_path"); + TFmrTableRef fmrTable = TFmrTableRef{fmrTableId}; + + auto partIdsForTables = GetTestPartIdsForTable(fmrTableId); + auto partIdStats = GetTestPartIdStats(); + TFmrPartitionerSettings settings{.MaxDataWeightPerPart = 50, .MaxParts = 2}; + TFmrPartitioner partitioner(partIdsForTables, partIdStats, settings); + + auto [gottenTasks, status] = partitioner.PartitionFmrTablesIntoTasks({fmrTable}); + UNIT_ASSERT_VALUES_EQUAL(status, false); + } + Y_UNIT_TEST(SeveralFullPartitionsInTask) { + auto fmrTableId = TFmrTableId("test_cluster", "test_path"); + TFmrTableRef fmrTable = TFmrTableRef{fmrTableId}; + + auto partIdsForTables = GetTestPartIdsForTable(fmrTableId); + auto partIdStats = GetTestPartIdStats(); + TFmrPartitionerSettings settings{.MaxDataWeightPerPart = 1000000, .MaxParts = 1}; + TFmrPartitioner partitioner(partIdsForTables, partIdStats, settings); + + auto [gottenTasks, status] = partitioner.PartitionFmrTablesIntoTasks({fmrTable}); + UNIT_ASSERT_VALUES_EQUAL(status, true); + + std::vector<std::vector<TFmrTableInputRef>> expectedTasks = { + { + TFmrTableInputRef{ + .TableId = fmrTableId.Id, + .TableRanges = { + TTableRange{.PartId = FirstPartId, .MinChunk = 0, .MaxChunk = 4}, + TTableRange{.PartId = SecondPartId, .MinChunk = 0, .MaxChunk = 2}, + } + } + } + }; + UNIT_ASSERT_VALUES_EQUAL(ChangeGottenTasksFormat(gottenTasks), expectedTasks); + } + Y_UNIT_TEST(SeveralInputTables) { + std::vector<TFmrTableId> inputFmrTableIds{ + TFmrTableId("test_cluster_1", "test_path_1"), + TFmrTableId("test_cluster_2", "test_path_2"), + TFmrTableId("test_cluster_3", "test_path_3"), + }; + std::vector<TFmrTableRef> inputTables; + for (auto& id: inputFmrTableIds) { + inputTables.emplace_back(TFmrTableRef{.FmrTableId = id}); + } + + auto partIdsForTables = GetTestPartIdsForMultipleTables(inputFmrTableIds); + auto partIdStats = GetTestPartIdStatsForMultipleTables(); + TFmrPartitionerSettings settings{.MaxDataWeightPerPart = 50, .MaxParts = 1000}; + TFmrPartitioner partitioner(partIdsForTables, partIdStats, settings); + auto [gottenTasks, status] = partitioner.PartitionFmrTablesIntoTasks(inputTables); + UNIT_ASSERT_VALUES_EQUAL(status, true); + + std::vector<std::vector<TFmrTableInputRef>> expectedTasks = { + {TFmrTableInputRef{ + .TableId = TFmrTableId("test_cluster_1", "test_path_1").Id, + .TableRanges = { + TTableRange{.PartId = FirstTablePartId, .MinChunk = 0, .MaxChunk = 1} + } + }}, + {TFmrTableInputRef{ + .TableId = TFmrTableId("test_cluster_2", "test_path_2").Id, + .TableRanges = { + TTableRange{.PartId = SecondTablePartId, .MinChunk = 0, .MaxChunk = 1} + } + }}, + + {TFmrTableInputRef{ + .TableId = TFmrTableId("test_cluster_3", "test_path_3").Id, + .TableRanges = { + TTableRange{.PartId = ThirdTablePartId, .MinChunk = 0, .MaxChunk = 2} + } + }}, + {TFmrTableInputRef{ + .TableId = TFmrTableId("test_cluster_3", "test_path_3").Id, + .TableRanges = { + TTableRange{.PartId = ThirdTablePartId, .MinChunk = 2, .MaxChunk = 3} + } + }}, + { + TFmrTableInputRef{ + .TableId = TFmrTableId("test_cluster_1", "test_path_1").Id, + .TableRanges = { + TTableRange{.PartId = FirstTablePartId, .MinChunk = 1, .MaxChunk = 2} + } + }, + TFmrTableInputRef{ + .TableId = TFmrTableId("test_cluster_2", "test_path_2").Id, + .TableRanges = { + TTableRange{.PartId = SecondTablePartId, .MinChunk = 1, .MaxChunk = 3} + } + } + } + }; + UNIT_ASSERT_VALUES_EQUAL(ChangeGottenTasksFormat(gottenTasks), expectedTasks); + } +} + +} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/coordinator/impl/ya.make b/yt/yql/providers/yt/fmr/coordinator/impl/ya.make index 20abd2bbc84..5d0057b51f8 100644 --- a/yt/yql/providers/yt/fmr/coordinator/impl/ya.make +++ b/yt/yql/providers/yt/fmr/coordinator/impl/ya.make @@ -2,6 +2,7 @@ LIBRARY() SRCS( yql_yt_coordinator_impl.cpp + yql_yt_partitioner.cpp ) PEERDIR( @@ -11,8 +12,10 @@ PEERDIR( library/cpp/yson/node yt/cpp/mapreduce/common yt/yql/providers/yt/fmr/coordinator/interface - yql/essentials/utils/log + yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface + yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/impl yql/essentials/utils + yql/essentials/utils/log ) RESOURCE( diff --git a/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.cpp b/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.cpp index 10ce8469243..22d9ebc6da2 100644 --- a/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.cpp +++ b/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.cpp @@ -1,6 +1,7 @@ #include <thread> #include <library/cpp/resource/resource.h> #include <yt/cpp/mapreduce/common/helpers.h> +#include <yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_partitioner.h> #include <yql/essentials/utils/log/log.h> #include <yql/essentials/utils/yql_panic.h> #include "yql_yt_coordinator_impl.h" @@ -17,39 +18,16 @@ TFmrCoordinatorSettings::TFmrCoordinatorSettings() { namespace { -struct TCoordinatorTaskInfo { - TTask::TPtr Task; - ETaskStatus TaskStatus; - TString OperationId; -}; - -struct TOperationInfo { - std::unordered_set<TString> TaskIds; // for now each operation consists only of one task, until paritioner is implemented - EOperationStatus OperationStatus; - std::vector<TFmrError> ErrorMessages; - TString SessionId; - std::vector<TString> OutputTableIds = {}; -}; - -struct TIdempotencyKeyInfo { - TString OperationId; - TInstant OperationCreationTime; -}; - -struct TCoordinatorFmrTableStats { - TTableStats Stats; - TString PartId; // only one PartId for now -}; - class TFmrCoordinator: public IFmrCoordinator { public: - TFmrCoordinator(const TFmrCoordinatorSettings& settings) + TFmrCoordinator(const TFmrCoordinatorSettings& settings, IYtCoordinatorService::TPtr ytCoordinatorService) : WorkersNum_(settings.WorkersNum), RandomProvider_(settings.RandomProvider), StopCoordinator_(false), TimeToSleepBetweenClearKeyRequests_(settings.TimeToSleepBetweenClearKeyRequests), IdempotencyKeyStoreTime_(settings.IdempotencyKeyStoreTime), - DefaultFmrOperationSpec_(settings.DefaultFmrOperationSpec) + DefaultFmrOperationSpec_(settings.DefaultFmrOperationSpec), + YtCoordinatorService_(ytCoordinatorService) { StartClearingIdempotencyKeys(); } @@ -73,13 +51,20 @@ public: IdempotencyKeys_[*IdempotencyKey] = TIdempotencyKeyInfo{.OperationId = operationId, .OperationCreationTime=TInstant::Now()}; } - TString taskId = GenerateId(); - auto taskParams = MakeDefaultTaskParamsFromOperation(request.OperationParams); + auto fmrOperationSpec = GetMergedFmrOperationSpec(request.FmrOperationSpec); + auto taskParams = PartitionOperationIntoSeveralTasks(request.OperationParams, fmrOperationSpec, request.ClusterConnections); - TTask::TPtr createdTask = MakeTask(request.TaskType, taskId, taskParams, request.SessionId, request.ClusterConnections, GetJobSettings(request.FmrOperationSpec)); - Tasks_[taskId] = TCoordinatorTaskInfo{.Task = createdTask, .TaskStatus = ETaskStatus::Accepted, .OperationId = operationId}; + std::unordered_set<TString> taskIds; + + for (auto& currentTaskParams: taskParams) { + TString taskId = GenerateId(); + TTask::TPtr createdTask = MakeTask(request.TaskType, taskId, currentTaskParams, request.SessionId, request.ClusterConnections, fmrOperationSpec); + Tasks_[taskId] = TCoordinatorTaskInfo{.Task = createdTask, .TaskStatus = ETaskStatus::Accepted, .OperationId = operationId}; + TasksToRun_.emplace(createdTask, taskId); + taskIds.emplace(taskId); + } - Operations_[operationId] = {.TaskIds = {taskId}, .OperationStatus = EOperationStatus::Accepted, .SessionId = request.SessionId}; + Operations_[operationId] = {.TaskIds = taskIds, .OperationStatus = EOperationStatus::Accepted, .SessionId = request.SessionId}; YQL_CLOG(DEBUG, FastMapReduce) << "Starting operation with id " << operationId; return NThreading::MakeFuture(TStartOperationResponse(EOperationStatus::Accepted, operationId)); } @@ -96,8 +81,11 @@ public: auto operationStatus = operationInfo.OperationStatus; auto errorMessages = operationInfo.ErrorMessages; std::vector<TTableStats> outputTablesStats; - for (auto& tableId : operationInfo.OutputTableIds) { - outputTablesStats.emplace_back(FmrTableStatistics_[tableId].Stats); + if (operationStatus == EOperationStatus::Completed) { + // Calculating output table stats only in case of successful completion of opereation + for (auto& tableId : operationInfo.OutputTableIds) { + outputTablesStats.emplace_back(CalculateTableStats(tableId)); + } } return NThreading::MakeFuture(TGetOperationResponse(operationStatus, errorMessages, outputTablesStats)); } @@ -111,15 +99,14 @@ public: YQL_LOG_CTX_ROOT_SESSION_SCOPE(Operations_[operationId].SessionId); YQL_CLOG(DEBUG, FastMapReduce) << "Deleting operation with id " << operationId; auto taskIds = Operations_[operationId].TaskIds; - YQL_ENSURE(taskIds.size() == 1); - auto taskId = *taskIds.begin(); - YQL_ENSURE(Tasks_.contains(taskId)); - - auto taskStatus = Tasks_[taskId].TaskStatus; - if (taskStatus == ETaskStatus::InProgress) { - TaskToDeleteIds_.insert(taskId); // Task is currently running, send signal to worker to cancel - } else { - ClearTask(taskId); // Task either hasn't begun running or finished, remove info + for (auto& taskId: taskIds){ + YQL_ENSURE(Tasks_.contains(taskId)); + auto taskStatus = Tasks_[taskId].TaskStatus; + if (taskStatus == ETaskStatus::InProgress) { + TaskToDeleteIds_.insert(taskId); // Task is currently running, send signal to worker to cancel + } else { + ClearTask(taskId); // Task either hasn't begun running or finished, remove info + } } return NThreading::MakeFuture(TDeleteOperationResponse(EOperationStatus::Aborted)); @@ -148,9 +135,9 @@ public: for (auto& requestTaskState: request.TaskStates) { auto taskId = requestTaskState->TaskId; + YQL_ENSURE(Tasks_.contains(taskId)); auto operationId = Tasks_[taskId].OperationId; YQL_LOG_CTX_ROOT_SESSION_SCOPE(Operations_[operationId].SessionId); - YQL_ENSURE(Tasks_.contains(taskId)); auto taskStatus = requestTaskState->TaskStatus; YQL_ENSURE(taskStatus != ETaskStatus::Accepted); SetUnfinishedTaskStatus(taskId, taskStatus, requestTaskState->TaskErrorMessage); @@ -159,48 +146,52 @@ public: } auto statistics = requestTaskState->Stats; + YQL_CLOG(TRACE, FastMapReduce) << " Task with id " << taskId << " has current status " << taskStatus << Endl; + bool isOperationCompleted = (GetOperationStatus(operationId) == EOperationStatus::Completed); for (auto& [fmrTableId, tableStats]: statistics.OutputTables) { - if (FmrTableStatistics_.contains(fmrTableId.TableId)) { - auto curTableStats = FmrTableStatistics_[fmrTableId.TableId]; - YQL_ENSURE( - tableStats.Chunks >= curTableStats.Stats.Chunks && - tableStats.DataWeight >= curTableStats.Stats.DataWeight && - tableStats.Rows >= curTableStats.Stats.Rows - ); - YQL_ENSURE(fmrTableId.PartId == curTableStats.PartId); - if (taskStatus == ETaskStatus::Completed) { - YQL_CLOG(DEBUG, FastMapReduce) << "Current statistic from table with id" << fmrTableId.TableId << "_" << fmrTableId.PartId << ": " << tableStats; - } + Operations_[operationId].OutputTableIds.emplace(fmrTableId.TableId); + PartIdStats_[fmrTableId.PartId] = tableStats.PartIdChunkStats; + if (isOperationCompleted) { + YQL_CLOG(INFO, FastMapReduce) << "Operation with id " << operationId << " has finished successfully"; + CalculateTableStats(fmrTableId.TableId, true); } - Operations_[operationId].OutputTableIds.emplace_back(fmrTableId.TableId); - FmrTableStatistics_[fmrTableId.TableId] = TCoordinatorFmrTableStats{ - .Stats = tableStats, - .PartId = fmrTableId.PartId - }; + // TODO - проверка на валидность возвращаемой воркером статистики? } } - std::vector<TTask::TPtr> tasksToRun; - for (auto& taskToRunInfo: Tasks_) { - if (taskToRunInfo.second.TaskStatus == ETaskStatus::Accepted) { - SetUnfinishedTaskStatus(taskToRunInfo.first, ETaskStatus::InProgress); - tasksToRun.emplace_back(taskToRunInfo.second.Task); + std::vector<TTask::TPtr> currentTasksToRun; + ui64 filledSlots = 0; + while (filledSlots < request.AvailableSlots) { + if (TasksToRun_.empty()) { + break; } + auto [task, taskId] = TasksToRun_.front(); + TasksToRun_.pop(); + if (!Tasks_.contains(taskId)) { + continue; + } + auto& taskInfo = Tasks_[taskId]; + YQL_ENSURE(taskInfo.TaskStatus == ETaskStatus::Accepted); + SetUnfinishedTaskStatus(taskId, ETaskStatus::InProgress); + SetPartIdsForTask(task); + currentTasksToRun.emplace_back(task); + ++filledSlots; } - return NThreading::MakeFuture(THeartbeatResponse{.TasksToRun = tasksToRun, .TaskToDeleteIds = TaskToDeleteIds_}); + + return NThreading::MakeFuture(THeartbeatResponse{.TasksToRun = currentTasksToRun, .TaskToDeleteIds = TaskToDeleteIds_}); } NThreading::TFuture<TGetFmrTableInfoResponse> GetFmrTableInfo(const TGetFmrTableInfoRequest& request) override { TGuard<TMutex> guard(Mutex_); TGetFmrTableInfoResponse response; auto tableId = request.TableId; - if (!FmrTableStatistics_.contains(tableId)) { + if (!PartIdsForTables_.contains(tableId)) { response.ErrorMessages = {TFmrError{ .Component = EFmrComponent::Coordinator, .ErrorMessage = "Fmr table id " + tableId + " was not found" }}; return NThreading::MakeFuture(response); } - response.TableStats = FmrTableStatistics_[tableId].Stats; + response.TableStats = CalculateTableStats(tableId); return NThreading::MakeFuture(response); } @@ -219,11 +210,11 @@ private: if (Operations_.contains(operationId)) { auto& operationInfo = Operations_[operationId]; auto operationStatus = operationInfo.OperationStatus; - auto& taskIds = operationInfo.TaskIds; - YQL_ENSURE(taskIds.size() == 1); - auto taskId = *operationInfo.TaskIds.begin(); if (operationStatus != EOperationStatus::Accepted && operationStatus != EOperationStatus::InProgress) { - ClearTask(taskId); + auto& taskIds = operationInfo.TaskIds; + for (auto& taskId: taskIds) { + ClearTask(taskId); + } } } } else { @@ -245,7 +236,14 @@ private: YQL_ENSURE(Tasks_.contains(taskId)); auto& taskInfo = Tasks_[taskId]; TaskToDeleteIds_.erase(taskId); - Operations_.erase(taskInfo.OperationId); + + YQL_ENSURE(Operations_.contains(taskInfo.OperationId)); + auto& currentTaskIdsForOperation = Operations_[taskInfo.OperationId]; + currentTaskIdsForOperation.TaskIds.erase(taskId); + if (currentTaskIdsForOperation.TaskIds.empty()) { + // All task for operation are cleared, can clear it + Operations_.erase(taskInfo.OperationId); + } Tasks_.erase(taskId); } @@ -256,6 +254,7 @@ private: if (taskInfo.TaskStatus != ETaskStatus::Accepted && taskInfo.TaskStatus != ETaskStatus::InProgress) { return; } + YQL_CLOG(TRACE, FastMapReduce) << "Setting task status for task id" << taskId << " from " << taskInfo.TaskStatus << " to new Task status " << newTaskStatus << "\n"; taskInfo.TaskStatus = newTaskStatus; operationInfo.OperationStatus = GetOperationStatus(taskInfo.OperationId); if (taskErrorMessage) { @@ -265,102 +264,89 @@ private: } EOperationStatus GetOperationStatus(const TString& operationId) { - if (! Operations_.contains(operationId)) { + if (!Operations_.contains(operationId)) { return EOperationStatus::NotFound; } std::unordered_set<TString> taskIds = Operations_[operationId].TaskIds; - YQL_ENSURE(taskIds.size() == 1); + std::unordered_set<ETaskStatus> taskStatuses; + + for (auto& taskId: taskIds) { + taskStatuses.emplace(Tasks_[taskId].TaskStatus); + } + YQL_ENSURE(!taskStatuses.contains(ETaskStatus::Unknown)); - auto taskId = *taskIds.begin(); - ETaskStatus taskStatus = Tasks_[taskId].TaskStatus; - return static_cast<EOperationStatus>(taskStatus); + if (taskStatuses.contains(ETaskStatus::Failed)) { + return EOperationStatus::Failed; + } + if (taskStatuses.contains(ETaskStatus::InProgress)) { + return EOperationStatus::InProgress; + } + if (taskStatuses.contains(ETaskStatus::InProgress)) { + return EOperationStatus::InProgress; + } + if (taskStatuses.contains(ETaskStatus::Accepted)) { + return EOperationStatus::Accepted; + } + return EOperationStatus::Completed; } - TTableRange GetTableRangeFromId(const TString& tableId) { - if (!FmrTableStatistics_.contains(tableId)) { - TString partId = GenerateId(); - FmrTableStatistics_[tableId] = TCoordinatorFmrTableStats{.Stats=TTableStats{}, .PartId=partId}; - return TTableRange{.PartId = partId}; - } - auto fmrTableStats = FmrTableStatistics_[tableId]; - return TTableRange{ - .PartId = fmrTableStats.PartId, - .MinChunk = 0, - .MaxChunk = fmrTableStats.Stats.Chunks - }; + TFmrPartitionerSettings GetFmrPartitionerSettings(const NYT::TNode& fmrOperationSpec) { + TFmrPartitionerSettings settings; + auto& fmrPartitionSettings = fmrOperationSpec["partition"]["fmr_table"]; + settings.MaxDataWeightPerPart = fmrPartitionSettings["max_data_weight_per_part"].AsInt64(); + settings.MaxParts = fmrPartitionSettings["max_parts"].AsInt64(); + return settings; } - std::vector<TTaskTableRef> TaskInputTablesFromOperationInputTables(const std::vector<TOperationTableRef>& operationTables) { - std::vector<TTaskTableRef> taskInputTables; - for (auto& elem: operationTables) { - if (const TYtTableRef* ytTableRef = std::get_if<TYtTableRef>(&elem)) { - taskInputTables.emplace_back(*ytTableRef); - } else { - TFmrTableRef fmrTableRef = std::get<TFmrTableRef>(elem); - TString inputTableId = fmrTableRef.FmrTableId.Id; - TFmrTableInputRef tableInput{ - .TableId = inputTableId, - .TableRanges = {GetTableRangeFromId(inputTableId)} - }; - taskInputTables.emplace_back(tableInput); - } + TYtPartitionerSettings GetYtPartitionerSettings(const NYT::TNode& fmrOperationSpec) { + TYtPartitionerSettings settings; + auto& ytPartitionSettings = fmrOperationSpec["partition"]["yt_table"]; + settings.MaxDataWeightPerPart = ytPartitionSettings["max_data_weight_per_part"].AsInt64(); + settings.MaxParts = ytPartitionSettings["max_parts"].AsInt64(); + return settings; + } + + std::vector<TTaskParams> PartitionOperationIntoSeveralTasks(const TOperationParams& operationParams, const NYT::TNode& fmrOperationSpec, const std::unordered_map<TFmrTableId, TClusterConnection>& clusterConnections) { + auto fmrPartitionerSettings = GetFmrPartitionerSettings(fmrOperationSpec); + auto ytPartitionerSettings = GetYtPartitionerSettings(fmrOperationSpec); + auto fmrPartitioner = TFmrPartitioner(PartIdsForTables_,PartIdStats_, fmrPartitionerSettings); // TODO - fix this + + std::vector<TYtTableRef> ytInputTables; + std::vector<TFmrTableRef> fmrInputTables; + GetOperationInputTables(ytInputTables, fmrInputTables, operationParams); + + TPartitionResult partitionResult = PartitionInputTablesIntoTasks(ytInputTables, fmrInputTables, fmrPartitioner, YtCoordinatorService_, clusterConnections, ytPartitionerSettings); + if (!partitionResult.PartitionStatus) { + ythrow yexception() << "Failed to partition input tables into tasks"; + // TODO - return FAILED_PARTITIONING status instead. } - return taskInputTables; + return GetOutputTaskParams(partitionResult, operationParams); } - std::vector<TFmrTableOutputRef> TaskOutputTablesFromOperationOutputTables(const std::vector<TFmrTableRef>& operationTables) { - std::vector<TFmrTableOutputRef> taskOutputTables; - for (auto& fmrTableRef: operationTables) { - TString outputTableId = fmrTableRef.FmrTableId.Id; - TFmrTableOutputRef tableOutput{ - .TableId = outputTableId, - .PartId = GetTableRangeFromId(outputTableId).PartId - }; - taskOutputTables.emplace_back(tableOutput); + void GetOperationInputTables(std::vector<TYtTableRef>& ytInputTables, std::vector<TFmrTableRef>& fmrInputTables, const TOperationParams& operationParams) { + TOperationInputTablesGetter tablesGetter{}; + std::visit(tablesGetter, operationParams); + + auto& inputTables = tablesGetter.OperationTableRef; + for (auto& table: inputTables) { + auto ytTable = std::get_if<TYtTableRef>(&table); + auto fmrTable = std::get_if<TFmrTableRef>(&table); + if (ytTable) { + ytInputTables.emplace_back(*ytTable); + } else { + fmrInputTables.emplace_back(*fmrTable); } - return taskOutputTables; + } } - TTaskParams MakeDefaultTaskParamsFromOperation(const TOperationParams& operationParams) { - if (const TUploadOperationParams* uploadOperationParams = std::get_if<TUploadOperationParams>(&operationParams)) { - TUploadTaskParams uploadTaskParams{}; - uploadTaskParams.Output = uploadOperationParams->Output; - TString inputTableId = uploadOperationParams->Input.FmrTableId.Id; - TFmrTableInputRef fmrTableInput{ - .TableId = inputTableId, - .TableRanges = {GetTableRangeFromId(inputTableId)} - }; - uploadTaskParams.Input = fmrTableInput; - return uploadTaskParams; - } else if (const TDownloadOperationParams* downloadOperationParams = std::get_if<TDownloadOperationParams>(&operationParams)) { - TDownloadTaskParams downloadTaskParams{}; - downloadTaskParams.Input = downloadOperationParams->Input; - TString outputTableId = downloadOperationParams->Output.FmrTableId.Id; - TFmrTableOutputRef fmrTableOutput{ - .TableId = outputTableId, - .PartId = GetTableRangeFromId(outputTableId).PartId - }; - downloadTaskParams.Output = fmrTableOutput; - return downloadTaskParams; - } else if (const TMergeOperationParams* mergeOperationParams = std::get_if<TMergeOperationParams>(&operationParams)) { - TMergeTaskParams mergeTaskParams; - mergeTaskParams.Input = TaskInputTablesFromOperationInputTables(mergeOperationParams->Input); - TFmrTableOutputRef outputTable; - mergeTaskParams.Output = TFmrTableOutputRef{.TableId = mergeOperationParams->Output.FmrTableId.Id}; - return mergeTaskParams; - } else if (const TMapOperationParams* mapOperationParams = std::get_if<TMapOperationParams>(&operationParams)) { - TMapTaskParams mapTaskParams; - mapTaskParams.Input = TaskInputTablesFromOperationInputTables(mapOperationParams->Input); - mapTaskParams.Output = TaskOutputTablesFromOperationOutputTables(mapOperationParams->Output); - mapTaskParams.Executable = mapOperationParams->Executable; - return mapTaskParams; - } else { - ythrow yexception() << "Unknown operation params"; - } + std::vector<TTaskParams> GetOutputTaskParams(const TPartitionResult& partitionResult, const TOperationParams& operationParams) { + TOutputTaskParamsGetter taskGetter{.PartitionResult = partitionResult}; + std::visit(taskGetter, operationParams); + return taskGetter.TaskParams; } - NYT::TNode GetJobSettings(const TMaybe<NYT::TNode>& currentFmrOperationSpec) { - // For now fmr operation spec only consists of job settings + NYT::TNode GetMergedFmrOperationSpec(const TMaybe<NYT::TNode>& currentFmrOperationSpec) { + // just pass whole merged operation spec for simplicity here if (!currentFmrOperationSpec) { return DefaultFmrOperationSpec_; } @@ -369,7 +355,76 @@ private: return resultFmrOperationSpec; } + void SetPartIdsForTask(TTask::TPtr task) { + // TODO - add failover, clearing previous partId if exists + TString partId = GenerateId(); + + auto* downloadTaskParams = std::get_if<TDownloadTaskParams>(&task->TaskParams); + auto* mergeTaskParams = std::get_if<TMergeTaskParams>(&task->TaskParams); + auto* mapTaskParams = std::get_if<TMapTaskParams>(&task->TaskParams); + if (downloadTaskParams) { + TString tableId = downloadTaskParams->Output.TableId; + downloadTaskParams->Output.PartId = partId; + PartIdsForTables_[tableId].emplace_back(partId); + } else if (mergeTaskParams) { + TString tableId = mergeTaskParams->Output.TableId; + mergeTaskParams->Output.PartId = partId; + PartIdsForTables_[tableId].emplace_back(partId); + } else if (mapTaskParams) { + for (auto& fmrTableOutputRef: mapTaskParams->Output) { + TString tableId = fmrTableOutputRef.TableId; + fmrTableOutputRef.PartId = partId; + PartIdsForTables_[tableId].emplace_back(partId); + } + } + } + + TTableStats CalculateTableStats(const TString& tableId, bool isOperationFinished = false) { + if (OperationTableStats_.contains(tableId)) { + return OperationTableStats_[tableId]; + } + TTableStats tableStats{}; + auto& partIds = PartIdsForTables_.at(tableId); + YQL_CLOG(DEBUG, FastMapReduce) << "Calculating table stats for table with id " << tableId << " with " << partIds.size() << " part ids"; + for (auto& part: partIds) { + auto& partStats = PartIdStats_[part]; + tableStats.Chunks += partStats.size(); + YQL_CLOG(DEBUG, FastMapReduce) << " Gotten " << partStats.size() << " chunks for part id " << part; + for (auto& chunkStats: PartIdStats_[part]) { + tableStats.DataWeight += chunkStats.DataWeight; + tableStats.Rows += chunkStats.Rows; + } + } + if (isOperationFinished) { + // Stats for table won't change, inserting into map for caching + OperationTableStats_[tableId] = tableStats; + } + return tableStats; + } + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + + struct TCoordinatorTaskInfo { + TTask::TPtr Task; + ETaskStatus TaskStatus; + TString OperationId; + }; + + struct TOperationInfo { + std::unordered_set<TString> TaskIds; + EOperationStatus OperationStatus; + std::vector<TFmrError> ErrorMessages; + TString SessionId; + std::unordered_set<TString> OutputTableIds = {}; + }; + + struct TIdempotencyKeyInfo { + TString OperationId; + TInstant OperationCreationTime; + }; + std::unordered_map<TString, TCoordinatorTaskInfo> Tasks_; // TaskId -> current info about it + std::queue<std::pair<TTask::TPtr, TString>> TasksToRun_; // Task, and TaskId std::unordered_set<TString> TaskToDeleteIds_; // TaskIds we want to pass to worker for deletion std::unordered_map<TString, TOperationInfo> Operations_; // OperationId -> current info about it std::unordered_map<TString, TIdempotencyKeyInfo> IdempotencyKeys_; // IdempotencyKey -> current info about it @@ -382,14 +437,90 @@ private: std::atomic<bool> StopCoordinator_; TDuration TimeToSleepBetweenClearKeyRequests_; TDuration IdempotencyKeyStoreTime_; - std::unordered_map<TFmrTableId, TCoordinatorFmrTableStats> FmrTableStatistics_; // TableId -> Statistics + + std::unordered_map<TFmrTableId, std::vector<TString>> PartIdsForTables_; // TableId -> List of all corresponding partIds + std::unordered_map<TString, std::vector<TChunkStats>> PartIdStats_; // PartId -> Detailed statistic for each chunk + std::unordered_map<TString, TTableStats> OperationTableStats_; // TableId -> Statistic for fmr table, filled when operation completes + + NYT::TNode DefaultFmrOperationSpec_; + IYtCoordinatorService::TPtr YtCoordinatorService_; // Needed for partitioning of yt tables + + ////////////////////////////////////////////////////////////////////////////////////////////////////////// + + // Helper structs for partitioning operation into tasks + + struct TOperationInputTablesGetter { + std::vector<TOperationTableRef> OperationTableRef; // will be filled when std::visit is called + + void operator () (const TUploadOperationParams& uploadOperationParams) { + OperationTableRef.emplace_back(uploadOperationParams.Input); + } + void operator () (const TDownloadOperationParams& downloadOperationParams) { + OperationTableRef.emplace_back(downloadOperationParams.Input); + } + void operator () (const TMergeOperationParams& mergeOperationParams) { + OperationTableRef = mergeOperationParams.Input; + } + void operator () (const TMapOperationParams& mapOperationParams) { + OperationTableRef = mapOperationParams.Input; + } + }; + + struct TOutputTaskParamsGetter { + std::vector<TTaskParams> TaskParams; // Will be filled when std::visit is called + TPartitionResult PartitionResult; + + void operator () (const TUploadOperationParams& uploadOperationParams) { + for (auto& task: PartitionResult.TaskInputs) { + TUploadTaskParams uploadTaskParams; + YQL_ENSURE(task.Inputs.size() == 1, "Upload task should have exactly one fmr table partition input"); + auto& fmrTablePart = task.Inputs[0]; + uploadTaskParams.Input = std::get<TFmrTableInputRef>(fmrTablePart); + uploadTaskParams.Output = uploadOperationParams.Output; + TaskParams.emplace_back(uploadTaskParams); + } + } + void operator () (const TDownloadOperationParams& downloadOperationParams) { + for (auto& task: PartitionResult.TaskInputs) { + TDownloadTaskParams downloadTaskParams; + YQL_ENSURE(task.Inputs.size() == 1, "Download task should have exactly one yt table partition input"); + auto& ytTablePart = task.Inputs[0]; + downloadTaskParams.Input = std::get<TYtTableTaskRef>(ytTablePart); + downloadTaskParams.Output = TFmrTableOutputRef{.TableId = downloadOperationParams.Output.FmrTableId.Id}; + // PartId for tasks which write to table data service will be set later + TaskParams.emplace_back(downloadTaskParams); + } + } + void operator () (const TMergeOperationParams& mergeOperationParams) { + for (auto& task: PartitionResult.TaskInputs) { + TMergeTaskParams mergeTaskParams; + mergeTaskParams.Input = task; + mergeTaskParams.Output = TFmrTableOutputRef{.TableId = mergeOperationParams.Output.FmrTableId.Id}; + TaskParams.emplace_back(mergeTaskParams); + } + } + void operator () (const TMapOperationParams& mapOperationParams) { + for (auto& task: PartitionResult.TaskInputs) { + TMapTaskParams mapTaskParams; + mapTaskParams.Input = task; + std::vector<TFmrTableOutputRef> fmrTableOutputRefs; + std::transform(mapOperationParams.Output.begin(), mapOperationParams.Output.end(), std::back_inserter(fmrTableOutputRefs), [] (const TFmrTableRef& fmrTableRef) { + return TFmrTableOutputRef{.TableId = fmrTableRef.FmrTableId.Id}; + }); + + mapTaskParams.Output = fmrTableOutputRefs; + mapTaskParams.Executable = mapOperationParams.Executable; // TODO - change Executable to mapper + TaskParams.emplace_back(mapTaskParams); + } + } + }; }; } // namespace -IFmrCoordinator::TPtr MakeFmrCoordinator(const TFmrCoordinatorSettings& settings) { - return MakeIntrusive<TFmrCoordinator>(settings); +IFmrCoordinator::TPtr MakeFmrCoordinator(const TFmrCoordinatorSettings& settings, IYtCoordinatorService::TPtr ytCoordinatorService) { + return MakeIntrusive<TFmrCoordinator>(settings, ytCoordinatorService); } } // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.h b/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.h index 563752dfc7b..ceac2a7b8cc 100644 --- a/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.h +++ b/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.h @@ -6,6 +6,8 @@ #include <util/system/guard.h> #include <util/generic/queue.h> #include <yt/yql/providers/yt/fmr/coordinator/interface/yql_yt_coordinator.h> +#include <yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface/yql_yt_coordinator_service_interface.h> +#include <yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/impl/yql_yt_coordinator_service_impl.h> namespace NYql::NFmr { @@ -19,6 +21,9 @@ struct TFmrCoordinatorSettings { TFmrCoordinatorSettings(); }; -IFmrCoordinator::TPtr MakeFmrCoordinator(const TFmrCoordinatorSettings& settings = TFmrCoordinatorSettings()); +IFmrCoordinator::TPtr MakeFmrCoordinator( + const TFmrCoordinatorSettings& settings = TFmrCoordinatorSettings(), + IYtCoordinatorService::TPtr ytCoordinatorService = MakeYtCoordinatorService() +); } // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_partitioner.cpp b/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_partitioner.cpp new file mode 100644 index 00000000000..9f3f305130a --- /dev/null +++ b/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_partitioner.cpp @@ -0,0 +1,168 @@ +#include "yql_yt_partitioner.h" +#include <library/cpp/iterator/enumerate.h> +#include <yql/essentials/utils/log/log.h> +#include <yql/essentials/utils/yql_panic.h> +#include <yt/cpp/mapreduce/common/helpers.h> + +namespace NYql::NFmr { + +TFmrPartitioner::TFmrPartitioner( + const std::unordered_map<TFmrTableId, std::vector<TString>>& partIdsForTables, + const std::unordered_map<TString, std::vector<TChunkStats>>& partIdStats, + const TFmrPartitionerSettings& settings +) + : PartIdsForTables_(partIdsForTables), PartIdStats_(partIdStats), Settings_(settings) +{ +} + +std::pair<std::vector<TTaskTableInputRef>, bool> TFmrPartitioner::PartitionFmrTablesIntoTasks(const std::vector<TFmrTableRef>& fmrTables) { + // TODO - return matrix with ranges for all tables, in order to support table_index correctly. + if (fmrTables.empty()) { + return {{}, true}; + } + const ui64 maxDataWeightPerPart = Settings_.MaxDataWeightPerPart; + std::vector<TTaskTableInputRef> currentFmrTasks; + std::vector<TLeftoverRange> leftoverRanges; + // First try to create tasks in which all chunks have the same partId, then handle leftovers (end of chunks for each partId) + for (const auto& fmrTable: fmrTables) { + YQL_ENSURE(PartIdsForTables_.contains(fmrTable.FmrTableId)); + auto partIds = PartIdsForTables_.at(fmrTable.FmrTableId); + for (auto& partId: partIds) { + std::vector<TChunkStats> stats = PartIdStats_.at(partId); + HandleFmrPartition(fmrTable.FmrTableId, partId, stats, maxDataWeightPerPart, currentFmrTasks, leftoverRanges); + if (!CheckMaxTasksSize(currentFmrTasks)) { + return {{}, false}; + } + } + } + HandleFmrLeftoverRanges(maxDataWeightPerPart, currentFmrTasks, leftoverRanges); + if (!CheckMaxTasksSize(currentFmrTasks)) { + return {{}, false}; + } + return {currentFmrTasks, true}; +} + +void TFmrPartitioner::HandleFmrPartition( + const TFmrTableId& fmrTable, + const TString& partId, + const std::vector<TChunkStats> stats, + ui64 maxDataWeightPerPart, + std::vector<TTaskTableInputRef>& currentFmrTasks, + std::vector<TLeftoverRange>& leftoverRanges +) { + ui64 curDataWeight = 0; + i64 curMinChunk = -1; + + for (ui64 i = 0; i < stats.size();) { + if (curDataWeight + stats[i].DataWeight <= maxDataWeightPerPart) { + // check if we can add this chunk to current task, or have to split + curDataWeight += stats[i].DataWeight; + if (curMinChunk == -1) { + curMinChunk = i; + } + ++i; + } else { + if (curMinChunk != -1) { + std::vector<TTableRange> tableRange{TTableRange{.PartId = partId, .MinChunk = static_cast<ui64>(curMinChunk), .MaxChunk = i}}; + TFmrTableInputRef fmrTableInput{.TableId = fmrTable.Id, .TableRanges = tableRange}; + currentFmrTasks.emplace_back(TTaskTableInputRef{.Inputs = {fmrTableInput}}); + } + curMinChunk = -1; + curDataWeight = 0; + ui64 j = i; + while (j < stats.size()) { + // iterate to create separate tasks for all chunks which are larger then maxDataWeight + if (stats[j].DataWeight < maxDataWeightPerPart) { + break; + } + std::vector<TTableRange> tableRange{TTableRange{.PartId = partId, .MinChunk = j, .MaxChunk = j + 1}}; + TFmrTableInputRef fmrTableInput{.TableId = fmrTable.Id, .TableRanges = tableRange}; + currentFmrTasks.emplace_back(TTaskTableInputRef{.Inputs = {fmrTableInput}}); + ++j; + } + i = j; + } + } + + if (curMinChunk != -1) { + TTableRange leftoverTableRange{.PartId = partId, .MinChunk = static_cast<ui64>(curMinChunk), .MaxChunk = stats.size()}; + leftoverRanges.emplace_back(TLeftoverRange{.TableId = fmrTable.Id, .TableRange = leftoverTableRange, .DataWeight = curDataWeight}); + } +} + +void TFmrPartitioner::HandleFmrLeftoverRanges( + ui64 maxDataWeightPerPart, + std::vector<TTaskTableInputRef>& fmrTasks, + std::vector<TLeftoverRange>& leftoverRanges +) { + TTaskTableInputRef currentTask{}; + ui64 curDataWeight = 0; + TFmrTableInputRef curFmrTable; + TString curTableId; + for (auto& range: leftoverRanges) { + if (curDataWeight + range.DataWeight > maxDataWeightPerPart) { + if (curFmrTable != TFmrTableInputRef()) { + currentTask.Inputs.emplace_back(curFmrTable); + curFmrTable = TFmrTableInputRef(); + curTableId = range.TableId; + } + fmrTasks.emplace_back(currentTask); + currentTask = TTaskTableInputRef(); + curDataWeight = 0; + } + if (range.TableId != curTableId && curFmrTable != TFmrTableInputRef()) { + currentTask.Inputs.emplace_back(curFmrTable); + curFmrTable = TFmrTableInputRef(); + } + curTableId = range.TableId; + curFmrTable.TableId = curTableId; + curFmrTable.TableRanges.emplace_back(range.TableRange); + curDataWeight += range.DataWeight; + } + + currentTask.Inputs.emplace_back(curFmrTable); + fmrTasks.emplace_back(currentTask); +} + +bool TFmrPartitioner::CheckMaxTasksSize(const std::vector<TTaskTableInputRef>& currentFmrTasks) { + return currentFmrTasks.size() <= Settings_.MaxParts; +} + +TPartitionResult PartitionInputTablesIntoTasks( + const std::vector<TYtTableRef>& ytInputTables, + const std::vector<TFmrTableRef> fmrInputTables, + TFmrPartitioner& partitioner, + IYtCoordinatorService::TPtr ytCoordinatorService, + const std::unordered_map<TFmrTableId, TClusterConnection> &clusterConnections, + const TYtPartitionerSettings& ytPartitionSettings +) { + + std::vector<TTaskTableRef> tasks; + std::vector<TTaskTableInputRef> currentTasks; + + auto [gottenFmrTasks, fmrPartitionStatus] = partitioner.PartitionFmrTablesIntoTasks(fmrInputTables); + if (!fmrPartitionStatus) { + return TPartitionResult{.PartitionStatus = false}; + } + YQL_CLOG(INFO, FastMapReduce) << "Successfully partitioned input fmr tables into " << gottenFmrTasks.size() << " tasks"; + for (auto& fmrTask: gottenFmrTasks) { + YQL_CLOG(DEBUG, FastMapReduce) << fmrTask; + currentTasks.emplace_back(fmrTask); + } + if (ytInputTables.empty()) { + return TPartitionResult{.TaskInputs = currentTasks, .PartitionStatus = true}; + } + auto settings = ytPartitionSettings; + if (settings.MaxParts <= gottenFmrTasks.size()) { + return TPartitionResult{.PartitionStatus = false}; + } + settings.MaxParts = ytPartitionSettings.MaxParts - gottenFmrTasks.size(); + auto [gottenYtTasks, ytPartitionStatus] = ytCoordinatorService->PartitionYtTables(ytInputTables, clusterConnections, settings); + for (auto& ytTask: gottenYtTasks) { + currentTasks.emplace_back(TTaskTableInputRef{.Inputs = {ytTask}}); + } + YQL_CLOG(INFO, FastMapReduce) << "Gotten " << currentTasks.size() << " yt and fmr tasks to run from operation input tables"; + return TPartitionResult{.TaskInputs = currentTasks, .PartitionStatus = ytPartitionStatus}; +} + +} diff --git a/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_partitioner.h b/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_partitioner.h new file mode 100644 index 00000000000..d053ecbcfdd --- /dev/null +++ b/yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_partitioner.h @@ -0,0 +1,66 @@ +#include <yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h> +#include <yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface/yql_yt_coordinator_service_interface.h> + +namespace NYql::NFmr { + +struct TPartitionResult { + std::vector<TTaskTableInputRef> TaskInputs; + bool PartitionStatus = false; +}; + +struct TFmrPartitionerSettings { + ui64 MaxDataWeightPerPart = 0; + ui64 MaxParts = 0; +}; + +class TFmrPartitioner { +public: + TFmrPartitioner( + const std::unordered_map<TFmrTableId, std::vector<TString>>& partIdsForTables, + const std::unordered_map<TString, std::vector<TChunkStats>>& partIdStats, + const TFmrPartitionerSettings& settings + ); + + std::pair<std::vector<TTaskTableInputRef>, bool> PartitionFmrTablesIntoTasks(const std::vector<TFmrTableRef>& fmrTables); + +private: + struct TLeftoverRange { + TString TableId; + TTableRange TableRange; + ui64 DataWeight; + }; + + void HandleFmrPartition( + const TFmrTableId& fmrTable, + const TString& partId, + const std::vector<TChunkStats> stats, + ui64 maxDataWeightPerPart, + std::vector<TTaskTableInputRef>& currentFmrTasks, + std::vector<TLeftoverRange>& leftoverRanges + ); + + void HandleFmrLeftoverRanges( + ui64 maxDataWeightPerPart, + std::vector<TTaskTableInputRef>& fmrTasks, + std::vector<TLeftoverRange>& leftoverRanges + ); + + bool CheckMaxTasksSize(const std::vector<TTaskTableInputRef>& currentFmrTasks); + +private: + const std::unordered_map<TFmrTableId, std::vector<TString>> PartIdsForTables_; // TableId -> all corresponding part ids. + const std::unordered_map<TString, std::vector<TChunkStats>> PartIdStats_; // PartId -> statistics for all existing chunks in it. + const TFmrPartitionerSettings Settings_; +}; + +TPartitionResult PartitionInputTablesIntoTasks( + const std::vector<TYtTableRef>& ytInputTables, + const std::vector<TFmrTableRef> fmrInputTables, + TFmrPartitioner& partitioner, + IYtCoordinatorService::TPtr ytCoordinatorService, + const std::unordered_map<TFmrTableId, TClusterConnection> &clusterConnections, + const TYtPartitionerSettings& ytPartitionSettings +); + +} // namespace NYql::NFmr + diff --git a/yt/yql/providers/yt/fmr/coordinator/interface/proto_helpers/yql_yt_coordinator_proto_helpers.cpp b/yt/yql/providers/yt/fmr/coordinator/interface/proto_helpers/yql_yt_coordinator_proto_helpers.cpp index 1e2407b1ea5..d237adab447 100644 --- a/yt/yql/providers/yt/fmr/coordinator/interface/proto_helpers/yql_yt_coordinator_proto_helpers.cpp +++ b/yt/yql/providers/yt/fmr/coordinator/interface/proto_helpers/yql_yt_coordinator_proto_helpers.cpp @@ -13,6 +13,7 @@ NProto::THeartbeatRequest HeartbeatRequestToProto(const THeartbeatRequest& heart protoHeartbeatRequest.AddTaskStates(); protoHeartbeatRequest.MutableTaskStates(i)->Swap(&protoTaskState); } + protoHeartbeatRequest.SetAvailableSlots(heartbeatRequest.AvailableSlots); return protoHeartbeatRequest; } @@ -26,6 +27,7 @@ THeartbeatRequest HeartbeatRequestFromProto(const NProto::THeartbeatRequest prot taskStates.emplace_back(TIntrusivePtr<TTaskState>(new TTaskState(curTaskState))); } heartbeatRequest.TaskStates = taskStates; + heartbeatRequest.AvailableSlots = protoHeartbeatRequest.GetAvailableSlots(); return heartbeatRequest; } @@ -70,7 +72,7 @@ NProto::TStartOperationRequest StartOperationRequestToProto(const TStartOperatio protoStartOperationRequest.SetIdempotencyKey(*startOperationRequest.IdempotencyKey); } protoStartOperationRequest.SetNumRetries(startOperationRequest.NumRetries); - auto clusterConnections = *protoStartOperationRequest.MutableClusterConnections(); + auto& clusterConnections = *protoStartOperationRequest.MutableClusterConnections(); for (auto& [tableName, conn]: startOperationRequest.ClusterConnections) { clusterConnections[tableName.Id] = ClusterConnectionToProto(conn); } @@ -122,6 +124,11 @@ NProto::TGetOperationResponse GetOperationResponseToProto(const TGetOperationRes auto protoError = FmrErrorToProto(errorMessage); curError->Swap(&protoError); } + for (auto& tableStats: getOperationResponse.OutputTablesStats) { + auto* curTableStats = protoGetOperationResponse.AddTableStats(); + auto protoTableStats = TableStatsToProto(tableStats); + curTableStats->Swap(&protoTableStats); + } return protoGetOperationResponse; } @@ -129,11 +136,17 @@ TGetOperationResponse GetOperationResponseFromProto(const NProto::TGetOperationR TGetOperationResponse getOperationResponse; getOperationResponse.Status = static_cast<EOperationStatus>(protoGetOperationReponse.GetStatus()); std::vector<TFmrError> errorMessages; + std::vector<TTableStats> outputTableStats; for (size_t i = 0; i < protoGetOperationReponse.ErrorMessagesSize(); ++i) { TFmrError errorMessage = FmrErrorFromProto(protoGetOperationReponse.GetErrorMessages(i)); errorMessages.emplace_back(errorMessage); } + for (size_t i = 0; i < protoGetOperationReponse.TableStatsSize(); ++i) { + TTableStats tableStats = TableStatsFromProto(protoGetOperationReponse.GetTableStats(i)); + outputTableStats.emplace_back(tableStats); + } getOperationResponse.ErrorMessages = errorMessages; + getOperationResponse.OutputTablesStats = outputTableStats; return getOperationResponse; } diff --git a/yt/yql/providers/yt/fmr/coordinator/interface/yql_yt_coordinator.h b/yt/yql/providers/yt/fmr/coordinator/interface/yql_yt_coordinator.h index a42cb0d35a6..be016c230b2 100644 --- a/yt/yql/providers/yt/fmr/coordinator/interface/yql_yt_coordinator.h +++ b/yt/yql/providers/yt/fmr/coordinator/interface/yql_yt_coordinator.h @@ -11,6 +11,7 @@ struct THeartbeatRequest { ui32 WorkerId; TString VolatileId; std::vector<TTaskState::TPtr> TaskStates; + ui64 AvailableSlots = 0; }; // Worker sends requests in loop or long polling @@ -57,7 +58,7 @@ struct TGetFmrTableInfoRequest { }; struct TGetFmrTableInfoResponse { - TTableStats TableStats; // for only one PartId + TTableStats TableStats; std::vector<TFmrError> ErrorMessages = {}; }; diff --git a/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/ut/ya.make b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/ut/ya.make new file mode 100644 index 00000000000..0f2075ad1b3 --- /dev/null +++ b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/ut/ya.make @@ -0,0 +1,13 @@ +UNITTEST() + +SRCS( + yql_yt_coordinator_service_ut.cpp +) + +PEERDIR( + yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/ut/yql_yt_coordinator_service_ut.cpp b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/ut/yql_yt_coordinator_service_ut.cpp new file mode 100644 index 00000000000..75ade8a9e05 --- /dev/null +++ b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/ut/yql_yt_coordinator_service_ut.cpp @@ -0,0 +1,43 @@ +#include <library/cpp/testing/unittest/registar.h> +#include <util/stream/file.h> +#include <util/system/tempfile.h> +#include <yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/yql_yt_file_coordinator_service.h> + +namespace NYql::NFmr { + +Y_UNIT_TEST_SUITE(FileYtCoordinatorServiceTests) { + Y_UNIT_TEST(PartitionFiles) { + const i64 FileNums = 5; + + std::vector<THolder<TTempFileHandle>> fileHandles(FileNums); + std::vector<TYtTableRef> ytTables(FileNums, TYtTableRef()); + std::vector<i64> fileLengths = {30, 10, 20, 5, 40}; + + for (int i = 0; i < FileNums; ++i) { + fileHandles[i] = MakeHolder<TTempFileHandle>(); + auto curFileName= fileHandles[i]->Name(); + ytTables[i].FilePath = curFileName; + TFileOutput writer(curFileName); + writer.Write(TString("1") * fileLengths[i]); + writer.Flush(); + } + + auto fileService = MakeFileYtCoordinatorService(); + auto settings = TYtPartitionerSettings{.MaxDataWeightPerPart = 50, .MaxParts = 100}; + auto [gottenPartitions, status] = fileService->PartitionYtTables(ytTables, {}, settings); + UNIT_ASSERT_VALUES_EQUAL(status, true); + + std::vector<std::vector<TString>> expectedFilePartitions = { + {fileHandles[0]->Name(), fileHandles[1]->Name()}, + {fileHandles[2]->Name(), fileHandles[3]->Name()}, + {fileHandles[4]->Name()} + }; + std::vector<std::vector<TString>> gottenFileParititons; + for (auto& part: gottenPartitions) { + gottenFileParititons.emplace_back(part.FilePaths); + } + UNIT_ASSERT_VALUES_EQUAL(gottenFileParititons, expectedFilePartitions); + } +} + +} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/ya.make b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/ya.make new file mode 100644 index 00000000000..b307ab11fae --- /dev/null +++ b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/ya.make @@ -0,0 +1,16 @@ +LIBRARY() + +SRCS( + yql_yt_file_coordinator_service.cpp +) + +PEERDIR( + yt/cpp/mapreduce/common + yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface +) + +YQL_LAST_ABI_VERSION() + +END() + +RECURSE_FOR_TESTS(ut) diff --git a/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/yql_yt_file_coordinator_service.cpp b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/yql_yt_file_coordinator_service.cpp new file mode 100644 index 00000000000..1d5dfb2d019 --- /dev/null +++ b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/yql_yt_file_coordinator_service.cpp @@ -0,0 +1,59 @@ +#include "yql_yt_file_coordinator_service.h" + +#include <library/cpp/yson/parser.h> +#include <util/stream/file.h> +#include <util/system/fstat.h> +#include <yt/cpp/mapreduce/common/helpers.h> +#include <yt/yql/providers/yt/gateway/file/yql_yt_file_text_yson.h> +#include <yt/yql/providers/yt/lib/yson_helpers/yson_helpers.h> +#include <yql/essentials/utils/yql_panic.h> + +namespace NYql::NFmr { + +namespace { + +class TFileYtCoordinatorService: public IYtCoordinatorService { +public: + + std::pair<std::vector<TYtTableTaskRef>, bool> PartitionYtTables( + const std::vector<TYtTableRef>& ytTables, + const std::unordered_map<TFmrTableId, TClusterConnection>& /*clusterConnections*/, + const TYtPartitionerSettings& settings + ) override { + const i64 maxDataWeightPerPart = settings.MaxDataWeightPerPart; + std::vector<TYtTableTaskRef> ytPartitions; + TYtTableTaskRef curYtTableTaskRef{}; + i64 curFileLength = 0; + for (auto& ytTable: ytTables) { + YQL_ENSURE(ytTable.FilePath); + auto fileLength = GetFileLength(*ytTable.FilePath); + if (fileLength + curFileLength > maxDataWeightPerPart) { + ytPartitions.emplace_back(curYtTableTaskRef); + if (ytPartitions.size() > settings.MaxParts) { + return {{}, false}; + } + curYtTableTaskRef = TYtTableTaskRef{}; + curFileLength = 0; + } + TString ytPath = NYT::AddPathPrefix(ytTable.Path, "//"); + auto richPath = NYT::TRichYPath(ytPath).Append(true); + // append RichPath just in case, TODO - figure out if we actually need to use it somewhere + curYtTableTaskRef.RichPaths.emplace_back(richPath); + curYtTableTaskRef.FilePaths.emplace_back(*ytTable.FilePath); + curFileLength += fileLength; + } + ytPartitions.emplace_back(curYtTableTaskRef); + if (ytPartitions.size() > settings.MaxParts) { + return {{}, false}; + } + return {ytPartitions, true}; + } +}; + +} // namespace + +IYtCoordinatorService::TPtr MakeFileYtCoordinatorService() { + return MakeIntrusive<TFileYtCoordinatorService>(); +} + +} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/yql_yt_file_coordinator_service.h b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/yql_yt_file_coordinator_service.h new file mode 100644 index 00000000000..7d5ca59e547 --- /dev/null +++ b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/yql_yt_file_coordinator_service.h @@ -0,0 +1,7 @@ +#include <yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface/yql_yt_coordinator_service_interface.h> + +namespace NYql::NFmr { + +IYtCoordinatorService::TPtr MakeFileYtCoordinatorService(); + +} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/impl/ya.make b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/impl/ya.make new file mode 100644 index 00000000000..bea26d3843a --- /dev/null +++ b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/impl/ya.make @@ -0,0 +1,13 @@ +LIBRARY() + +SRCS( + yql_yt_coordinator_service_impl.cpp +) + +PEERDIR( + yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/impl/yql_yt_coordinator_service_impl.cpp b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/impl/yql_yt_coordinator_service_impl.cpp new file mode 100644 index 00000000000..bb556251ed9 --- /dev/null +++ b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/impl/yql_yt_coordinator_service_impl.cpp @@ -0,0 +1,96 @@ +#include "yql_yt_coordinator_service_impl.h" + +#include <library/cpp/yt/error/error.h> +#include <yt/cpp/mapreduce/common/helpers.h> +#include <yt/yql/providers/yt/fmr/utils/yql_yt_client.h> +#include <yql/essentials/utils/log/log.h> +#include <yql/essentials/utils/yql_panic.h> + +namespace NYql::NFmr { + +namespace { + +class TYtCoordinatorService: public IYtCoordinatorService { +public: + + std::pair<std::vector<TYtTableTaskRef>, bool> PartitionYtTables( + const std::vector<TYtTableRef>& ytTables, + const std::unordered_map<TFmrTableId, TClusterConnection>& clusterConnections, + const TYtPartitionerSettings& settings + ) override { + auto getTablePartitionsOptions = NYT::TGetTablePartitionsOptions() + .PartitionMode(NYT::ETablePartitionMode::Unordered) + .DataWeightPerPartition(settings.MaxDataWeightPerPart) + .MaxPartitionCount(settings.MaxParts) + .AdjustDataWeightPerPartition(false); // TODO - add adjust data weight into partitioner settings + + std::vector<TYtTableTaskRef> ytPartitions; + auto groupedYtTables = GroupYtTables(ytTables, clusterConnections); + for (auto& [ytTables, clusterConnection]: groupedYtTables) { + auto client = CreateClient(clusterConnection); + auto transaction = client->AttachTransaction(GetGuid(clusterConnection.TransactionId)); + TVector<NYT::TRichYPath> richPaths; + for (auto& ytTable: ytTables ) { + TString ytPath = NYT::AddPathPrefix(ytTable.Path, "//"); + richPaths.emplace_back(NYT::TRichYPath(ytPath).Cluster(ytTable.Cluster)); + } + try { + NYT::TMultiTablePartitions partitions = transaction->GetTablePartitions(richPaths, getTablePartitionsOptions); + + for (const auto& partition : partitions.Partitions) { + TYtTableTaskRef ytTableTaskRef{}; + for (const auto& richPath : partition.TableRanges) { + ytTableTaskRef.RichPaths.emplace_back(richPath); + } + ytPartitions.emplace_back(ytTableTaskRef); + } + } catch (NYT::TErrorException& ex) { + YQL_CLOG(INFO, FastMapReduce) << "Failed to partition yt tables with message: " << CurrentExceptionMessage(); + return {{}, false}; + } + } + YQL_CLOG(INFO, FastMapReduce) << "partitioned input yt tables into " << ytPartitions.size() << " tasks"; + for (auto& task: ytPartitions) { + YQL_CLOG(DEBUG, FastMapReduce) << task; + } + return {ytPartitions, true}; + } + +private: + struct TGroupedYtTablesByCluster { + std::vector<TYtTableRef> YtTables; + TClusterConnection ClusterConnection; + }; + + std::vector<TGroupedYtTablesByCluster> GroupYtTables( + const std::vector<TYtTableRef>& ytTables, + const std::unordered_map<TFmrTableId, TClusterConnection>& clusterConnections + ) { + std::vector<TGroupedYtTablesByCluster> tableGroups; + std::unordered_map<TString, ui64> ytServerToGroups; + for (auto& ytTable: ytTables) { + auto fmrTableId = TFmrTableId(ytTable.Cluster, ytTable.Path); + auto clusterConnection = clusterConnections.at(fmrTableId); + auto ytServerName = clusterConnection.YtServerName; + if (!ytServerToGroups.contains(ytServerName)) { + tableGroups.emplace_back(TGroupedYtTablesByCluster{ + .YtTables = {ytTable}, + .ClusterConnection = clusterConnection + }); + ytServerToGroups[ytServerName] = tableGroups.size() - 1; + } else { + auto index = ytServerToGroups[ytServerName]; + tableGroups[index].YtTables.emplace_back(ytTable); + } + } + return tableGroups; + } +}; + +} // namespace + +IYtCoordinatorService::TPtr MakeYtCoordinatorService() { + return MakeIntrusive<TYtCoordinatorService>(); +} + +} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/impl/yql_yt_coordinator_service_impl.h b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/impl/yql_yt_coordinator_service_impl.h new file mode 100644 index 00000000000..44a95233140 --- /dev/null +++ b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/impl/yql_yt_coordinator_service_impl.h @@ -0,0 +1,7 @@ +#include <yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface/yql_yt_coordinator_service_interface.h> + +namespace NYql::NFmr { + +IYtCoordinatorService::TPtr MakeYtCoordinatorService(); + +} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface/ya.make b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface/ya.make new file mode 100644 index 00000000000..612f56f02d0 --- /dev/null +++ b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface/ya.make @@ -0,0 +1,14 @@ +LIBRARY() + +SRCS( + yql_yt_coordinator_service_interface.cpp +) + +PEERDIR( + yt/cpp/mapreduce/interface + yt/yql/providers/yt/fmr/request_options +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface/yql_yt_coordinator_service_interface.cpp b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface/yql_yt_coordinator_service_interface.cpp new file mode 100644 index 00000000000..450c685eced --- /dev/null +++ b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface/yql_yt_coordinator_service_interface.cpp @@ -0,0 +1 @@ +#include "yql_yt_coordinator_service_interface.h" diff --git a/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface/yql_yt_coordinator_service_interface.h b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface/yql_yt_coordinator_service_interface.h new file mode 100644 index 00000000000..13c01f5d8a8 --- /dev/null +++ b/yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/interface/yql_yt_coordinator_service_interface.h @@ -0,0 +1,25 @@ +#pragma once + +#include <yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h> + +namespace NYql::NFmr { + +struct TYtPartitionerSettings { + ui64 MaxDataWeightPerPart = 0; + ui64 MaxParts = 0; +}; + +class IYtCoordinatorService: public TThrRefBase { +public: + virtual ~IYtCoordinatorService() = default; + + using TPtr = TIntrusivePtr<IYtCoordinatorService>; + + virtual std::pair<std::vector<TYtTableTaskRef>, bool> PartitionYtTables( + const std::vector<TYtTableRef>& ytTables, + const std::unordered_map<TFmrTableId, TClusterConnection>& clusterConnections, + const TYtPartitionerSettings& settings + ) = 0; +}; + +} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/fmr_tool_lib/ya.make b/yt/yql/providers/yt/fmr/fmr_tool_lib/ya.make index b0c5e5cdf4f..dbada26432e 100644 --- a/yt/yql/providers/yt/fmr/fmr_tool_lib/ya.make +++ b/yt/yql/providers/yt/fmr/fmr_tool_lib/ya.make @@ -8,12 +8,14 @@ PEERDIR( yt/yql/providers/yt/gateway/fmr yt/yql/providers/yt/fmr/coordinator/client yt/yql/providers/yt/fmr/coordinator/impl + yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file + yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/impl yt/yql/providers/yt/fmr/job/impl yt/yql/providers/yt/fmr/job_factory/impl yt/yql/providers/yt/fmr/table_data_service/local yt/yql/providers/yt/fmr/worker/impl - yt/yql/providers/yt/fmr/yt_service/file - yt/yql/providers/yt/fmr/yt_service/impl + yt/yql/providers/yt/fmr/yt_job_service/file + yt/yql/providers/yt/fmr/yt_job_service/impl ) YQL_LAST_ABI_VERSION() diff --git a/yt/yql/providers/yt/fmr/fmr_tool_lib/yql_yt_fmr_initializer.cpp b/yt/yql/providers/yt/fmr/fmr_tool_lib/yql_yt_fmr_initializer.cpp index 1d6bc504a5e..7849baa2517 100644 --- a/yt/yql/providers/yt/fmr/fmr_tool_lib/yql_yt_fmr_initializer.cpp +++ b/yt/yql/providers/yt/fmr/fmr_tool_lib/yql_yt_fmr_initializer.cpp @@ -11,7 +11,7 @@ std::pair<IYtGateway::TPtr, IFmrWorker::TPtr> InitializeFmrGateway(IYtGateway::T coordinatorSettings.DefaultFmrOperationSpec = fmrOperationSpec; } - auto coordinator = MakeFmrCoordinator(coordinatorSettings); + auto coordinator = isFileGateway ? MakeFmrCoordinator(coordinatorSettings, MakeFileYtCoordinatorService()) : MakeFmrCoordinator(coordinatorSettings, MakeYtCoordinatorService()); if (!coordinatorServerUrl.empty()) { TFmrCoordinatorClientSettings coordinatorClientSettings; THttpURL parsedUrl; @@ -26,10 +26,10 @@ std::pair<IYtGateway::TPtr, IFmrWorker::TPtr> InitializeFmrGateway(IYtGateway::T IFmrWorker::TPtr worker = nullptr; if (!disableLocalFmrWorker) { auto tableDataService = MakeLocalTableDataService(TLocalTableDataServiceSettings(3)); - auto fmrYtSerivce = isFileGateway ? MakeFileYtSerivce() : MakeFmrYtSerivce(); + auto fmrYtJobSerivce = isFileGateway ? MakeFileYtJobSerivce() : MakeYtJobSerivce(); - auto func = [tableDataService, fmrYtSerivce] (NFmr::TTask::TPtr task, std::shared_ptr<std::atomic<bool>> cancelFlag) mutable { - return RunJob(task, tableDataService, fmrYtSerivce, cancelFlag); + auto func = [tableDataService, fmrYtJobSerivce] (NFmr::TTask::TPtr task, std::shared_ptr<std::atomic<bool>> cancelFlag) mutable { + return RunJob(task, tableDataService, fmrYtJobSerivce, cancelFlag); }; NFmr::TFmrJobFactorySettings settings{.Function=func}; diff --git a/yt/yql/providers/yt/fmr/fmr_tool_lib/yql_yt_fmr_initializer.h b/yt/yql/providers/yt/fmr/fmr_tool_lib/yql_yt_fmr_initializer.h index 24f17928e17..743f856eb8c 100644 --- a/yt/yql/providers/yt/fmr/fmr_tool_lib/yql_yt_fmr_initializer.h +++ b/yt/yql/providers/yt/fmr/fmr_tool_lib/yql_yt_fmr_initializer.h @@ -4,11 +4,14 @@ #include <yt/yql/providers/yt/fmr/worker/impl/yql_yt_worker_impl.h> #include <yt/yql/providers/yt/fmr/coordinator/client/yql_yt_coordinator_client.h> #include <yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.h> +#include <yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file/yql_yt_file_coordinator_service.h> +#include <yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/impl/yql_yt_coordinator_service_impl.h> #include <yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.h> #include <yt/yql/providers/yt/fmr/job_factory/impl/yql_yt_job_factory_impl.h> #include <yt/yql/providers/yt/fmr/table_data_service/local/yql_yt_table_data_service_local.h> -#include <yt/yql/providers/yt/fmr/yt_service/file/yql_yt_file_yt_service.h> -#include <yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.h> +#include <yt/yql/providers/yt/fmr/yt_job_service/file/yql_yt_file_yt_job_service.h> +#include <yt/yql/providers/yt/fmr/yt_job_service/impl/yql_yt_job_service_impl.h> + namespace NYql::NFmr { diff --git a/yt/yql/providers/yt/fmr/job/impl/ut/ya.make b/yt/yql/providers/yt/fmr/job/impl/ut/ya.make index a997d1c1983..6b372d758b3 100644 --- a/yt/yql/providers/yt/fmr/job/impl/ut/ya.make +++ b/yt/yql/providers/yt/fmr/job/impl/ut/ya.make @@ -7,8 +7,9 @@ SRCS( ) PEERDIR( + yt/cpp/mapreduce/common yt/yql/providers/yt/fmr/job/impl - yt/yql/providers/yt/fmr/yt_service/mock + yt/yql/providers/yt/fmr/yt_job_service/mock yt/yql/providers/yt/fmr/table_data_service/local yql/essentials/utils/log yql/essentials/parser/pg_wrapper diff --git a/yt/yql/providers/yt/fmr/job/impl/ut/yql_yt_job_ut.cpp b/yt/yql/providers/yt/fmr/job/impl/ut/yql_yt_job_ut.cpp index 6b9ce068652..614fe30a44d 100644 --- a/yt/yql/providers/yt/fmr/job/impl/ut/yql_yt_job_ut.cpp +++ b/yt/yql/providers/yt/fmr/job/impl/ut/yql_yt_job_ut.cpp @@ -1,8 +1,9 @@ #include <library/cpp/testing/unittest/registar.h> +#include <yt/cpp/mapreduce/common/helpers.h> #include <yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.h> #include <yt/yql/providers/yt/fmr/table_data_service/local/yql_yt_table_data_service_local.h> #include <yt/yql/providers/yt/fmr/utils/yql_yt_table_data_service_key.h> -#include <yt/yql/providers/yt/fmr/yt_service/mock/yql_yt_yt_service_mock.h> +#include <yt/yql/providers/yt/fmr/yt_job_service/mock/yql_yt_job_service_mock.h> namespace NYql::NFmr { @@ -36,12 +37,13 @@ TString GetTextYson(const TString& binaryYsonContent) { Y_UNIT_TEST_SUITE(FmrJobTests) { Y_UNIT_TEST(DownloadTable) { ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1)); - TYtTableRef input = TYtTableRef("test_cluster", "test_path"); - std::unordered_map<TYtTableRef, TString> inputTables{{input, TableContent_1}}; + auto richPath = NYT::TRichYPath("//test_path").Cluster("test_cluster"); + TYtTableTaskRef input = TYtTableTaskRef{.RichPaths = {richPath}}; + std::unordered_map<TString, TString> inputTables{{NYT::NodeToCanonicalYsonString(NYT::PathToNode(richPath)), TableContent_1}}; std::unordered_map<TYtTableRef, TString> outputTables; - NYql::NFmr::IYtService::TPtr ytService = MakeMockYtService(inputTables, outputTables); + NYql::NFmr::IYtJobService::TPtr ytJobService = MakeMockYtJobService(inputTables, outputTables); std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false); - IFmrJob::TPtr job = MakeFmrJob(tableDataServicePtr, ytService); + IFmrJob::TPtr job = MakeFmrJob(tableDataServicePtr, ytJobService); TFmrTableOutputRef output = TFmrTableOutputRef("test_table_id", "test_part_id"); TDownloadTaskParams params = TDownloadTaskParams(input, output); @@ -53,7 +55,10 @@ Y_UNIT_TEST_SUITE(FmrJobTests) { auto statistics = std::get_if<TStatistics>(&res); UNIT_ASSERT_C(!err, err->ErrorMessage); - UNIT_ASSERT_EQUAL(statistics->OutputTables.at(output).Rows, 4); + auto detailedChunkStats = statistics->OutputTables.at(output).PartIdChunkStats; + UNIT_ASSERT_VALUES_EQUAL(detailedChunkStats.size(), 1); // coordinator settings taken from file with default values, so large chunk size + UNIT_ASSERT_VALUES_EQUAL(detailedChunkStats[0].Rows, 4); + auto resultTableContent = tableDataServicePtr->Get(tableDataServiceExpectedOutputKey).GetValueSync(); UNIT_ASSERT_C(resultTableContent, "Result table content is empty"); UNIT_ASSERT_NO_DIFF(*resultTableContent, GetBinaryYson(TableContent_1)); @@ -61,11 +66,12 @@ Y_UNIT_TEST_SUITE(FmrJobTests) { Y_UNIT_TEST(UploadTable) { ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1)); - std::unordered_map<TYtTableRef, TString> inputTables, outputTables; - NYql::NFmr::IYtService::TPtr ytService = MakeMockYtService(inputTables, outputTables); + std::unordered_map<TString, TString> inputTables; + std::unordered_map<TYtTableRef, TString> outputTables; + NYql::NFmr::IYtJobService::TPtr ytJobService = MakeMockYtJobService(inputTables, outputTables); std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false); - IFmrJob::TPtr job = MakeFmrJob(tableDataServicePtr, ytService); + IFmrJob::TPtr job = MakeFmrJob(tableDataServicePtr, ytJobService); TYtTableRef output = TYtTableRef("test_cluster", "test_path"); std::vector<TTableRange> ranges = {{"test_part_id"}}; @@ -89,20 +95,21 @@ Y_UNIT_TEST_SUITE(FmrJobTests) { std::vector<TTableRange> ranges = {{"test_part_id"}}; TFmrTableInputRef input_1 = TFmrTableInputRef{.TableId = "test_table_id_1", .TableRanges = ranges}; - TYtTableRef input_2 = TYtTableRef("test_path", "test_cluster"); + auto richPath = NYT::TRichYPath("//test_path").Cluster("test_cluster"); + TYtTableTaskRef input_2 = TYtTableTaskRef{.RichPaths = {richPath}}; TFmrTableInputRef input_3 = TFmrTableInputRef{.TableId = "test_table_id_3", .TableRanges = ranges}; - std::unordered_map<TYtTableRef, TString> inputTables{{input_2, TableContent_2}}; + std::unordered_map<TString, TString> inputTables{{NYT::NodeToCanonicalYsonString(NYT::PathToNode(richPath)), TableContent_2}}; std::unordered_map<TYtTableRef, TString> outputTables; - NYql::NFmr::IYtService::TPtr ytService = MakeMockYtService(inputTables, outputTables); + NYql::NFmr::IYtJobService::TPtr ytJobService = MakeMockYtJobService(inputTables, outputTables); auto cancelFlag = std::make_shared<std::atomic<bool>>(false); - IFmrJob::TPtr job = MakeFmrJob(tableDataServicePtr, ytService); + IFmrJob::TPtr job = MakeFmrJob(tableDataServicePtr, ytJobService); TTaskTableRef input_table_ref_1 = {input_1}; TTaskTableRef input_table_ref_2 = {input_2}; TTaskTableRef input_table_ref_3 = {input_3}; TFmrTableOutputRef output = TFmrTableOutputRef("test_table_id_output", "test_part_id"); std::vector<TTaskTableRef> inputs = {input_table_ref_1, input_table_ref_2, input_table_ref_3}; - auto params = TMergeTaskParams(inputs, output); + auto params = TMergeTaskParams{.Input = TTaskTableInputRef{.Inputs = inputs}, .Output = output}; auto tableDataServiceExpectedOutputKey = GetTableDataServiceKey(output.TableId, output.PartId, 0); auto key_1 = GetTableDataServiceKey(input_1.TableId, "test_part_id", 0); @@ -130,17 +137,19 @@ Y_UNIT_TEST_SUITE(FmrJobTests) { Y_UNIT_TEST_SUITE(TaskRunTests) { Y_UNIT_TEST(RunDownloadTask) { ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1)); - TYtTableRef input = TYtTableRef("test_cluster", "test_path"); - std::unordered_map<TYtTableRef, TString> inputTables{{input, TableContent_1}}; + auto richPath = NYT::TRichYPath("//test_path").Cluster("test_cluster"); + TYtTableTaskRef input = TYtTableTaskRef{.RichPaths = {richPath}}; + TFmrTableId inputFmrId("test_cluster", "test_path"); + std::unordered_map<TString, TString> inputTables{{NYT::NodeToCanonicalYsonString(NYT::PathToNode(richPath)), TableContent_1}}; std::unordered_map<TYtTableRef, TString> outputTables; - NYql::NFmr::IYtService::TPtr ytService = MakeMockYtService(inputTables, outputTables); + NYql::NFmr::IYtJobService::TPtr ytJobService = MakeMockYtJobService(inputTables, outputTables); std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false); TFmrTableOutputRef output = TFmrTableOutputRef("test_table_id", "test_part_id"); auto tableDataServiceExpectedOutputKey = GetTableDataServiceKey(output.TableId, output.PartId, 0); TDownloadTaskParams params = TDownloadTaskParams(input, output); TTask::TPtr task = MakeTask(ETaskType::Download, "test_task_id", params, "test_session_id", {{TFmrTableId("test_cluster", "test_path"), TClusterConnection()}}); - ETaskStatus status = RunJob(task, tableDataServicePtr, ytService, cancelFlag).TaskStatus; + ETaskStatus status = RunJob(task, tableDataServicePtr, ytJobService, cancelFlag).TaskStatus; UNIT_ASSERT_EQUAL(status, ETaskStatus::Completed); auto resultTableContent = tableDataServicePtr->Get(tableDataServiceExpectedOutputKey).GetValueSync(); @@ -151,8 +160,9 @@ Y_UNIT_TEST_SUITE(TaskRunTests) { Y_UNIT_TEST(RunUploadTask) { ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1)); - std::unordered_map<TYtTableRef, TString> inputTables, outputTables; - NYql::NFmr::IYtService::TPtr ytService = MakeMockYtService(inputTables, outputTables); + std::unordered_map<TString, TString> inputTables; + std::unordered_map<TYtTableRef, TString> outputTables; + NYql::NFmr::IYtJobService::TPtr ytJobService = MakeMockYtJobService(inputTables, outputTables); std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false); std::vector<TTableRange> ranges = {{"test_part_id"}}; @@ -163,7 +173,7 @@ Y_UNIT_TEST_SUITE(TaskRunTests) { TTask::TPtr task = MakeTask(ETaskType::Upload, "test_task_id", params, "test_session_id", {{TFmrTableId("test_cluster", "test_path"), TClusterConnection()}}); auto key = GetTableDataServiceKey(input.TableId, "test_part_id", 0); tableDataServicePtr->Put(key, GetBinaryYson(TableContent_1)); - ETaskStatus status = RunJob(task, tableDataServicePtr, ytService, cancelFlag).TaskStatus; + ETaskStatus status = RunJob(task, tableDataServicePtr, ytJobService, cancelFlag).TaskStatus; UNIT_ASSERT_EQUAL(status, ETaskStatus::Completed); UNIT_ASSERT(outputTables.contains(output)); @@ -172,8 +182,9 @@ Y_UNIT_TEST_SUITE(TaskRunTests) { Y_UNIT_TEST(RunUploadTaskWithNoTable) { ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1)); - std::unordered_map<TYtTableRef, TString> inputTables, outputTables; - NYql::NFmr::IYtService::TPtr ytService = MakeMockYtService(inputTables, outputTables); + std::unordered_map<TString, TString> inputTables; + std::unordered_map<TYtTableRef, TString> outputTables; + NYql::NFmr::IYtJobService::TPtr ytJobService = MakeMockYtJobService(inputTables, outputTables); std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false); std::vector<TTableRange> ranges = {{"test_part_id"}}; @@ -185,7 +196,7 @@ Y_UNIT_TEST_SUITE(TaskRunTests) { // No tables in tableDataService UNIT_ASSERT_EXCEPTION_CONTAINS( - RunJob(task, tableDataServicePtr, ytService, cancelFlag), + RunJob(task, tableDataServicePtr, ytJobService, cancelFlag), yexception, "No data for chunk:test_table_id:test_part_id" ); @@ -195,11 +206,13 @@ Y_UNIT_TEST_SUITE(TaskRunTests) { ITableDataService::TPtr tableDataServicePtr = MakeLocalTableDataService(TLocalTableDataServiceSettings(1)); std::vector<TTableRange> ranges = {{"test_part_id"}}; TFmrTableInputRef input_1 = TFmrTableInputRef{.TableId = "test_table_id_1", .TableRanges = ranges}; - TYtTableRef input_2 = TYtTableRef("test_path", "test_cluster"); + auto richPath = NYT::TRichYPath("//test_path").Cluster("test_cluster"); + TYtTableTaskRef input_2 = TYtTableTaskRef{.RichPaths = {richPath}}; + TFmrTableId inputFmrId_2("test_cluster", "test_path"); TFmrTableInputRef input_3 = TFmrTableInputRef{.TableId = "test_table_id_3", .TableRanges = ranges}; - std::unordered_map<TYtTableRef, TString> inputTables{{input_2, TableContent_2}}; + std::unordered_map<TString, TString> inputTables{{NYT::NodeToCanonicalYsonString(NYT::PathToNode(richPath)), TableContent_2}}; std::unordered_map<TYtTableRef, TString> outputTables; - NYql::NFmr::IYtService::TPtr ytService = MakeMockYtService(inputTables, outputTables); + NYql::NFmr::IYtJobService::TPtr ytJobService = MakeMockYtJobService(inputTables, outputTables); std::shared_ptr<std::atomic<bool>> cancelFlag = std::make_shared<std::atomic<bool>>(false); TTaskTableRef input_table_ref_1 = {input_1}; @@ -207,7 +220,7 @@ Y_UNIT_TEST_SUITE(TaskRunTests) { TTaskTableRef input_table_ref_3 = {input_3}; TFmrTableOutputRef output = TFmrTableOutputRef("test_table_id_output", "test_part_id"); std::vector<TTaskTableRef> inputs = {input_table_ref_1, input_table_ref_2, input_table_ref_3}; - auto params = TMergeTaskParams(inputs, output); + auto params = TMergeTaskParams{.Input = TTaskTableInputRef{.Inputs = inputs}, .Output = output}; auto tableDataServiceExpectedOutputKey = GetTableDataServiceKey(output.TableId, output.PartId, 0); TTask::TPtr task = MakeTask(ETaskType::Merge, "test_task_id", params, "test_session_id", {{TFmrTableId("test_cluster", "test_path"), TClusterConnection()}}); @@ -217,7 +230,7 @@ Y_UNIT_TEST_SUITE(TaskRunTests) { tableDataServicePtr->Put(key_1, GetBinaryYson(TableContent_1)); tableDataServicePtr->Put(key_3, GetBinaryYson(TableContent_3)); - ETaskStatus status = RunJob(task, tableDataServicePtr, ytService, cancelFlag).TaskStatus; + ETaskStatus status = RunJob(task, tableDataServicePtr, ytJobService, cancelFlag).TaskStatus; UNIT_ASSERT_EQUAL(status, ETaskStatus::Completed); auto resultTableContentMaybe = tableDataServicePtr->Get(tableDataServiceExpectedOutputKey).GetValueSync(); diff --git a/yt/yql/providers/yt/fmr/job/impl/ut/yql_yt_table_data_service_writer_ut.cpp b/yt/yql/providers/yt/fmr/job/impl/ut/yql_yt_table_data_service_writer_ut.cpp index 925cdfff125..4d1a9860e48 100644 --- a/yt/yql/providers/yt/fmr/job/impl/ut/yql_yt_table_data_service_writer_ut.cpp +++ b/yt/yql/providers/yt/fmr/job/impl/ut/yql_yt_table_data_service_writer_ut.cpp @@ -12,7 +12,7 @@ const std::vector<TString> TableYsonRows = { "{\"key\"=\"150\";\"subkey\"=\"4\";\"value\"=\"qzz\"};" }; -TTableStats WriteDataToTableDataSerice( +TTableChunkStats WriteDataToTableDataSerice( ITableDataService::TPtr tableDataService, const std::vector<TString>& tableYsonRows, ui64 chunkSize, @@ -34,18 +34,28 @@ TTableStats WriteDataToTableDataSerice( Y_UNIT_TEST_SUITE(FmrWriterTests) { Y_UNIT_TEST(WriteYsonRows) { - ui64 totalSize = 0; - for (auto& row: TableYsonRows) { + ui64 totalSize = 0, firstPartSize = 0, secPartSize = 0; + for (ui64 i = 0; i < TableYsonRows.size(); ++i) { + auto& row = TableYsonRows[i]; totalSize += row.size(); + if (i < 2) { + firstPartSize += row.size(); + } else { + secPartSize += row.size(); + } } ui64 chunkSize = totalSize / 2; ITableDataService::TPtr tableDataService = MakeLocalTableDataService(TLocalTableDataServiceSettings(1)); + auto stats = WriteDataToTableDataSerice(tableDataService, TableYsonRows, chunkSize); - auto realChunks = stats.Chunks; - auto realDataWeight =stats.DataWeight; - UNIT_ASSERT_VALUES_EQUAL(realChunks, 2); - UNIT_ASSERT_VALUES_EQUAL(realDataWeight, totalSize); + UNIT_ASSERT_VALUES_EQUAL(stats.PartId, "partId"); + std::vector<TChunkStats> gottenPartIdChunkStats = stats.PartIdChunkStats; + std::vector<TChunkStats> expectedChunkStats = { + TChunkStats{.Rows = 2, .DataWeight = firstPartSize}, + TChunkStats{.Rows = 2, .DataWeight = secPartSize} + }; + UNIT_ASSERT(gottenPartIdChunkStats == expectedChunkStats); TString expectedFirstChunkTableContent = JoinRange(TStringBuf(), TableYsonRows.begin(), TableYsonRows.begin() + 2); TString expectedSecondChunkTableContent = JoinRange(TStringBuf(), TableYsonRows.begin() + 2, TableYsonRows.end()); diff --git a/yt/yql/providers/yt/fmr/job/impl/ya.make b/yt/yql/providers/yt/fmr/job/impl/ya.make index 02b5058984a..dc29cebd265 100644 --- a/yt/yql/providers/yt/fmr/job/impl/ya.make +++ b/yt/yql/providers/yt/fmr/job/impl/ya.make @@ -11,10 +11,10 @@ PEERDIR( library/cpp/yson/node yt/cpp/mapreduce/interface yt/yql/providers/yt/fmr/job/interface - yt/yql/providers/yt/fmr/request_options yt/yql/providers/yt/fmr/utils yt/yql/providers/yt/fmr/table_data_service/interface yql/essentials/utils + yql/essentials/utils/log ) YQL_LAST_ABI_VERSION() diff --git a/yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.cpp b/yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.cpp index e3ceec83dc7..24926bb415a 100644 --- a/yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.cpp +++ b/yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.cpp @@ -3,12 +3,13 @@ #include <util/stream/file.h> +#include <yt/cpp/mapreduce/common/helpers.h> // Для логов, потом мб убрать #include <yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.h> #include <yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_reader.h> #include <yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_writer.h> #include <yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h> #include <yt/yql/providers/yt/fmr/utils/yql_yt_parse_records.h> -#include <yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.h> +#include <yt/yql/providers/yt/fmr/yt_job_service/interface/yql_yt_job_service.h> #include <yql/essentials/utils/log/log.h> @@ -17,8 +18,8 @@ namespace NYql::NFmr { class TFmrJob: public IFmrJob { public: - TFmrJob(ITableDataService::TPtr tableDataService, IYtService::TPtr ytService, const TFmrJobSettings& settings) - : TableDataService_(tableDataService), YtService_(ytService), Settings_(settings) + TFmrJob(ITableDataService::TPtr tableDataService, IYtJobService::TPtr ytJobService, const TFmrJobSettings& settings) + : TableDataService_(tableDataService), YtJobService_(ytJobService), Settings_(settings) { } @@ -28,25 +29,26 @@ public: std::shared_ptr<std::atomic<bool>> cancelFlag ) override { try { - const auto ytTable = params.Input; - const auto cluster = params.Input.Cluster; - const auto path = params.Input.Path; + const auto ytTableTaskRef = params.Input; const auto output = params.Output; const auto tableId = output.TableId; const auto partId = output.PartId; - YQL_CLOG(DEBUG, FastMapReduce) << "Downloading " << cluster << '.' << path; YQL_ENSURE(clusterConnections.size() == 1); - auto ytTableReader = YtService_->MakeReader(ytTable, clusterConnections.begin()->second, Settings_.YtReaderSettings); + + std::vector<NYT::TRawTableReaderPtr> ytTableReaders = GetYtTableReaders(ytTableTaskRef, clusterConnections); auto tableDataServiceWriter = MakeIntrusive<TFmrTableDataServiceWriter>(tableId, partId, TableDataService_, Settings_.FmrWriterSettings); - ParseRecords(ytTableReader, tableDataServiceWriter, Settings_.ParseRecordSettings.DonwloadReadBlockCount, Settings_.ParseRecordSettings.DonwloadReadBlockSize, cancelFlag); + for (auto& ytTableReader: ytTableReaders) { + ParseRecords(ytTableReader, tableDataServiceWriter, Settings_.ParseRecordSettings.DonwloadReadBlockCount, Settings_.ParseRecordSettings.DonwloadReadBlockSize, cancelFlag); + } tableDataServiceWriter->Flush(); - TTableStats stats = tableDataServiceWriter->GetStats(); + TTableChunkStats stats = tableDataServiceWriter->GetStats(); auto statistics = TStatistics({{output, stats}}); return statistics; } catch (...) { + YQL_CLOG(ERROR, FastMapReduce) << "Gotten error inside download: " << CurrentExceptionMessage(); return TError(CurrentExceptionMessage()); } } @@ -63,16 +65,15 @@ public: const auto tableId = params.Input.TableId; const auto tableRanges = params.Input.TableRanges; - YQL_CLOG(DEBUG, FastMapReduce) << "Uploading " << cluster << '.' << path; - auto tableDataServiceReader = MakeIntrusive<TFmrTableDataServiceReader>(tableId, tableRanges, TableDataService_, Settings_.FmrReaderSettings); YQL_ENSURE(clusterConnections.size() == 1); - auto ytTableWriter = YtService_->MakeWriter(ytTable, clusterConnections.begin()->second, Settings_.YtWriterSettings); + auto ytTableWriter = YtJobService_->MakeWriter(ytTable, clusterConnections.begin()->second, Settings_.YtWriterSettings); ParseRecords(tableDataServiceReader, ytTableWriter, Settings_.ParseRecordSettings.UploadReadBlockCount, Settings_.ParseRecordSettings.UploadReadBlockSize, cancelFlag); ytTableWriter->Flush(); return TStatistics(); } catch (...) { + YQL_CLOG(ERROR, FastMapReduce) << "Gotten error inside upload: " << CurrentExceptionMessage(); return TError(CurrentExceptionMessage()); } } @@ -83,19 +84,25 @@ public: std::shared_ptr<std::atomic<bool>> cancelFlag ) override { try { - const auto inputs = params.Input; + const auto taskTableInputRef = params.Input; const auto output = params.Output; - YQL_CLOG(DEBUG, FastMapReduce) << "Merging " << inputs.size() << " inputs"; auto& parseRecordSettings = Settings_.ParseRecordSettings; auto tableDataServiceWriter = MakeIntrusive<TFmrTableDataServiceWriter>(output.TableId, output.PartId, TableDataService_, Settings_.FmrWriterSettings); auto threadPool = CreateThreadPool(parseRecordSettings.MergeNumThreads); TMaybe<TMutex> mutex = TMutex(); - for (const auto& inputTableRef : inputs) { - auto inputTableReader = GetTableInputStream(inputTableRef, clusterConnections); - threadPool->SafeAddFunc([&, inputTableReader] { - ParseRecords(inputTableReader, tableDataServiceWriter, parseRecordSettings.MergeReadBlockCount, parseRecordSettings.MergeReadBlockSize, cancelFlag, mutex); + for (const auto& inputTableRef : taskTableInputRef.Inputs) { + threadPool->SafeAddFunc([&, tableDataServiceWriter] { + try { + auto inputTableReaders = GetTableInputStreams(inputTableRef, clusterConnections); + for (auto& tableReader: inputTableReaders) { + ParseRecords(tableReader, tableDataServiceWriter, parseRecordSettings.MergeReadBlockCount, parseRecordSettings.MergeReadBlockSize, cancelFlag, mutex); + } + } catch (...) { + YQL_CLOG(ERROR, FastMapReduce) << CurrentExceptionMessage(); + throw yexception() << CurrentExceptionMessage(); + } }); } threadPool->Stop(); @@ -103,6 +110,7 @@ public: tableDataServiceWriter->Flush(); return TStatistics({{output, tableDataServiceWriter->GetStats()}}); } catch (...) { + YQL_CLOG(ERROR, FastMapReduce) << "Gotten error inside merge: " << CurrentExceptionMessage(); return TError(CurrentExceptionMessage()); } } @@ -112,48 +120,68 @@ public: const std::unordered_map<TFmrTableId, TClusterConnection>& /* clusterConnections */, std::shared_ptr<std::atomic<bool>> /* cancelFlag */ ) override { - Cerr << "MAP NOT IMPLEMENTED" << Endl; YQL_CLOG(ERROR, FastMapReduce) << "MAP NOT IMPLEMENTED"; ythrow yexception() << "Not implemented"; } private: - NYT::TRawTableReaderPtr GetTableInputStream(const TTaskTableRef& tableRef, const std::unordered_map<TFmrTableId, TClusterConnection>& clusterConnections) const { - auto ytTable = std::get_if<TYtTableRef>(&tableRef); + std::vector<NYT::TRawTableReaderPtr> GetTableInputStreams(const TTaskTableRef& tableRef, const std::unordered_map<TFmrTableId, TClusterConnection>& clusterConnections) const { + auto ytTableTaskRef = std::get_if<TYtTableTaskRef>(&tableRef); auto fmrTable = std::get_if<TFmrTableInputRef>(&tableRef); - if (ytTable) { - TFmrTableId tableId = {ytTable->Cluster, ytTable->Path}; - auto clusterConnection = clusterConnections.at(tableId); - return YtService_->MakeReader(*ytTable, clusterConnection, Settings_.YtReaderSettings); + if (ytTableTaskRef) { + return GetYtTableReaders(*ytTableTaskRef, clusterConnections); } else if (fmrTable) { - return MakeIntrusive<TFmrTableDataServiceReader>(fmrTable->TableId, fmrTable->TableRanges, TableDataService_, Settings_.FmrReaderSettings); + return {MakeIntrusive<TFmrTableDataServiceReader>(fmrTable->TableId, fmrTable->TableRanges, TableDataService_, Settings_.FmrReaderSettings)}; } else { ythrow yexception() << "Unsupported table type"; } } + std::vector<NYT::TRawTableReaderPtr> GetYtTableReaders(const TYtTableTaskRef& ytTableTaskRef, const std::unordered_map<TFmrTableId, TClusterConnection>& clusterConnections) const { + std::vector<NYT::TRawTableReaderPtr> ytTableReaders; + if (!ytTableTaskRef.FilePaths.empty()) { + // underlying gateway is file, so create readers from filepaths. + for (auto& filePath: ytTableTaskRef.FilePaths) { + ytTableReaders.emplace_back(YtJobService_->MakeReader(filePath)); + YQL_CLOG(DEBUG, FastMapReduce) << "Creating reader for file path " << filePath; + } + } else { + for (auto& richPath: ytTableTaskRef.RichPaths) { + YQL_ENSURE(richPath.Cluster_); + + // TODO - вместо этого написать нормальные хелперы из RichPath в структуры и назад + TStringBuf choppedPath; + YQL_ENSURE(TStringBuf(richPath.Path_).AfterPrefix("//", choppedPath)); + auto fmrTableId = TFmrTableId(*richPath.Cluster_, TString(choppedPath)); + auto clusterConnection = clusterConnections.at(fmrTableId); + ytTableReaders.emplace_back(YtJobService_->MakeReader(richPath, clusterConnection, Settings_.YtReaderSettings)); + } + } + return ytTableReaders; + } + private: ITableDataService::TPtr TableDataService_; - IYtService::TPtr YtService_; + IYtJobService::TPtr YtJobService_; TFmrJobSettings Settings_; }; IFmrJob::TPtr MakeFmrJob( ITableDataService::TPtr tableDataService, - IYtService::TPtr ytService, + IYtJobService::TPtr ytJobService, const TFmrJobSettings& settings ) { - return MakeIntrusive<TFmrJob>(tableDataService, ytService, settings); + return MakeIntrusive<TFmrJob>(tableDataService, ytJobService, settings); } TJobResult RunJob( TTask::TPtr task, ITableDataService::TPtr tableDataService, - IYtService::TPtr ytService, + IYtJobService::TPtr ytJobService, std::shared_ptr<std::atomic<bool>> cancelFlag ) { TFmrJobSettings jobSettings = GetJobSettingsFromTask(task); - IFmrJob::TPtr job = MakeFmrJob(tableDataService, ytService, jobSettings); + IFmrJob::TPtr job = MakeFmrJob(tableDataService, ytJobService, jobSettings); auto processTask = [job, task, cancelFlag] (auto&& taskParams) { using T = std::decay_t<decltype(taskParams)>; diff --git a/yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.h b/yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.h index 09b17090427..93affcaf4ed 100644 --- a/yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.h +++ b/yt/yql/providers/yt/fmr/job/impl/yql_yt_job_impl.h @@ -4,7 +4,7 @@ #include <yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_reader.h> #include <yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_writer.h> #include <yt/yql/providers/yt/fmr/job/interface/yql_yt_job.h> -#include <yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.h> +#include <yt/yql/providers/yt/fmr/yt_job_service/interface/yql_yt_job_service.h> #include <yt/yql/providers/yt/fmr/job_factory/impl/yql_yt_job_factory_impl.h> namespace NYql::NFmr { @@ -28,9 +28,9 @@ struct TFmrJobSettings { ui64 NumThreads = 0; }; -IFmrJob::TPtr MakeFmrJob(ITableDataService::TPtr tableDataService, IYtService::TPtr ytService, const TFmrJobSettings& settings = {}); +IFmrJob::TPtr MakeFmrJob(ITableDataService::TPtr tableDataService, IYtJobService::TPtr ytJobService, const TFmrJobSettings& settings = {}); -TJobResult RunJob(TTask::TPtr task, ITableDataService::TPtr tableDataService, IYtService::TPtr ytService, std::shared_ptr<std::atomic<bool>> cancelFlag); +TJobResult RunJob(TTask::TPtr task, ITableDataService::TPtr tableDataService, IYtJobService::TPtr ytJobService, std::shared_ptr<std::atomic<bool>> cancelFlag); TFmrJobSettings GetJobSettingsFromTask(TTask::TPtr task); diff --git a/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_reader.cpp b/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_reader.cpp index 9b2caf44aa4..48476ef9d91 100644 --- a/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_reader.cpp +++ b/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_reader.cpp @@ -16,6 +16,7 @@ TFmrTableDataServiceReader::TFmrTableDataServiceReader( TableDataService_(tableDataService), ReadAheadChunks_(settings.ReadAheadChunks) { + SetMinChunkInNewRange(); ReadAhead(); } @@ -40,12 +41,12 @@ size_t TFmrTableDataServiceReader::DoRead(void* buf, size_t len) { try { data = chunk.Data.GetValueSync(); } catch (...) { - ythrow yexception() << "Error reading chunk: " << chunk.Meta << "Error: " << CurrentExceptionMessage(); + ythrow yexception() << "Error reading chunk: " << chunk.Meta.ToString() << "Error: " << CurrentExceptionMessage(); } if (data) { DataBuffer_.Assign(data->data(), data->size()); } else { - ythrow yexception() << "No data for chunk:" << chunk.Meta; + ythrow yexception() << "No data for chunk:" << chunk.Meta.ToString(); } PendingChunks_.pop(); @@ -71,10 +72,17 @@ void TFmrTableDataServiceReader::ReadAhead() { CurrentChunk_++; } else { CurrentRange_++; + SetMinChunkInNewRange(); } } } +void TFmrTableDataServiceReader::SetMinChunkInNewRange() { + if (CurrentRange_ < TableRanges_.size()) { + CurrentChunk_ = TableRanges_[0].MinChunk; + } +} + bool TFmrTableDataServiceReader::Retry(const TMaybe<ui32>&, const TMaybe<ui64>&, const std::exception_ptr&) { return false; } @@ -85,4 +93,8 @@ bool TFmrTableDataServiceReader::HasRangeIndices() const { return false; } +TString TFmrTableDataServiceReader::TFmrChunkMeta::ToString() const { + return TStringBuilder() << TableId << ":" << PartId << ":" << std::to_string(Chunk); +} + } // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_reader.h b/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_reader.h index d56bb0eb0d7..822865b58d7 100644 --- a/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_reader.h +++ b/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_reader.h @@ -13,11 +13,6 @@ struct TFmrReaderSettings { ui64 ReadAheadChunks = 1; }; -struct TPendingFmrChunk { - NThreading::TFuture<TMaybe<TString>> Data; - TFmrChunkMeta Meta; -}; - class TFmrTableDataServiceReader: public NYT::TRawTableReader { public: TFmrTableDataServiceReader( @@ -34,9 +29,24 @@ public: bool HasRangeIndices() const override; private: + struct TFmrChunkMeta { + TString TableId; + TString PartId; + ui64 Chunk = 0; + + TString ToString() const; + }; + + struct TPendingFmrChunk { + NThreading::TFuture<TMaybe<TString>> Data; + TFmrChunkMeta Meta; + }; + size_t DoRead(void* buf, size_t len) override; void ReadAhead(); + void SetMinChunkInNewRange(); + const TString TableId_; std::vector<TTableRange> TableRanges_; ITableDataService::TPtr TableDataService_; diff --git a/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_writer.cpp b/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_writer.cpp index 139e57368cf..9063c48da33 100644 --- a/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_writer.cpp +++ b/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_writer.cpp @@ -1,6 +1,7 @@ #include "yql_yt_table_data_service_writer.h" #include <library/cpp/threading/future/wait/wait.h> #include <util/string/join.h> +#include <yql/essentials/utils/log/log.h> #include <yql/essentials/utils/yql_panic.h> @@ -27,7 +28,7 @@ void TFmrTableDataServiceWriter::DoWrite(const void* buf, size_t len) { } void TFmrTableDataServiceWriter::NotifyRowEnd() { - ++Rows_; + ++CurrentChunkRows_; if (TableContent_.size() >= MaxRowWeight_) { ythrow yexception() << "Current row size: " << TableContent_.size() << " is larger than max row weight: " << MaxRowWeight_; } @@ -74,17 +75,16 @@ void TFmrTableDataServiceWriter::PutRows() { } } ); - ++ChunkCount_; DataWeight_ += TableContent_.Size(); + PartIdChunkStats_.emplace_back(TChunkStats{.Rows = CurrentChunkRows_, .DataWeight = TableContent_.Size()}); + CurrentChunkRows_ = 0; + ++ChunkCount_; TableContent_.Clear(); } -TTableStats TFmrTableDataServiceWriter::GetStats() { - return TTableStats{ - .Chunks = ChunkCount_, - .Rows = Rows_, - .DataWeight = DataWeight_, - }; +TTableChunkStats TFmrTableDataServiceWriter::GetStats() { + YQL_CLOG(DEBUG, FastMapReduce) << " Finished writing to table data service for table Id: " << TableId_ << " and part Id " << PartId_ ; + return TTableChunkStats{.PartId = PartId_, .PartIdChunkStats = PartIdChunkStats_}; } } // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_writer.h b/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_writer.h index 544b9fc3bd1..ca42498b276 100644 --- a/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_writer.h +++ b/yt/yql/providers/yt/fmr/job/impl/yql_yt_table_data_service_writer.h @@ -10,8 +10,6 @@ namespace NYql::NFmr { - - struct TFmrWriterSettings { ui64 ChunkSize = 1024 * 1024; ui64 MaxInflightChunks = 1; @@ -27,7 +25,7 @@ public: const TFmrWriterSettings& settings = TFmrWriterSettings() ); - TTableStats GetStats(); + TTableChunkStats GetStats(); void NotifyRowEnd() override; @@ -44,7 +42,7 @@ private: const TString PartId_; ITableDataService::TPtr TableDataService_; ui64 DataWeight_ = 0; - ui64 Rows_ = 0; + ui64 CurrentChunkRows_ = 0; TBuffer TableContent_; const ui64 ChunkSize_; // size at which we push to table data service @@ -52,6 +50,7 @@ private: const ui64 MaxRowWeight_; ui64 ChunkCount_ = 0; + std::vector<TChunkStats> PartIdChunkStats_; struct TFmrWriterState { ui64 CurInflightChunks = 0; diff --git a/yt/yql/providers/yt/fmr/job/interface/ya.make b/yt/yql/providers/yt/fmr/job/interface/ya.make index 7d256622f7e..c2439772033 100644 --- a/yt/yql/providers/yt/fmr/job/interface/ya.make +++ b/yt/yql/providers/yt/fmr/job/interface/ya.make @@ -4,6 +4,10 @@ SRCS( yql_yt_job.cpp ) +PEERDIR( + yt/yql/providers/yt/fmr/request_options +) + YQL_LAST_ABI_VERSION() END() diff --git a/yt/yql/providers/yt/fmr/job_factory/impl/ut/yql_yt_job_factory_ut.cpp b/yt/yql/providers/yt/fmr/job_factory/impl/ut/yql_yt_job_factory_ut.cpp index bcf702709c7..5977af62134 100644 --- a/yt/yql/providers/yt/fmr/job_factory/impl/ut/yql_yt_job_factory_ut.cpp +++ b/yt/yql/providers/yt/fmr/job_factory/impl/ut/yql_yt_job_factory_ut.cpp @@ -10,7 +10,10 @@ namespace NYql::NFmr { TTask::TPtr CreateTestTask() { - auto input = TYtTableRef("test_cluster", "test_path"); + auto input = TYtTableTaskRef{ + .RichPaths = {NYT::TRichYPath("test_path").Cluster("test_cluster")}, + .FilePaths = {"test_file_path"} + }; auto output = TFmrTableOutputRef("test_table_id"); auto params = TDownloadTaskParams(input, output); return MakeTask(ETaskType::Download, "test_task_id", params, "test_session_id"); diff --git a/yt/yql/providers/yt/fmr/job_factory/impl/yql_yt_job_factory_impl.cpp b/yt/yql/providers/yt/fmr/job_factory/impl/yql_yt_job_factory_impl.cpp index 9c6c0b5b054..610d4f3b7cb 100644 --- a/yt/yql/providers/yt/fmr/job_factory/impl/yql_yt_job_factory_impl.cpp +++ b/yt/yql/providers/yt/fmr/job_factory/impl/yql_yt_job_factory_impl.cpp @@ -42,6 +42,10 @@ public: return future; } + ui64 GetMaxParallelJobCount() override { + return NumThreads_; + } + void Start() override { ThreadPool_ = CreateThreadPool(NumThreads_); } @@ -52,7 +56,7 @@ public: private: THolder<IThreadPool> ThreadPool_; - i32 NumThreads_; + ui64 NumThreads_; std::function<TJobResult(TTask::TPtr, std::shared_ptr<std::atomic<bool>>)> Function_; const TIntrusivePtr<IRandomProvider> RandomProvider_; }; diff --git a/yt/yql/providers/yt/fmr/job_factory/impl/yql_yt_job_factory_impl.h b/yt/yql/providers/yt/fmr/job_factory/impl/yql_yt_job_factory_impl.h index 68ef1a5c379..77d41106e30 100644 --- a/yt/yql/providers/yt/fmr/job_factory/impl/yql_yt_job_factory_impl.h +++ b/yt/yql/providers/yt/fmr/job_factory/impl/yql_yt_job_factory_impl.h @@ -13,7 +13,7 @@ struct TJobResult { }; struct TFmrJobFactorySettings { - ui32 NumThreads = 3; + ui64 NumThreads = 3; std::function<TJobResult(TTask::TPtr, std::shared_ptr<std::atomic<bool>>)> Function; TIntrusivePtr<IRandomProvider> RandomProvider = CreateDefaultRandomProvider(); }; diff --git a/yt/yql/providers/yt/fmr/job_factory/interface/ya.make b/yt/yql/providers/yt/fmr/job_factory/interface/ya.make index 32ecd264ae5..bef9aae0b2d 100644 --- a/yt/yql/providers/yt/fmr/job_factory/interface/ya.make +++ b/yt/yql/providers/yt/fmr/job_factory/interface/ya.make @@ -7,6 +7,7 @@ SRCS( PEERDIR( library/cpp/threading/future yql/essentials/utils + yt/yql/providers/yt/fmr/request_options ) YQL_LAST_ABI_VERSION() diff --git a/yt/yql/providers/yt/fmr/job_factory/interface/yql_yt_job_factory.h b/yt/yql/providers/yt/fmr/job_factory/interface/yql_yt_job_factory.h index 27393ff3399..cfb2370d320 100644 --- a/yt/yql/providers/yt/fmr/job_factory/interface/yql_yt_job_factory.h +++ b/yt/yql/providers/yt/fmr/job_factory/interface/yql_yt_job_factory.h @@ -13,6 +13,8 @@ public: virtual ~IFmrJobFactory() = default; virtual NThreading::TFuture<TTaskState::TPtr> StartJob(TTask::TPtr task, std::shared_ptr<std::atomic<bool>> cancelFlag) = 0; + + virtual ui64 GetMaxParallelJobCount() = 0; }; } // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/proto/coordinator.proto b/yt/yql/providers/yt/fmr/proto/coordinator.proto index 34e16ed26b5..d3a9307dcd1 100644 --- a/yt/yql/providers/yt/fmr/proto/coordinator.proto +++ b/yt/yql/providers/yt/fmr/proto/coordinator.proto @@ -8,6 +8,7 @@ message THeartbeatRequest { uint32 WorkerId = 1; string VolatileId = 2; repeated TTaskState TaskStates = 3; + uint64 AvailableSlots = 4; } message THeartbeatResponse { @@ -33,6 +34,7 @@ message TStartOperationResponse { message TGetOperationResponse { EOperationStatus Status = 1; repeated TFmrError ErrorMessages = 2; + repeated TTableStats TableStats = 3; } message TDeleteOperationResponse { diff --git a/yt/yql/providers/yt/fmr/proto/request_options.proto b/yt/yql/providers/yt/fmr/proto/request_options.proto index 60c8471846f..92db8d0b0e7 100644 --- a/yt/yql/providers/yt/fmr/proto/request_options.proto +++ b/yt/yql/providers/yt/fmr/proto/request_options.proto @@ -18,7 +18,6 @@ enum ETaskStatus { TASK_IN_PROGRESS = 2; TASK_FAILED = 3; TASK_COMPLETED = 4; - TASK_ABORTED = 5; } enum ETaskType { @@ -26,6 +25,7 @@ enum ETaskType { TASK_TYPE_DOWNLOAD = 1; TASK_TYPE_UPLOAD = 2; TASK_TYPE_MERGE = 3; + TASK_TYPE_MAP = 4; } enum EFmrComponent { @@ -47,6 +47,7 @@ message TFmrError { optional uint32 WorkerId = 4; optional string TaskId = 5; optional string OperationId = 6; + optional string JobId = 7; } message TYtTableRef { @@ -55,6 +56,11 @@ message TYtTableRef { optional string FilePath = 3; } +message TYtTableTaskRef { + repeated string RichPath = 1; + repeated string FilePath = 2; +} + message TFmrTableId { string Id = 1; } @@ -85,13 +91,23 @@ message TTableStats { uint64 DataWeight = 3; } -message TFmrStatisticsObject { - TFmrTableOutputRef Table = 1; - TTableStats Statistic = 2; +message TChunkStats { + uint64 Rows = 1; + uint64 DataWeight = 2; +} + +message TTableChunkStats { + string PartId = 1; + repeated TChunkStats PartIdChunkStats = 2; +} + +message TStatisticsObject { + TFmrTableOutputRef FmrTableOutputRef = 1; + TTableChunkStats TableChunkStats = 2; } message TStatistics { - repeated TFmrStatisticsObject OutputTables = 1; + repeated TStatisticsObject OutputTables = 1; } message TOperationTableRef { @@ -103,11 +119,15 @@ message TOperationTableRef { message TTaskTableRef { oneof TaskTableRef { - TYtTableRef YtTableRef = 1; + TYtTableTaskRef YtTableTaskRef = 1; TFmrTableInputRef FmrTableInputRef = 2; } } +message TTaskTableInputRef { + repeated TTaskTableRef Inputs = 1; +} + message TUploadOperationParams { TFmrTableRef Input = 1; TYtTableRef Output = 2; @@ -124,7 +144,7 @@ message TDownloadOperationParams { } message TDownloadTaskParams { - TYtTableRef Input = 1; + TYtTableTaskRef Input = 1; TFmrTableOutputRef Output = 2; } @@ -134,15 +154,28 @@ message TMergeOperationParams { } message TMergeTaskParams { - repeated TTaskTableRef Input = 1; + TTaskTableInputRef Input = 1; TFmrTableOutputRef Output = 2; } +message TMapOperationParams { + repeated TOperationTableRef Input = 1; + repeated TFmrTableRef Output = 2; + string Executable = 3; +} + +message TMapTaskParams { + TTaskTableInputRef Input = 1; + repeated TFmrTableOutputRef Output = 2; + string Executable = 3; +} + message TOperationParams { oneof TOperationParams { TUploadOperationParams UploadOperationParams = 1; TDownloadOperationParams DownloadOperationParams = 2; TMergeOperationParams MergeOperationParams = 3; + TMapOperationParams MapOperationParams = 4; } } @@ -151,6 +184,7 @@ message TTaskParams { TUploadTaskParams UploadTaskParams = 1; TDownloadTaskParams DownloadTaskParams = 2; TMergeTaskParams MergeTaskParams = 3; + TMapTaskParams MapTaskParams = 4; } } diff --git a/yt/yql/providers/yt/fmr/request_options/proto_helpers/ya.make b/yt/yql/providers/yt/fmr/request_options/proto_helpers/ya.make index 36f1b3ebcaf..5e1ec3d0437 100644 --- a/yt/yql/providers/yt/fmr/request_options/proto_helpers/ya.make +++ b/yt/yql/providers/yt/fmr/request_options/proto_helpers/ya.make @@ -5,6 +5,7 @@ SRCS( ) PEERDIR( + yt/cpp/mapreduce/common yt/yql/providers/yt/fmr/proto ) diff --git a/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.cpp b/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.cpp index b0c368008dc..34f01f4884d 100644 --- a/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.cpp +++ b/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.cpp @@ -1,5 +1,7 @@ #include "yql_yt_request_proto_helpers.h" #include <library/cpp/yson/node/node_io.h> +#include <yt/cpp/mapreduce/common/helpers.h> +#include <yt/cpp/mapreduce/interface/serialize.h> namespace NYql::NFmr { @@ -17,6 +19,7 @@ NProto::TFmrError FmrErrorToProto(const TFmrError& error) { if (error.OperationId) { protoError.SetOperationId(*error.OperationId); } + protoError.SetJobId(*error.JobId); return protoError; } @@ -34,6 +37,7 @@ TFmrError FmrErrorFromProto(const NProto::TFmrError& protoError) { if (protoError.HasOperationId()) { fmrError.OperationId = protoError.GetOperationId(); } + fmrError.JobId = protoError.GetJobId(); return fmrError; } @@ -57,6 +61,32 @@ TYtTableRef YtTableRefFromProto(const NProto::TYtTableRef protoYtTableRef) { return ytTableRef; } +NProto::TYtTableTaskRef YtTableTaskRefToProto(const TYtTableTaskRef& ytTableTaskRef) { + NProto::TYtTableTaskRef protoYtTableTaskRef; + for (auto& richPath: ytTableTaskRef.RichPaths) { + TString serializedRichPath = NYT::NodeToYsonString(NYT::PathToNode(richPath)); + protoYtTableTaskRef.AddRichPath(serializedRichPath); + } + for (auto& filePath: ytTableTaskRef.FilePaths) { + protoYtTableTaskRef.AddFilePath(filePath); + } + return protoYtTableTaskRef; +} + +TYtTableTaskRef YtTableTaskRefFromProto(const NProto::TYtTableTaskRef protoYtTableTaskRef) { + TYtTableTaskRef ytTableTaskRef; + for (auto& serializedPath: protoYtTableTaskRef.GetRichPath()) { + auto node = NYT::NodeFromYsonString(serializedPath); + NYT::TRichYPath richPath; + NYT::Deserialize(richPath, node); + ytTableTaskRef.RichPaths.emplace_back(richPath); + } + for (auto& filePath: protoYtTableTaskRef.GetFilePath()) { + ytTableTaskRef.FilePaths.emplace_back(filePath); + } + return ytTableTaskRef; +} + NProto::TFmrTableId FmrTableIdToProto(const TFmrTableId& fmrTableId) { NProto::TFmrTableId protoFmrTableId; protoFmrTableId.SetId(fmrTableId.Id); @@ -148,14 +178,47 @@ TTableStats TableStatsFromProto(const NProto::TTableStats& protoTableStats) { }; } +NProto::TChunkStats ChunkStatsToProto(const TChunkStats& chunkStats) { + NProto::TChunkStats protoChunkStats; + protoChunkStats.SetRows(chunkStats.Rows); + protoChunkStats.SetDataWeight(chunkStats.DataWeight); + return protoChunkStats; +} + +TChunkStats ChunkStatsFromProto(const NProto::TChunkStats& protoChunkStats) { + return TChunkStats{.Rows = protoChunkStats.GetRows(), .DataWeight = protoChunkStats.GetDataWeight()}; +} + +NProto::TTableChunkStats TableChunkStatsToProto(const TTableChunkStats& tableChunkStats) { + NProto::TTableChunkStats protoTableChunkStats; + protoTableChunkStats.SetPartId(tableChunkStats.PartId); + for (auto& chunkStats: tableChunkStats.PartIdChunkStats) { + NProto::TChunkStats protoChunkStats = ChunkStatsToProto(chunkStats); + auto* curPartIdChunkStats = protoTableChunkStats.AddPartIdChunkStats(); + curPartIdChunkStats->Swap(&protoChunkStats); + } + return protoTableChunkStats; +} + +TTableChunkStats TableChunkStatsFromProto(const NProto::TTableChunkStats& protoTableChunkStats) { + TTableChunkStats tableChunkStats; + tableChunkStats.PartId = protoTableChunkStats.GetPartId(); + std::vector<TChunkStats> partIdChunkStats; + for (auto& stat: protoTableChunkStats.GetPartIdChunkStats()) { + partIdChunkStats.emplace_back(ChunkStatsFromProto(stat)); + } + tableChunkStats.PartIdChunkStats = partIdChunkStats; + return tableChunkStats; +} + NProto::TStatistics StatisticsToProto(const TStatistics& stats) { NProto::TStatistics protoStatistics; - for (auto& [fmrTableOutputRef, tableStat]: stats.OutputTables) { - NProto::TFmrTableOutputRef protoFmrTableOutputref = FmrTableOutputRefToProto(fmrTableOutputRef); - NProto::TTableStats protoStats = TableStatsToProto(tableStat); - NProto::TFmrStatisticsObject statTableObject; - statTableObject.MutableTable()->Swap(&protoFmrTableOutputref); - statTableObject.MutableStatistic()->Swap(&protoStats); + for (auto& [fmrTableOutputRef, tableChunkStats]: stats.OutputTables) { + NProto::TFmrTableOutputRef protoFmrTableOutputRef = FmrTableOutputRefToProto(fmrTableOutputRef); + NProto::TTableChunkStats protoTableChunkStats = TableChunkStatsToProto(tableChunkStats); + NProto::TStatisticsObject statTableObject; + statTableObject.MutableFmrTableOutputRef()->Swap(&protoFmrTableOutputRef); + statTableObject.MutableTableChunkStats()->Swap(&protoTableChunkStats); auto* curOutputTable = protoStatistics.AddOutputTables(); curOutputTable->Swap(&statTableObject); } @@ -163,12 +226,12 @@ NProto::TStatistics StatisticsToProto(const TStatistics& stats) { } TStatistics StatisticsFromProto(const NProto::TStatistics& protoStats) { - std::unordered_map<TFmrTableOutputRef, TTableStats> outputTables; + std::unordered_map<TFmrTableOutputRef, TTableChunkStats> outputTables; for (size_t i = 0; i < protoStats.OutputTablesSize(); ++i) { - NProto::TFmrStatisticsObject protoStatTableObject = protoStats.GetOutputTables(i); - TFmrTableOutputRef fmrTableOutputRef = FmrTableOutputRefFromProto(protoStatTableObject.GetTable()); - TTableStats tableStats = TableStatsFromProto(protoStatTableObject.GetStatistic()); - outputTables[fmrTableOutputRef] = tableStats; + NProto::TStatisticsObject protoStatTableObject = protoStats.GetOutputTables(i); + TFmrTableOutputRef fmrTableOutputRef = FmrTableOutputRefFromProto(protoStatTableObject.GetFmrTableOutputRef()); + TTableChunkStats tableChunkStats = TableChunkStatsFromProto(protoStatTableObject.GetTableChunkStats()); + outputTables[fmrTableOutputRef] = tableChunkStats; } return TStatistics{.OutputTables = outputTables}; } @@ -193,14 +256,14 @@ TOperationTableRef OperationTableRefFromProto(const NProto::TOperationTableRef& } else { tableRef = FmrTableRefFromProto(protoOperationTableRef.GetFmrTableRef()); } - return {tableRef}; + return tableRef; } NProto::TTaskTableRef TaskTableRefToProto(const TTaskTableRef& taskTableRef) { NProto::TTaskTableRef protoTaskTableRef; - if (auto* ytTableRefPtr = std::get_if<TYtTableRef>(&taskTableRef)) { - NProto::TYtTableRef protoYtTableRef = YtTableRefToProto(*ytTableRefPtr); - protoTaskTableRef.MutableYtTableRef()->Swap(&protoYtTableRef); + if (auto* ytTableTaskRefPtr = std::get_if<TYtTableTaskRef>(&taskTableRef)) { + NProto::TYtTableTaskRef protoYtTableTaskRef = YtTableTaskRefToProto(*ytTableTaskRefPtr); + protoTaskTableRef.MutableYtTableTaskRef()->Swap(&protoYtTableTaskRef); } else { auto* fmrTableInputRefPtr = std::get_if<TFmrTableInputRef>(&taskTableRef); NProto::TFmrTableInputRef protoFmrTableInputRef = FmrTableInputRefToProto(*fmrTableInputRefPtr); @@ -211,13 +274,31 @@ NProto::TTaskTableRef TaskTableRefToProto(const TTaskTableRef& taskTableRef) { } TTaskTableRef TaskTableRefFromProto(const NProto::TTaskTableRef& protoTaskTableRef) { - std::variant<TYtTableRef, TFmrTableInputRef> tableRef; - if (protoTaskTableRef.HasYtTableRef()) { - tableRef = YtTableRefFromProto(protoTaskTableRef.GetYtTableRef()); + std::variant<TYtTableTaskRef, TFmrTableInputRef> tableRef; + if (protoTaskTableRef.HasYtTableTaskRef()) { + tableRef = YtTableTaskRefFromProto(protoTaskTableRef.GetYtTableTaskRef()); } else { tableRef = FmrTableInputRefFromProto(protoTaskTableRef.GetFmrTableInputRef()); } - return {tableRef}; + return tableRef; +} + +NProto::TTaskTableInputRef TaskTableInputRefToProto(const TTaskTableInputRef& taskTableInputRef) { + NProto::TTaskTableInputRef protoTaskTableInputRef; + for (auto& taskTableRef: taskTableInputRef.Inputs) { + auto protoTaskTableRef = TaskTableRefToProto(taskTableRef); + auto* curInput = protoTaskTableInputRef.AddInputs(); + curInput->Swap(&protoTaskTableRef); + } + return protoTaskTableInputRef; +} + +TTaskTableInputRef TaskTableInputRefFromProto(const NProto::TTaskTableInputRef& protoTaskTableInputRef) { + std::vector<TTaskTableRef> inputs; + for (auto& protoTaskTableRef: protoTaskTableInputRef.GetInputs()) { + inputs.emplace_back(TaskTableRefFromProto(protoTaskTableRef)); + } + return TTaskTableInputRef{.Inputs = inputs}; } NProto::TUploadOperationParams UploadOperationParamsToProto(const TUploadOperationParams& uploadOperationParams) { @@ -229,6 +310,13 @@ NProto::TUploadOperationParams UploadOperationParamsToProto(const TUploadOperati return protoUploadOperationParams; } +TUploadOperationParams UploadOperationParamsFromProto(const NProto::TUploadOperationParams& protoUploadOperationParams) { + return TUploadOperationParams( + FmrTableRefFromProto(protoUploadOperationParams.GetInput()), + YtTableRefFromProto(protoUploadOperationParams.GetOutput()) + ); +} + NProto::TUploadTaskParams UploadTaskParamsToProto(const TUploadTaskParams& uploadTaskParams) { NProto::TUploadTaskParams protoUploadTaskParams; auto input = FmrTableInputRefToProto(uploadTaskParams.Input); @@ -238,13 +326,6 @@ NProto::TUploadTaskParams UploadTaskParamsToProto(const TUploadTaskParams& uploa return protoUploadTaskParams; } -TUploadOperationParams UploadOperationParamsFromProto(const NProto::TUploadOperationParams& protoUploadOperationParams) { - return TUploadOperationParams( - FmrTableRefFromProto(protoUploadOperationParams.GetInput()), - YtTableRefFromProto(protoUploadOperationParams.GetOutput()) - ); -} - TUploadTaskParams UploadTaskParamsFromProto(const NProto::TUploadTaskParams& protoUploadTaskParams) { TUploadTaskParams uploadTaskParams; uploadTaskParams.Input = FmrTableInputRefFromProto(protoUploadTaskParams.GetInput()); @@ -261,25 +342,25 @@ NProto::TDownloadOperationParams DownloadOperationParamsToProto(const TDownloadO return protoDownloadOperationParams; } +TDownloadOperationParams DownloadOperationParamsFromProto(const NProto::TDownloadOperationParams& protoDownloadOperationParams) { + return TDownloadOperationParams( + YtTableRefFromProto(protoDownloadOperationParams.GetInput()), + FmrTableRefFromProto(protoDownloadOperationParams.GetOutput()) + ); +} + NProto::TDownloadTaskParams DownloadTaskParamsToProto(const TDownloadTaskParams& downloadTaskParams) { NProto::TDownloadTaskParams protoDownloadTaskParams; - auto input = YtTableRefToProto(downloadTaskParams.Input); + auto input = YtTableTaskRefToProto(downloadTaskParams.Input); auto output = FmrTableOutputRefToProto(downloadTaskParams.Output); protoDownloadTaskParams.MutableInput()->Swap(&input); protoDownloadTaskParams.MutableOutput()->Swap(&output); return protoDownloadTaskParams; } -TDownloadOperationParams DownloadOperationParamsFromProto(const NProto::TDownloadOperationParams& protoDownloadOperationParams) { - return TDownloadOperationParams( - YtTableRefFromProto(protoDownloadOperationParams.GetInput()), - FmrTableRefFromProto(protoDownloadOperationParams.GetOutput()) - ); -} - TDownloadTaskParams DownloadTaskParamsFromProto(const NProto::TDownloadTaskParams& protoDownloadTaskParams) { TDownloadTaskParams downloadTaskParams; - downloadTaskParams.Input = YtTableRefFromProto(protoDownloadTaskParams.GetInput()); + downloadTaskParams.Input = YtTableTaskRefFromProto(protoDownloadTaskParams.GetInput()); downloadTaskParams.Output = FmrTableOutputRefFromProto(protoDownloadTaskParams.GetOutput()); return downloadTaskParams; } @@ -296,18 +377,6 @@ NProto::TMergeOperationParams MergeOperationParamsToProto(const TMergeOperationP return protoMergeOperationParams; } -NProto::TMergeTaskParams MergeTaskParamsToProto(const TMergeTaskParams& mergeTaskParams) { - NProto::TMergeTaskParams protoMergeTaskParams; - for (size_t i = 0; i < mergeTaskParams.Input.size(); ++i) { - auto inputTable = TaskTableRefToProto(mergeTaskParams.Input[i]); - auto* curInput = protoMergeTaskParams.AddInput(); - curInput->Swap(&inputTable); - } - auto outputTable = FmrTableOutputRefToProto(mergeTaskParams.Output); - protoMergeTaskParams.MutableOutput()->Swap(&outputTable); - return protoMergeTaskParams; -} - TMergeOperationParams MergeOperationParamsFromProto(const NProto::TMergeOperationParams& protoMergeOperationParams) { TMergeOperationParams mergeOperationParams( {}, @@ -320,18 +389,72 @@ TMergeOperationParams MergeOperationParamsFromProto(const NProto::TMergeOperatio return mergeOperationParams; } +NProto::TMergeTaskParams MergeTaskParamsToProto(const TMergeTaskParams& mergeTaskParams) { + NProto::TMergeTaskParams protoMergeTaskParams; + auto inputTables = TaskTableInputRefToProto(mergeTaskParams.Input); + protoMergeTaskParams.MutableInput()->Swap(&inputTables); + auto outputTable = FmrTableOutputRefToProto(mergeTaskParams.Output); + protoMergeTaskParams.MutableOutput()->Swap(&outputTable); + return protoMergeTaskParams; +} + TMergeTaskParams MergeTaskParamsFromProto(const NProto::TMergeTaskParams& protoMergeTaskParams) { TMergeTaskParams mergeTaskParams; - std::vector<TTaskTableRef> input; - for (size_t i = 0; i < protoMergeTaskParams.InputSize(); ++i) { - TTaskTableRef inputTable = TaskTableRefFromProto(protoMergeTaskParams.GetInput(i)); - input.emplace_back(inputTable); - } - mergeTaskParams.Input = input; + mergeTaskParams.Input = TaskTableInputRefFromProto(protoMergeTaskParams.GetInput()); mergeTaskParams.Output = FmrTableOutputRefFromProto(protoMergeTaskParams.GetOutput()); return mergeTaskParams; } +NProto::TMapOperationParams MapOperationParamsToProto(const TMapOperationParams& mapOperationParams) { + NProto::TMapOperationParams protoMapOperationParams; + for (auto& operationTableRef: mapOperationParams.Input) { + auto protoOperationTableRef = OperationTableRefToProto(operationTableRef); + protoMapOperationParams.AddInput()->Swap(&protoOperationTableRef); + } + for (auto& fmrTableRef: mapOperationParams.Output) { + auto protoFmrTableRef = FmrTableRefToProto(fmrTableRef); + protoMapOperationParams.AddOutput()->Swap(&protoFmrTableRef); + } + protoMapOperationParams.SetExecutable(mapOperationParams.Executable); + return protoMapOperationParams; +} + +TMapOperationParams MapOperationParamsFromProto(const NProto::TMapOperationParams& protoMapOperationParams) { + std::vector<TOperationTableRef> inputTables; + std::vector<TFmrTableRef> outputTables; + for (auto& protoOperationTableRef: protoMapOperationParams.GetInput()) { + inputTables.emplace_back(OperationTableRefFromProto(protoOperationTableRef)); + } + for (auto& protoFmrTableRef: protoMapOperationParams.GetOutput()) { + outputTables.emplace_back(FmrTableRefFromProto(protoFmrTableRef)); + } + return TMapOperationParams{.Input = inputTables, .Output = outputTables, .Executable = protoMapOperationParams.GetExecutable()}; +} + +NProto::TMapTaskParams MapTaskParamsToProto(const TMapTaskParams& mapTaskParams) { + NProto::TMapTaskParams protoMapTaskParams; + auto protoTaskTableInputRef = TaskTableInputRefToProto(mapTaskParams.Input); + protoMapTaskParams.MutableInput()->Swap(&protoTaskTableInputRef); + for (auto& fmrTableOutputRef: mapTaskParams.Output) { + auto protoFmrTableOutputRef = FmrTableOutputRefToProto(fmrTableOutputRef); + protoMapTaskParams.AddOutput()->Swap(&protoFmrTableOutputRef); + } + protoMapTaskParams.SetExecutable(mapTaskParams.Executable); + return protoMapTaskParams; +} + +TMapTaskParams MapTaskParamsFromProto(const NProto::TMapTaskParams& protoMapTaskParams) { + TMapTaskParams mapTaskParams; + mapTaskParams.Input = TaskTableInputRefFromProto(protoMapTaskParams.GetInput()); + std::vector<TFmrTableOutputRef> outputTables; + for (auto& protoFmrTableOutputRef: protoMapTaskParams.GetOutput()) { + outputTables.emplace_back(FmrTableOutputRefFromProto(protoFmrTableOutputRef)); + } + mapTaskParams.Output = outputTables; + mapTaskParams.Executable = protoMapTaskParams.GetExecutable(); + return mapTaskParams; +} + NProto::TOperationParams OperationParamsToProto(const TOperationParams& operationParams) { NProto::TOperationParams protoOperationParams; if (auto* uploadOperationParamsPtr = std::get_if<TUploadOperationParams>(&operationParams)) { @@ -340,14 +463,29 @@ NProto::TOperationParams OperationParamsToProto(const TOperationParams& operatio } else if (auto* downloadOperationParamsPtr = std::get_if<TDownloadOperationParams>(&operationParams)) { NProto::TDownloadOperationParams protoDownloadOperationParams = DownloadOperationParamsToProto(*downloadOperationParamsPtr); protoOperationParams.MutableDownloadOperationParams()->Swap(&protoDownloadOperationParams); - } else { - auto* mergeOperationParamsPtr = std::get_if<TMergeOperationParams>(&operationParams); + } else if (auto* mergeOperationParamsPtr = std::get_if<TMergeOperationParams>(&operationParams)) { NProto::TMergeOperationParams protoMergeOperationParams = MergeOperationParamsToProto(*mergeOperationParamsPtr); protoOperationParams.MutableMergeOperationParams()->Swap(&protoMergeOperationParams); + } else { + auto* mapOperationParamsPtr = std::get_if<TMapOperationParams>(&operationParams); + NProto::TMapOperationParams protoMapOperationParams = MapOperationParamsToProto(*mapOperationParamsPtr); + protoOperationParams.MutableMapOperationParams()->Swap(&protoMapOperationParams); } return protoOperationParams; } +TOperationParams OperationParamsFromProto(const NProto::TOperationParams& protoOperationParams) { + if (protoOperationParams.HasDownloadOperationParams()) { + return DownloadOperationParamsFromProto(protoOperationParams.GetDownloadOperationParams()); + } else if (protoOperationParams.HasUploadOperationParams()) { + return UploadOperationParamsFromProto(protoOperationParams.GetUploadOperationParams()); + } else if (protoOperationParams.HasMergeOperationParams()) { + return MergeOperationParamsFromProto(protoOperationParams.GetMergeOperationParams()); + } else { + return MapOperationParamsFromProto(protoOperationParams.GetMapOperationParams()); + } +} + NProto::TTaskParams TaskParamsToProto(const TTaskParams& taskParams) { NProto::TTaskParams protoTaskParams; if (auto* uploadTaskParamsPtr = std::get_if<TUploadTaskParams>(&taskParams)) { @@ -356,22 +494,15 @@ NProto::TTaskParams TaskParamsToProto(const TTaskParams& taskParams) { } else if (auto* downloadTaskParamsPtr = std::get_if<TDownloadTaskParams>(&taskParams)) { NProto::TDownloadTaskParams protoDownloadTaskParams = DownloadTaskParamsToProto(*downloadTaskParamsPtr); protoTaskParams.MutableDownloadTaskParams()->Swap(&protoDownloadTaskParams); - } else { - auto* mergeTaskParamsPtr = std::get_if<TMergeTaskParams>(&taskParams); + } else if (auto* mergeTaskParamsPtr = std::get_if<TMergeTaskParams>(&taskParams)) { NProto::TMergeTaskParams protoMergeTaskParams = MergeTaskParamsToProto(*mergeTaskParamsPtr); protoTaskParams.MutableMergeTaskParams()->Swap(&protoMergeTaskParams); - } - return protoTaskParams; -} - -TOperationParams OperationParamsFromProto(const NProto::TOperationParams& protoOperationParams) { - if (protoOperationParams.HasDownloadOperationParams()) { - return DownloadOperationParamsFromProto(protoOperationParams.GetDownloadOperationParams()); - } else if (protoOperationParams.HasUploadOperationParams()) { - return UploadOperationParamsFromProto(protoOperationParams.GetUploadOperationParams()); } else { - return MergeOperationParamsFromProto(protoOperationParams.GetMergeOperationParams()); + auto* mapTaskParamsPtr = std::get_if<TMapTaskParams>(&taskParams); + NProto::TMapTaskParams protoMapTaskParams = MapTaskParamsToProto(*mapTaskParamsPtr); + protoTaskParams.MutableMapTaskParams()->Swap(&protoMapTaskParams); } + return protoTaskParams; } TTaskParams TaskParamsFromProto(const NProto::TTaskParams& protoTaskParams) { @@ -380,8 +511,10 @@ TTaskParams TaskParamsFromProto(const NProto::TTaskParams& protoTaskParams) { taskParams = DownloadTaskParamsFromProto(protoTaskParams.GetDownloadTaskParams()); } else if (protoTaskParams.HasUploadTaskParams()) { taskParams = UploadTaskParamsFromProto(protoTaskParams.GetUploadTaskParams()); - } else { + } else if (protoTaskParams.HasMergeTaskParams()) { taskParams = MergeTaskParamsFromProto(protoTaskParams.GetMergeTaskParams()); + } else { + taskParams = MapTaskParamsFromProto(protoTaskParams.GetMapTaskParams()); } return taskParams; } @@ -414,7 +547,7 @@ NProto::TTask TaskToProto(const TTask& task) { protoTask.MutableTaskParams()->Swap(&taskParams); protoTask.SetSessionId(task.SessionId); protoTask.SetNumRetries(task.NumRetries); - auto clusterConnections = *protoTask.MutableClusterConnections(); + auto& clusterConnections = *protoTask.MutableClusterConnections(); for (auto& [tableName, conn]: task.ClusterConnections) { clusterConnections[tableName.Id] = ClusterConnectionToProto(conn); } diff --git a/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.h b/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.h index 3f7ee93725b..cfeb5a424e9 100644 --- a/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.h +++ b/yt/yql/providers/yt/fmr/request_options/proto_helpers/yql_yt_request_proto_helpers.h @@ -13,6 +13,14 @@ NProto::TYtTableRef YtTableRefToProto(const TYtTableRef& ytTableRef); TYtTableRef YtTableRefFromProto(const NProto::TYtTableRef protoYtTableRef); +NProto::TYtTableTaskRef YtTableTaskRefToProto(const TYtTableTaskRef& ytTableTaskRef); + +TYtTableTaskRef YtTableTaskRefFromProto(const NProto::TYtTableTaskRef protoYtTableTaskRef); + +NProto::TFmrTableId FmrTableIdToProto(const TFmrTableId& fmrTableId); + +TFmrTableId FmrTableIdFromProto(const NProto::TFmrTableId& protoFmrTableId); + NProto::TFmrTableRef FmrTableRefToProto(const TFmrTableRef& fmrTableRef); TFmrTableRef FmrTableRefFromProto(const NProto::TFmrTableRef protoFmrTableRef); @@ -33,6 +41,14 @@ NProto::TTableStats TableStatsToProto(const TTableStats& tableStats); TTableStats TableStatsFromProto(const NProto::TTableStats& protoTableStats); +NProto::TChunkStats ChunkStatsToProto(const TChunkStats& chunkStats); + +TChunkStats ChunkStatsFromProto(const NProto::TChunkStats& protoChunkStats); + +NProto::TTableChunkStats TableChunkStatsToProto(const TTableChunkStats& tableChunkStats); + +TTableChunkStats TableChunkStatsFromProto(const NProto::TTableChunkStats& protoTableChunkStats); + NProto::TStatistics StatisticsToProto(const TStatistics& stats); TStatistics StatisticsFromProto(const NProto::TStatistics& protoStats); @@ -45,6 +61,10 @@ NProto::TTaskTableRef TaskTableRefToProto(const TTaskTableRef& taskTableRef); TTaskTableRef TaskTableRefFromProto(const NProto::TTaskTableRef& protoTaskTableRef); +NProto::TTaskTableInputRef TaskTableInputRefToProto(const TTaskTableInputRef& taskTableInputRef); + +TTaskTableInputRef TaskTableInputRefFromProto(const NProto::TTaskTableInputRef& protoTaskTableInputRef); + NProto::TUploadOperationParams UploadOperationParamsToProto(const TUploadOperationParams& uploadOperationParams); TUploadOperationParams UploadOperationParamsFromProto(const NProto::TUploadOperationParams& protoUploadOperationParams); @@ -69,6 +89,14 @@ NProto::TMergeTaskParams MergeTaskParamsToProto(const TMergeTaskParams& mergeTas TMergeTaskParams MergeTaskParamsFromProto(const NProto::TMergeTaskParams& protoMergeTaskParams); +NProto::TMapOperationParams MapOperationParamsToProto(const TMapOperationParams& mapOperationParams); + +TMapOperationParams MapOperationParamsFromProto(const NProto::TMapOperationParams& protoMapOperationParams); + +NProto::TMapTaskParams MapTaskParamsToProto(const TMapTaskParams& mapTaskParams); + +TMapTaskParams MapTaskParamsFromProto(const NProto::TMapTaskParams& protoMapTaskParams); + NProto::TOperationParams OperationParamsToProto(const TOperationParams& operationParams); TOperationParams OperationParamsFromProto(const NProto::TOperationParams& protoOperationParams); diff --git a/yt/yql/providers/yt/fmr/request_options/ya.make b/yt/yql/providers/yt/fmr/request_options/ya.make index 4e74eb8b185..ec57848ad29 100644 --- a/yt/yql/providers/yt/fmr/request_options/ya.make +++ b/yt/yql/providers/yt/fmr/request_options/ya.make @@ -7,6 +7,8 @@ SRCS( PEERDIR( library/cpp/yson/node library/cpp/threading/future + yt/cpp/mapreduce/common + yt/cpp/mapreduce/interface yql/essentials/public/issue ) diff --git a/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.cpp b/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.cpp index 6da844918da..a013f67329a 100644 --- a/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.cpp +++ b/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.cpp @@ -1,4 +1,5 @@ #include "yql_yt_request_options.h" +#include <yt/cpp/mapreduce/common/helpers.h> namespace NYql::NFmr { @@ -18,10 +19,6 @@ TTaskState::TPtr MakeTaskState(ETaskStatus taskStatus, const TString& taskId, co return MakeIntrusive<TTaskState>(taskStatus, taskId, taskErrorMessage, stats); } -TString TFmrChunkMeta::ToString() const { - return TStringBuilder() << TableId << ":" << PartId << ":" << std::to_string(Chunk); -} - } // namespace NYql::NFmr template<> @@ -41,11 +38,56 @@ void Out<NYql::NFmr::TFmrError>(IOutputStream& out, const NYql::NFmr::TFmrError& } template<> -void Out<NYql::NFmr::TFmrChunkMeta>(IOutputStream& out, const NYql::NFmr::TFmrChunkMeta& meta) { - out << meta.ToString(); +void Out<NYql::NFmr::TTableStats>(IOutputStream& out, const NYql::NFmr::TTableStats& tableStats) { + out << tableStats.Chunks << " chunks, " << tableStats.Rows << " rows, " << tableStats.DataWeight << " data weight"; } template<> -void Out<NYql::NFmr::TTableStats>(IOutputStream& out, const NYql::NFmr::TTableStats& tableStats) { - out << tableStats.Chunks << " chunks, " << tableStats.Rows << " rows, " << tableStats.DataWeight << " data weight"; +void Out<NYql::NFmr::TTableRange>(IOutputStream& out, const NYql::NFmr::TTableRange& range) { + out << "TableRange with part id: " << range.PartId << " , min chunk: " << range.MinChunk << " , max chunk: " << range.MaxChunk << "\n"; +} + +template<> +void Out<NYql::NFmr::TFmrTableInputRef>(IOutputStream& out, const NYql::NFmr::TFmrTableInputRef& inputRef) { + out << "FmrTableInputRef consisting of " << inputRef.TableRanges.size() << " table ranges:\n"; + out << "TableId: " << inputRef.TableId << "\n"; + for (auto& range: inputRef.TableRanges) { + out << range; + } +} + +template<> +void Out<NYql::NFmr::TYtTableTaskRef>(IOutputStream& out, const NYql::NFmr::TYtTableTaskRef& ytTableTaskRef) { + if (!ytTableTaskRef.FilePaths.empty()) { + out << "YtTableTaskRef consisting of " << ytTableTaskRef.FilePaths.size() << " file paths:\n"; + for (auto& filePath: ytTableTaskRef.FilePaths) { + out << filePath << " "; + } + } else { + out << "YtTableTaskRef consisting of " << ytTableTaskRef.RichPaths.size() << " rich yt paths:\n"; + for (auto& richPath: ytTableTaskRef.RichPaths) { + out << NodeToYsonString(NYT::PathToNode(richPath)) << "\n"; + } + } +} + +template<> +void Out<NYql::NFmr::TTaskTableRef>(IOutputStream& out, const NYql::NFmr::TTaskTableRef& taskTableRef) { + if (auto* ytTableTaskRef = std::get_if<NYql::NFmr::TYtTableTaskRef>(&taskTableRef)) { + out << *ytTableTaskRef; + } else { + out << std::get<NYql::NFmr::TFmrTableInputRef>(taskTableRef); + } +} + +template<> +void Out<NYql::NFmr::TTaskTableInputRef>(IOutputStream& out, const NYql::NFmr::TTaskTableInputRef& taskTableInputRef) { + for (auto& taskTableRef: taskTableInputRef.Inputs) { + out << taskTableRef; + } +} + +template<> +void Out<NYql::NFmr::TChunkStats>(IOutputStream& out, const NYql::NFmr::TChunkStats& chunkStats) { + out << chunkStats.Rows << " rows " << chunkStats.DataWeight << " dataWeight\n"; } diff --git a/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h b/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h index 743c20ea339..53bf1d3d4b5 100644 --- a/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h +++ b/yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h @@ -7,6 +7,8 @@ #include <util/string/builder.h> #include <vector> +#include <yt/cpp/mapreduce/interface/common.h> + namespace NYql::NFmr { enum class EOperationStatus { @@ -44,7 +46,8 @@ enum class EFmrComponent { enum class EFmrErrorReason { ReasonUnknown, - UserError // TODO Add more reasons + UserError + // TODO - return FallbackQuery or FallbackOperation instead of UserError, pass info to gateway. }; struct TFmrError { @@ -66,9 +69,18 @@ struct TYtTableRef { TString Cluster; TMaybe<TString> FilePath = Nothing(); + // TODO - maybe just use TRichYPath here also instead of Path and Cluster? + bool operator == (const TYtTableRef&) const = default; }; +struct TYtTableTaskRef { + std::vector<NYT::TRichYPath> RichPaths; + std::vector<TString> FilePaths; + + bool operator == (const TYtTableTaskRef&) const = default; +}; // corresponds to a partition of several yt input tables. + struct TFmrTableId { TString Id; @@ -83,26 +95,21 @@ struct TFmrTableId { struct TFmrTableRef { TFmrTableId FmrTableId; + bool operator == (const TFmrTableRef&) const = default; }; struct TTableRange { TString PartId; ui64 MinChunk = 0; ui64 MaxChunk = 1; -}; - -struct TFmrChunkMeta { - TString TableId; - TString PartId; - ui64 Chunk = 0; - - TString ToString() const; -}; + bool operator == (const TTableRange&) const = default; +}; // Corresnponds to range [MinChunk, MaxChunk) struct TFmrTableInputRef { TString TableId; std::vector<TTableRange> TableRanges; -}; + bool operator == (const TFmrTableInputRef&) const = default; +}; // Corresponds to part of table with fixed TableId but several PartIds, Empty TablesRanges means that this table is not present in task. struct TFmrTableOutputRef { TString TableId; @@ -118,6 +125,18 @@ struct TTableStats { bool operator == (const TTableStats&) const = default; }; +struct TChunkStats { + ui64 Rows = 0; + ui64 DataWeight = 0; + bool operator == (const TChunkStats&) const = default; +}; + +struct TTableChunkStats { + TString PartId; + std::vector<TChunkStats> PartIdChunkStats; + bool operator == (const TTableChunkStats&) const = default; +}; // detailed statistics for all chunks in partition + } // namespace NYql::NFmr namespace std { @@ -147,12 +166,12 @@ namespace std { namespace NYql::NFmr { struct TStatistics { - std::unordered_map<TFmrTableOutputRef, TTableStats> OutputTables; + std::unordered_map<TFmrTableOutputRef, TTableChunkStats> OutputTables; }; using TOperationTableRef = std::variant<TYtTableRef, TFmrTableRef>; -using TTaskTableRef = std::variant<TYtTableRef, TFmrTableInputRef>; +using TTaskTableRef = std::variant<TYtTableTaskRef, TFmrTableInputRef>; struct TUploadOperationParams { TFmrTableRef Input; @@ -170,7 +189,7 @@ struct TDownloadOperationParams { }; struct TDownloadTaskParams { - TYtTableRef Input; + TYtTableTaskRef Input; TFmrTableOutputRef Output; }; @@ -179,8 +198,12 @@ struct TMergeOperationParams { TFmrTableRef Output; }; +struct TTaskTableInputRef { + std::vector<TTaskTableRef> Inputs; +}; // Corresponds to task input tables, which can consist parts of either fmr or yt input tables. + struct TMergeTaskParams { - std::vector<TTaskTableRef> Input; + TTaskTableInputRef Input; TFmrTableOutputRef Output; }; @@ -191,7 +214,7 @@ struct TMapOperationParams { }; struct TMapTaskParams { - std::vector<TTaskTableRef> Input; + TTaskTableInputRef Input; std::vector<TFmrTableOutputRef> Output; TString Executable; }; diff --git a/yt/yql/providers/yt/fmr/table_data_service/local/yql_yt_table_data_service_local.cpp b/yt/yql/providers/yt/fmr/table_data_service/local/yql_yt_table_data_service_local.cpp index cd1e1a21699..b901fd3de89 100644 --- a/yt/yql/providers/yt/fmr/table_data_service/local/yql_yt_table_data_service_local.cpp +++ b/yt/yql/providers/yt/fmr/table_data_service/local/yql_yt_table_data_service_local.cpp @@ -1,5 +1,7 @@ #include "yql_yt_table_data_service_local.h" +#include <util/system/mutex.h> #include <yt/yql/providers/yt/fmr/utils/yql_yt_table_data_service_key.h> +#include <yql/essentials/utils/log/log.h> namespace NYql::NFmr { @@ -12,6 +14,8 @@ public: } NThreading::TFuture<void> Put(const TString& key, const TString& value) { + TGuard<TMutex> guard(Mutex_); + YQL_CLOG(TRACE, FastMapReduce) << "Putting key " << key << " to local table data service"; auto& map = Data_[std::hash<TString>()(key) % NumParts_]; auto it = map.find(key); if (it != map.end()) { @@ -22,6 +26,8 @@ public: } NThreading::TFuture<TMaybe<TString>> Get(const TString& key) { + TGuard<TMutex> guard(Mutex_); + YQL_CLOG(TRACE, FastMapReduce) << "Getting key " << key << " from local table data service"; TMaybe<TString> value = Nothing(); auto& map = Data_[std::hash<TString>()(key) % NumParts_]; auto it = map.find(key); @@ -32,6 +38,8 @@ public: } NThreading::TFuture<void> Delete(const TString& key) { + TGuard<TMutex> guard(Mutex_); + YQL_CLOG(TRACE, FastMapReduce) << "Deleting key " << key << " from local table data service"; auto& map = Data_[std::hash<TString>()(key) % NumParts_]; auto it = map.find(key); if (it == map.end()) { @@ -44,6 +52,7 @@ public: private: std::vector<std::unordered_map<TString, TString>> Data_; const ui32 NumParts_; + TMutex Mutex_ = TMutex(); }; } // namespace diff --git a/yt/yql/providers/yt/fmr/utils/ut/ya.make b/yt/yql/providers/yt/fmr/utils/ut/ya.make index 2e7c9b94a65..b2c94c2502a 100644 --- a/yt/yql/providers/yt/fmr/utils/ut/ya.make +++ b/yt/yql/providers/yt/fmr/utils/ut/ya.make @@ -7,7 +7,7 @@ SRCS( PEERDIR( yt/yql/providers/yt/fmr/utils yt/yql/providers/yt/fmr/request_options - yt/yql/providers/yt/fmr/yt_service/mock + yt/yql/providers/yt/fmr/yt_job_service/mock yql/essentials/parser/pg_wrapper yql/essentials/parser/pg_wrapper/interface yql/essentials/public/udf diff --git a/yt/yql/providers/yt/fmr/utils/ut/yql_yt_parse_records_ut.cpp b/yt/yql/providers/yt/fmr/utils/ut/yql_yt_parse_records_ut.cpp index a0736aa96f3..d642ca7a6f8 100644 --- a/yt/yql/providers/yt/fmr/utils/ut/yql_yt_parse_records_ut.cpp +++ b/yt/yql/providers/yt/fmr/utils/ut/yql_yt_parse_records_ut.cpp @@ -1,9 +1,9 @@ #include <library/cpp/testing/unittest/registar.h> +#include <yt/cpp/mapreduce/common/helpers.h> #include <yt/yql/providers/yt/fmr/utils/yql_yt_parse_records.h> -#include <yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.h> +#include <yt/yql/providers/yt/fmr/yt_job_service/mock/yql_yt_job_service_mock.h> #include <yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h> -#include <yt/yql/providers/yt/fmr/yt_service/mock/yql_yt_yt_service_mock.h> using namespace NYql::NFmr; @@ -12,13 +12,15 @@ Y_UNIT_TEST_SUITE(ParseRecordTests) { TString inputYsonContent = "{\"key\"=\"075\";\"subkey\"=\"1\";\"value\"=\"abc\"};\n" "{\"key\"=\"800\";\"subkey\"=\"2\";\"value\"=\"ddd\"};\n"; TYtTableRef testYtTable = TYtTableRef{.Path = "test_path", .Cluster = "hahn"}; - std::unordered_map<TYtTableRef, TString> inputTables{{testYtTable, inputYsonContent}}; + auto richPath = NYT::TRichYPath("test_path").Cluster("test_cluster"); + std::unordered_map<TString, TString> inputTables{{NYT::NodeToCanonicalYsonString(NYT::PathToNode(richPath)), inputYsonContent}}; + std::unordered_map<TYtTableRef, TString> outputTables; - auto ytService = MakeMockYtService(inputTables, outputTables); + auto ytJobService = MakeMockYtJobService(inputTables, outputTables); - auto reader = ytService->MakeReader(testYtTable, TClusterConnection()); - auto writer = ytService->MakeWriter(testYtTable, TClusterConnection()); + auto reader = ytJobService->MakeReader(richPath); + auto writer = ytJobService->MakeWriter(testYtTable, TClusterConnection()); auto cancelFlag = std::make_shared<std::atomic<bool>>(false); ParseRecords(reader, writer, 1, 10, cancelFlag); writer->Flush(); diff --git a/yt/yql/providers/yt/fmr/utils/ya.make b/yt/yql/providers/yt/fmr/utils/ya.make index 0d683128179..8bd317b26d5 100644 --- a/yt/yql/providers/yt/fmr/utils/ya.make +++ b/yt/yql/providers/yt/fmr/utils/ya.make @@ -1,6 +1,7 @@ LIBRARY() SRCS( + yql_yt_client.cpp yql_yt_log_context.cpp yql_yt_parse_records.cpp yql_yt_table_data_service_key.cpp @@ -8,8 +9,9 @@ SRCS( PEERDIR( library/cpp/http/io + yt/cpp/mapreduce/client yt/cpp/mapreduce/interface - yt/yql/providers/yt/fmr/yt_service/impl + yt/yql/providers/yt/fmr/request_options yt/yql/providers/yt/codec yql/essentials/utils ) diff --git a/yt/yql/providers/yt/fmr/utils/yql_yt_client.cpp b/yt/yql/providers/yt/fmr/utils/yql_yt_client.cpp new file mode 100644 index 00000000000..3eaeba98d9f --- /dev/null +++ b/yt/yql/providers/yt/fmr/utils/yql_yt_client.cpp @@ -0,0 +1,14 @@ +#include "yql_yt_client.h" + +namespace NYql::NFmr { + +NYT::IClientPtr CreateClient(const TClusterConnection& clusterConnection) { + NYT::TCreateClientOptions createOpts; + auto token = clusterConnection.Token; + if (token) { + createOpts.Token(*token); + } + return NYT::CreateClient(clusterConnection.YtServerName, createOpts); +} + +} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/utils/yql_yt_client.h b/yt/yql/providers/yt/fmr/utils/yql_yt_client.h new file mode 100644 index 00000000000..3d4f2f07224 --- /dev/null +++ b/yt/yql/providers/yt/fmr/utils/yql_yt_client.h @@ -0,0 +1,10 @@ +#pragma once + +#include <yt/cpp/mapreduce/interface/client.h> +#include <yt/yql/providers/yt/fmr/request_options/yql_yt_request_options.h> + +namespace NYql::NFmr { + +NYT::IClientPtr CreateClient(const TClusterConnection& clusterConnection); + +} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/worker/impl/ut/ya.make b/yt/yql/providers/yt/fmr/worker/impl/ut/ya.make index 790f845c3e8..7bde0ed7f9b 100644 --- a/yt/yql/providers/yt/fmr/worker/impl/ut/ya.make +++ b/yt/yql/providers/yt/fmr/worker/impl/ut/ya.make @@ -6,6 +6,7 @@ SRCS( PEERDIR( yt/yql/providers/yt/fmr/coordinator/impl + yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service/file yt/yql/providers/yt/fmr/job_factory/impl yt/yql/providers/yt/fmr/worker/impl ) diff --git a/yt/yql/providers/yt/fmr/worker/impl/ut/yql_yt_worker_ut.cpp b/yt/yql/providers/yt/fmr/worker/impl/ut/yql_yt_worker_ut.cpp index f19c5365bdf..79ccfa46f70 100644 --- a/yt/yql/providers/yt/fmr/worker/impl/ut/yql_yt_worker_ut.cpp +++ b/yt/yql/providers/yt/fmr/worker/impl/ut/yql_yt_worker_ut.cpp @@ -6,12 +6,13 @@ #include <util/thread/pool.h> #include <yt/yql/providers/yt/fmr/worker/impl/yql_yt_worker_impl.h> #include <yt/yql/providers/yt/fmr/coordinator/impl/yql_yt_coordinator_impl.h> +#include <yt/yql/providers/yt/fmr/coordinator/yt_coordinator_service//file/yql_yt_file_coordinator_service.h> #include <yt/yql/providers/yt/fmr/job_factory/impl/yql_yt_job_factory_impl.h> namespace NYql::NFmr { TDownloadOperationParams downloadOperationParams{ - .Input = TYtTableRef{"Path","Cluster"}, + .Input = TYtTableRef{"Path","Cluster", "FilePath"}, .Output = TFmrTableRef{{"Cluster", "Path"}} }; @@ -26,7 +27,7 @@ TStartOperationRequest CreateOperationRequest(ETaskType taskType = ETaskType::Do Y_UNIT_TEST_SUITE(FmrWorkerTests) { Y_UNIT_TEST(GetSuccessfulOperationResult) { - auto coordinator = MakeFmrCoordinator(); + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); auto operationResults = std::make_shared<TString>("no_result_yet"); auto func = [&] (TTask::TPtr /*task*/, std::shared_ptr<std::atomic<bool>> cancelFlag) { while (!cancelFlag->load()) { @@ -48,7 +49,7 @@ Y_UNIT_TEST_SUITE(FmrWorkerTests) { } Y_UNIT_TEST(CancelOperation) { - auto coordinator = MakeFmrCoordinator(); + auto coordinator = MakeFmrCoordinator(TFmrCoordinatorSettings(), MakeFileYtCoordinatorService()); auto operationResults = std::make_shared<TString>("no_result_yet"); auto func = [&] (TTask::TPtr /*task*/, std::shared_ptr<std::atomic<bool>> cancelFlag) { int numIterations = 0; @@ -79,7 +80,7 @@ Y_UNIT_TEST_SUITE(FmrWorkerTests) { TFmrCoordinatorSettings coordinatorSettings{}; coordinatorSettings.WorkersNum = 2; coordinatorSettings.RandomProvider = CreateDeterministicRandomProvider(3); - auto coordinator = MakeFmrCoordinator(coordinatorSettings); + auto coordinator = MakeFmrCoordinator(coordinatorSettings, MakeFileYtCoordinatorService()); std::shared_ptr<std::atomic<ui32>> operationResult = std::make_shared<std::atomic<ui32>>(0); auto func = [&] (TTask::TPtr /*task*/, std::shared_ptr<std::atomic<bool>> cancelFlag) { while (!cancelFlag->load()) { diff --git a/yt/yql/providers/yt/fmr/worker/impl/yql_yt_worker_impl.cpp b/yt/yql/providers/yt/fmr/worker/impl/yql_yt_worker_impl.cpp index ce06e6d0783..ee8ce692f00 100644 --- a/yt/yql/providers/yt/fmr/worker/impl/yql_yt_worker_impl.cpp +++ b/yt/yql/providers/yt/fmr/worker/impl/yql_yt_worker_impl.cpp @@ -50,15 +50,20 @@ public: } } + ui64 maxParallelJobCount = JobFactory_->GetMaxParallelJobCount(); + YQL_ENSURE(maxParallelJobCount >= WorkerState_->TaskStatuses.size()); + ui64 availableSlots = maxParallelJobCount - WorkerState_->TaskStatuses.size(); auto heartbeatRequest = THeartbeatRequest( WorkerId_, VolatileId_, - taskStates + taskStates, + availableSlots ); auto heartbeatResponseFuture = Coordinator_->SendHeartbeatResponse(heartbeatRequest); auto heartbeatResponse = heartbeatResponseFuture.GetValueSync(); std::vector<TTask::TPtr> tasksToRun = heartbeatResponse.TasksToRun; std::unordered_set<TString> taskToDeleteIds = heartbeatResponse.TaskToDeleteIds; + YQL_ENSURE(tasksToRun.size() <= availableSlots); with_lock(WorkerState_->Mutex) { for (auto task: tasksToRun) { diff --git a/yt/yql/providers/yt/fmr/yt_service/file/ut/ya.make b/yt/yql/providers/yt/fmr/yt_job_service/file/ut/ya.make index 6f4ff700b0d..884cda6f788 100644 --- a/yt/yql/providers/yt/fmr/yt_service/file/ut/ya.make +++ b/yt/yql/providers/yt/fmr/yt_job_service/file/ut/ya.make @@ -1,11 +1,11 @@ UNITTEST() SRCS( - yql_yt_file_yt_service_ut.cpp + yql_yt_file_yt_job_service_ut.cpp ) PEERDIR( - yt/yql/providers/yt/fmr/yt_service/file + yt/yql/providers/yt/fmr/yt_job_service/file yt/yql/providers/yt/gateway/file ) diff --git a/yt/yql/providers/yt/fmr/yt_service/file/ut/yql_yt_file_yt_service_ut.cpp b/yt/yql/providers/yt/fmr/yt_job_service/file/ut/yql_yt_file_yt_job_service_ut.cpp index acc5867c454..6a013963cd7 100644 --- a/yt/yql/providers/yt/fmr/yt_service/file/ut/yql_yt_file_yt_service_ut.cpp +++ b/yt/yql/providers/yt/fmr/yt_job_service/file/ut/yql_yt_file_yt_job_service_ut.cpp @@ -1,26 +1,26 @@ #include <library/cpp/testing/unittest/registar.h> #include <util/stream/file.h> -#include <yt/yql/providers/yt/fmr/yt_service/file/yql_yt_file_yt_service.h> -#include <yt/yql/providers/yt/gateway/file/yql_yt_file_text_yson.h> // TODO - REMOVE +#include <yt/yql/providers/yt/fmr/yt_job_service/file/yql_yt_file_yt_job_service.h> +#include <yt/yql/providers/yt/gateway/file/yql_yt_file_text_yson.h> namespace NYql::NFmr { -Y_UNIT_TEST_SUITE(FileYtServiceTest) { - Y_UNIT_TEST(CheckReaderAndWriter) { +Y_UNIT_TEST_SUITE(FileYtServiceTests) { + Y_UNIT_TEST(CheckFileReaderAndWriter) { TString inputYsonContent = "{\"key\"=\"075\";\"subkey\"=\"1\";\"value\"=\"abc\"};\n" "{\"key\"=\"800\";\"subkey\"=\"2\";\"value\"=\"ddd\"};\n"; TTempFileHandle file{}; TYtTableRef ytTable{.Path = "test_path", .Cluster = "hahn", .FilePath = file.Name()}; - auto fileService = MakeFileYtSerivce(); + auto fileService = MakeFileYtJobSerivce(); auto writer = fileService->MakeWriter(ytTable, TClusterConnection()); writer->Write(inputYsonContent.data(), inputYsonContent.size()); writer->Flush(); TFileInput input(file.Name()); - auto reader = fileService->MakeReader(ytTable, TClusterConnection()); + auto reader = fileService->MakeReader(file.Name()); TStringStream binaryYsonStream; TStringStream textYsonStream; binaryYsonStream << reader->ReadAll(); diff --git a/yt/yql/providers/yt/fmr/yt_service/file/ya.make b/yt/yql/providers/yt/fmr/yt_job_service/file/ya.make index dcd0969e3e6..13cfe7ebd3d 100644 --- a/yt/yql/providers/yt/fmr/yt_service/file/ya.make +++ b/yt/yql/providers/yt/fmr/yt_job_service/file/ya.make @@ -1,13 +1,13 @@ LIBRARY() SRCS( - yql_yt_file_yt_service.cpp + yql_yt_file_yt_job_service.cpp ) PEERDIR( library/cpp/yson yt/yql/providers/yt/gateway/file - yt/yql/providers/yt/fmr/yt_service/interface + yt/yql/providers/yt/fmr/yt_job_service/interface yt/yql/providers/yt/lib/yson_helpers yql/essentials/utils ) diff --git a/yt/yql/providers/yt/fmr/yt_job_service/file/yql_yt_file_yt_job_service.cpp b/yt/yql/providers/yt/fmr/yt_job_service/file/yql_yt_file_yt_job_service.cpp new file mode 100644 index 00000000000..d72f76bd419 --- /dev/null +++ b/yt/yql/providers/yt/fmr/yt_job_service/file/yql_yt_file_yt_job_service.cpp @@ -0,0 +1,68 @@ +#include "yql_yt_file_yt_job_service.h" +#include <library/cpp/yson/parser.h> +#include <util/stream/file.h> +#include <yt/cpp/mapreduce/common/helpers.h> +#include <yt/yql/providers/yt/gateway/file/yql_yt_file_text_yson.h> +#include <yt/yql/providers/yt/lib/yson_helpers/yson_helpers.h> +#include <yql/essentials/utils/yql_panic.h> + +namespace NYql::NFmr { + +namespace { + +class TFileYtTableWriter: public NYT::TRawTableWriter { +public: + TFileYtTableWriter(const TString& filePath): FilePath_(filePath) {} + + void NotifyRowEnd() override { + } + +private: + void DoWrite(const void* buf, size_t len) override { + Buffer_.Append(static_cast<const char*>(buf), len); + } + + void DoFlush() override { + TMemoryInput input(Buffer_.data(), Buffer_.size()); + TFile outputFile(FilePath_, OpenAlways | WrOnly | ForAppend); + TFileOutput outputFileStream(outputFile); + TDoubleHighPrecisionYsonWriter writer(&outputFileStream, ::NYson::EYsonType::ListFragment); + NYson::TYsonParser parser(&writer, &input, ::NYson::EYsonType::ListFragment); + parser.Parse(); + Buffer_.Clear(); + } + + TString FilePath_; + TBuffer Buffer_; +}; + +class TFileYtJobService: public NYql::NFmr::IYtJobService { +public: + +NYT::TRawTableReaderPtr MakeReader( + const std::variant<NYT::TRichYPath, TString>& inputTableRef, + const TClusterConnection& /*clusterConnection*/, + const TYtReaderSettings& /*readerSettings*/ +) override { + TString filePath = std::get<TString>(inputTableRef); + auto textYsonInputs = NFile::MakeTextYsonInputs({{filePath, NFile::TColumnsInfo{}}}, false); + return textYsonInputs[0]; + } + + NYT::TRawTableWriterPtr MakeWriter( + const TYtTableRef& ytTable, + const TClusterConnection& /*clusterConnection*/, + const TYtWriterSettings& /*writerSettings*/ + ) override { + YQL_ENSURE(ytTable.FilePath); + return MakeIntrusive<TFileYtTableWriter>(*ytTable.FilePath); + } +}; + +} // namespace + +IYtJobService::TPtr MakeFileYtJobSerivce() { + return MakeIntrusive<TFileYtJobService>(); +} + +} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/yt_job_service/file/yql_yt_file_yt_job_service.h b/yt/yql/providers/yt/fmr/yt_job_service/file/yql_yt_file_yt_job_service.h new file mode 100644 index 00000000000..a4f7c2426ce --- /dev/null +++ b/yt/yql/providers/yt/fmr/yt_job_service/file/yql_yt_file_yt_job_service.h @@ -0,0 +1,7 @@ +#include <yt/yql/providers/yt/fmr/yt_job_service/interface/yql_yt_job_service.h> + +namespace NYql::NFmr { + +IYtJobService::TPtr MakeFileYtJobSerivce(); + +} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/yt_job_service/impl/ya.make b/yt/yql/providers/yt/fmr/yt_job_service/impl/ya.make new file mode 100644 index 00000000000..53303655e61 --- /dev/null +++ b/yt/yql/providers/yt/fmr/yt_job_service/impl/ya.make @@ -0,0 +1,18 @@ +LIBRARY() + +SRCS( + yql_yt_job_service_impl.cpp +) + +PEERDIR( + library/cpp/yt/error + yt/cpp/mapreduce/client + yt/cpp/mapreduce/common + yt/yql/providers/yt/fmr/yt_job_service/interface + yql/essentials/utils + yql/essentials/utils/log +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.cpp b/yt/yql/providers/yt/fmr/yt_job_service/impl/yql_yt_job_service_impl.cpp index 4634a76031b..3c21287dc5b 100644 --- a/yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.cpp +++ b/yt/yql/providers/yt/fmr/yt_job_service/impl/yql_yt_job_service_impl.cpp @@ -1,28 +1,36 @@ +#include <library/cpp/yt/error/error.h> #include <yt/cpp/mapreduce/common/helpers.h> #include <yt/cpp/mapreduce/interface/client.h> +#include <yt/yql/providers/yt/fmr/utils/yql_yt_client.h> +#include <yql/essentials/utils/log/log.h> +#include <yql/essentials/utils/yql_panic.h> -#include "yql_yt_yt_service_impl.h" +#include "yql_yt_job_service_impl.h" namespace NYql::NFmr { namespace { -class TFmrYtService: public NYql::NFmr::IYtService { +class TFmrYtJobService: public IYtJobService { public: NYT::TRawTableReaderPtr MakeReader( - const TYtTableRef& ytTable, + const std::variant<NYT::TRichYPath, TString>& inputTableRef, const TClusterConnection& clusterConnection, const TYtReaderSettings& readerSettings ) override { + auto richPath = std::get<NYT::TRichYPath>(inputTableRef); + //YQL_CLOG(DEBUG, FastMapReduce) << "Creating reader for input yt table with path " << NYT::NodeToCanonicalYsonString(NYT::PathToNode(richPath)); + YQL_ENSURE(richPath.Cluster_); + TFmrTableId fmrId(*richPath.Cluster_, richPath.Path_); auto client = CreateClient(clusterConnection); auto transaction = client->AttachTransaction(GetGuid(clusterConnection.TransactionId)); - auto path = NYT::TRichYPath(NYT::AddPathPrefix(ytTable.Path, "//")); + auto controlAttributes = NYT::TControlAttributes(); if (!readerSettings.WithAttributes) { controlAttributes.EnableRangeIndex(false).EnableRowIndex(false); } auto readerOptions = NYT::TTableReaderOptions().ControlAttributes(controlAttributes); - return transaction->CreateRawReader(path, NYT::TFormat::YsonBinary(), readerOptions); + return transaction->CreateRawReader(richPath, NYT::TFormat::YsonBinary(), readerOptions); } NYT::TRawTableWriterPtr MakeWriter( @@ -33,29 +41,19 @@ public: auto client = CreateClient(clusterConnection); auto transaction = client->AttachTransaction(GetGuid(clusterConnection.TransactionId)); TString ytPath = NYT::AddPathPrefix(ytTable.Path, "//"); - auto richPath = NYT::TRichYPath(ytPath).Append(true); + auto richPath = NYT::TRichYPath(ytPath).Cluster(ytTable.Cluster).Append(true); auto writerOptions = NYT::TTableWriterOptions(); if (writerSetttings.MaxRowWeight) { writerOptions.Config(NYT::TNode()("max_row_weight", *writerSetttings.MaxRowWeight)); } return transaction->CreateRawWriter(richPath, NYT::TFormat::YsonBinary(), writerOptions); } - -private: - NYT::IClientPtr CreateClient(const TClusterConnection& clusterConnection) { - NYT::TCreateClientOptions createOpts; - auto token = clusterConnection.Token; - if (token) { - createOpts.Token(*token); - } - return NYT::CreateClient(clusterConnection.YtServerName, createOpts); - } }; } // namespace -IYtService::TPtr MakeFmrYtSerivce() { - return MakeIntrusive<TFmrYtService>(); +IYtJobService::TPtr MakeYtJobSerivce() { + return MakeIntrusive<TFmrYtJobService>(); } } // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/yt_job_service/impl/yql_yt_job_service_impl.h b/yt/yql/providers/yt/fmr/yt_job_service/impl/yql_yt_job_service_impl.h new file mode 100644 index 00000000000..bac4abf6ab7 --- /dev/null +++ b/yt/yql/providers/yt/fmr/yt_job_service/impl/yql_yt_job_service_impl.h @@ -0,0 +1,7 @@ +#include <yt/yql/providers/yt/fmr/yt_job_service/interface/yql_yt_job_service.h> + +namespace NYql::NFmr { + +IYtJobService::TPtr MakeYtJobSerivce(); + +} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/yt_service/interface/ya.make b/yt/yql/providers/yt/fmr/yt_job_service/interface/ya.make index 032f645e022..a824de924f5 100644 --- a/yt/yql/providers/yt/fmr/yt_service/interface/ya.make +++ b/yt/yql/providers/yt/fmr/yt_job_service/interface/ya.make @@ -1,7 +1,7 @@ LIBRARY() SRCS( - yql_yt_yt_service.cpp + yql_yt_job_service.cpp ) PEERDIR( diff --git a/yt/yql/providers/yt/fmr/yt_job_service/interface/yql_yt_job_service.cpp b/yt/yql/providers/yt/fmr/yt_job_service/interface/yql_yt_job_service.cpp new file mode 100644 index 00000000000..7b3bac7a269 --- /dev/null +++ b/yt/yql/providers/yt/fmr/yt_job_service/interface/yql_yt_job_service.cpp @@ -0,0 +1 @@ +#include "yql_yt_job_service.h" diff --git a/yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.h b/yt/yql/providers/yt/fmr/yt_job_service/interface/yql_yt_job_service.h index 77204493e70..0975fe729c7 100644 --- a/yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.h +++ b/yt/yql/providers/yt/fmr/yt_job_service/interface/yql_yt_job_service.h @@ -7,22 +7,23 @@ namespace NYql::NFmr { struct TYtReaderSettings { - bool WithAttributes = false; // Enable RowIndex and RangeIndex + bool WithAttributes = false; // Enable RowIndex and RangeIndex, for now only mode = false is supported. }; struct TYtWriterSettings { TMaybe<ui64> MaxRowWeight = Nothing(); }; -class IYtService: public TThrRefBase { +class IYtJobService: public TThrRefBase { public: - virtual ~IYtService() = default; + virtual ~IYtJobService() = default; - using TPtr = TIntrusivePtr<IYtService>; + using TPtr = TIntrusivePtr<IYtJobService>; + // Either RichPath to actual Yt table or filepath is passed depending on type of underlying gateway. virtual NYT::TRawTableReaderPtr MakeReader( - const TYtTableRef& ytTable, - const TClusterConnection& clusterConnection, + const std::variant<NYT::TRichYPath, TString>& inputTableRef, + const TClusterConnection& clusterConnection = TClusterConnection(), const TYtReaderSettings& settings = TYtReaderSettings() ) = 0; diff --git a/yt/yql/providers/yt/fmr/yt_service/mock/ya.make b/yt/yql/providers/yt/fmr/yt_job_service/mock/ya.make index e586d08ec3a..4896db2ef75 100644 --- a/yt/yql/providers/yt/fmr/yt_service/mock/ya.make +++ b/yt/yql/providers/yt/fmr/yt_job_service/mock/ya.make @@ -1,12 +1,13 @@ LIBRARY() SRCS( - yql_yt_yt_service_mock.cpp + yql_yt_job_service_mock.cpp ) PEERDIR( + yt/cpp/mapreduce/common yt/cpp/mapreduce/interface - yt/yql/providers/yt/fmr/yt_service/interface + yt/yql/providers/yt/fmr/yt_job_service/interface yql/essentials/utils ) diff --git a/yt/yql/providers/yt/fmr/yt_service/mock/yql_yt_yt_service_mock.cpp b/yt/yql/providers/yt/fmr/yt_job_service/mock/yql_yt_job_service_mock.cpp index 337c82d2b31..e31b7a22c17 100644 --- a/yt/yql/providers/yt/fmr/yt_service/mock/yql_yt_yt_service_mock.cpp +++ b/yt/yql/providers/yt/fmr/yt_job_service/mock/yql_yt_job_service_mock.cpp @@ -1,5 +1,6 @@ -#include "yql_yt_yt_service_mock.h" +#include "yql_yt_job_service_mock.h" +#include <yt/cpp/mapreduce/common/helpers.h> #include <yt/cpp/mapreduce/interface/io.h> #include <yql/essentials/utils/yql_panic.h> @@ -58,14 +59,20 @@ private: TBuffer Buffer_; }; -class TMockYtService: public NYql::NFmr::IYtService { +class TMockYtJobService: public NYql::NFmr::IYtJobService { public: - TMockYtService(const std::unordered_map<TYtTableRef, TString>& inputTables, std::unordered_map<TYtTableRef, TString>& outputTables) + TMockYtJobService(const std::unordered_map<TString, TString>& inputTables, std::unordered_map<TYtTableRef, TString>& outputTables) : InputTables_(inputTables), OutputTables_(outputTables) {} - NYT::TRawTableReaderPtr MakeReader(const TYtTableRef& ytTableRef, const TClusterConnection&, const TYtReaderSettings&) override { - YQL_ENSURE(InputTables_.contains(ytTableRef)); - return MakeIntrusive<TMockYtTableReader>(InputTables_[ytTableRef]); + virtual NYT::TRawTableReaderPtr MakeReader( + const std::variant<NYT::TRichYPath, TString>& inputTableRef, + const TClusterConnection& /*clusterConnection*/, + const TYtReaderSettings& /*settings*/ + ) override { + auto richPath = std::get<NYT::TRichYPath>(inputTableRef); + TString richPathStr = NYT::NodeToCanonicalYsonString(NYT::PathToNode(richPath)); + YQL_ENSURE(InputTables_.contains(richPathStr)); + return {MakeIntrusive<TMockYtTableReader>(InputTables_[richPathStr])}; } NYT::TRawTableWriterPtr MakeWriter(const TYtTableRef& ytTableRef, const TClusterConnection&, const TYtWriterSettings&) override { @@ -76,14 +83,14 @@ public: } private: - std::unordered_map<TYtTableRef, TString> InputTables_; // table -> textYsonContent + std::unordered_map<TString, TString> InputTables_; // rich yt path in string form -> total textYsonContent of it std::unordered_map<TYtTableRef, TString>& OutputTables_; }; } // namespace -IYtService::TPtr MakeMockYtService(const std::unordered_map<TYtTableRef, TString>& inputTables, std::unordered_map<TYtTableRef, TString>& outputTables) { - return MakeIntrusive<TMockYtService>(inputTables, outputTables); +IYtJobService::TPtr MakeMockYtJobService(const std::unordered_map<TString, TString>& inputTables, std::unordered_map<TYtTableRef, TString>& outputTables) { + return MakeIntrusive<TMockYtJobService>(inputTables, outputTables); } } // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/yt_job_service/mock/yql_yt_job_service_mock.h b/yt/yql/providers/yt/fmr/yt_job_service/mock/yql_yt_job_service_mock.h new file mode 100644 index 00000000000..f13dc79de1d --- /dev/null +++ b/yt/yql/providers/yt/fmr/yt_job_service/mock/yql_yt_job_service_mock.h @@ -0,0 +1,9 @@ +#pragma once + +#include <yt/yql/providers/yt/fmr/yt_job_service/interface/yql_yt_job_service.h> + +namespace NYql::NFmr { + +IYtJobService::TPtr MakeMockYtJobService(const std::unordered_map<TString, TString>& inputTables, std::unordered_map<TYtTableRef, TString>& outputTables); + +} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/yt_service/file/yql_yt_file_yt_service.cpp b/yt/yql/providers/yt/fmr/yt_service/file/yql_yt_file_yt_service.cpp deleted file mode 100644 index 1c33ed430e9..00000000000 --- a/yt/yql/providers/yt/fmr/yt_service/file/yql_yt_file_yt_service.cpp +++ /dev/null @@ -1,68 +0,0 @@ -#include "yql_yt_file_yt_service.h" -#include <library/cpp/yson/parser.h> -#include <util/stream/file.h> -#include <yt/yql/providers/yt/gateway/file/yql_yt_file_text_yson.h> -#include <yt/yql/providers/yt/lib/yson_helpers/yson_helpers.h> -#include <yql/essentials/utils/yql_panic.h> - -namespace NYql::NFmr { - -namespace { - -class TFileYtTableWriter: public NYT::TRawTableWriter { - public: - TFileYtTableWriter(const TString& filePath): FilePath_(filePath) {} - - void NotifyRowEnd() override { - } - - private: - void DoWrite(const void* buf, size_t len) override { - Buffer_.Append(static_cast<const char*>(buf), len); - } - - void DoFlush() override { - TMemoryInput input(Buffer_.data(), Buffer_.size()); - TFileOutput outputFileStream(FilePath_); - TDoubleHighPrecisionYsonWriter writer(&outputFileStream, ::NYson::EYsonType::ListFragment); - NYson::TYsonParser parser(&writer, &input, ::NYson::EYsonType::ListFragment); - parser.Parse(); - Buffer_.Clear(); - } - - TString FilePath_; - TBuffer Buffer_; - }; - - -class TFileYtService: public NYql::NFmr::IYtService { -public: - - NYT::TRawTableReaderPtr MakeReader( - const TYtTableRef& ytTable, - const TClusterConnection& /*clusterConnection*/, - const TYtReaderSettings& /*readerSettings*/ - ) override { - YQL_ENSURE(ytTable.FilePath); - auto textYsonInputs = NFile::MakeTextYsonInputs({{*ytTable.FilePath, NFile::TColumnsInfo{}}}, false); - return textYsonInputs[0]; - } - - NYT::TRawTableWriterPtr MakeWriter( - const TYtTableRef& ytTable, - const TClusterConnection& /*clusterConnection*/, - const TYtWriterSettings& /*writerSettings*/ - ) override { - YQL_ENSURE(ytTable.FilePath); - return MakeIntrusive<TFileYtTableWriter>(*ytTable.FilePath); - } - -}; - -} // namespace - -IYtService::TPtr MakeFileYtSerivce() { - return MakeIntrusive<TFileYtService>(); -} - -} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/yt_service/file/yql_yt_file_yt_service.h b/yt/yql/providers/yt/fmr/yt_service/file/yql_yt_file_yt_service.h deleted file mode 100644 index ea5e8027b9a..00000000000 --- a/yt/yql/providers/yt/fmr/yt_service/file/yql_yt_file_yt_service.h +++ /dev/null @@ -1,7 +0,0 @@ -#include <yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.h> - -namespace NYql::NFmr { - -IYtService::TPtr MakeFileYtSerivce(); - -} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/yt_service/impl/ya.make b/yt/yql/providers/yt/fmr/yt_service/impl/ya.make deleted file mode 100644 index 1b37156ed2f..00000000000 --- a/yt/yql/providers/yt/fmr/yt_service/impl/ya.make +++ /dev/null @@ -1,15 +0,0 @@ -LIBRARY() - -SRCS( - yql_yt_yt_service_impl.cpp -) - -PEERDIR( - yt/cpp/mapreduce/client - yt/cpp/mapreduce/common - yt/yql/providers/yt/fmr/yt_service/interface -) - -YQL_LAST_ABI_VERSION() - -END() diff --git a/yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.h b/yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.h deleted file mode 100644 index bf1962cac5a..00000000000 --- a/yt/yql/providers/yt/fmr/yt_service/impl/yql_yt_yt_service_impl.h +++ /dev/null @@ -1,7 +0,0 @@ -#include <yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.h> - -namespace NYql::NFmr { - -IYtService::TPtr MakeFmrYtSerivce(); - -} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.cpp b/yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.cpp deleted file mode 100644 index fbcafbc3a27..00000000000 --- a/yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "yql_yt_yt_service.h" diff --git a/yt/yql/providers/yt/fmr/yt_service/mock/yql_yt_yt_service_mock.h b/yt/yql/providers/yt/fmr/yt_service/mock/yql_yt_yt_service_mock.h deleted file mode 100644 index b10426c2f03..00000000000 --- a/yt/yql/providers/yt/fmr/yt_service/mock/yql_yt_yt_service_mock.h +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -#include <yt/yql/providers/yt/fmr/yt_service/interface/yql_yt_yt_service.h> - -namespace NYql::NFmr { - -IYtService::TPtr MakeMockYtService(const std::unordered_map<TYtTableRef, TString>& inputTables, std::unordered_map<TYtTableRef, TString>& outputTables); - -} // namespace NYql::NFmr diff --git a/yt/yql/providers/yt/gateway/file/yql_yt_file_text_yson.h b/yt/yql/providers/yt/gateway/file/yql_yt_file_text_yson.h index a28606598d3..01c400d8527 100644 --- a/yt/yql/providers/yt/gateway/file/yql_yt_file_text_yson.h +++ b/yt/yql/providers/yt/gateway/file/yql_yt_file_text_yson.h @@ -1,4 +1,5 @@ #pragma once + #include <yt/cpp/mapreduce/interface/fwd.h> #include <yt/cpp/mapreduce/interface/common.h> diff --git a/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.cpp b/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.cpp index ad261da0d62..6b2d145b24d 100644 --- a/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.cpp +++ b/yt/yql/providers/yt/gateway/fmr/yql_yt_fmr.cpp @@ -134,6 +134,7 @@ public: TString sessionId = options.SessionId(); auto config = options.Config(); TRunResult result; + YQL_ENSURE(fmrOperationResult.TablesStats.size() == outputTables.size()); for (size_t i = 0; i < outputTables.size(); ++i) { auto outputTable = outputTables[i]; TFmrTableId fmrOutputTableId = {outputTable.Cluster, outputTable.Path}; diff --git a/yt/yql/providers/yt/gateway/lib/yt_helpers.cpp b/yt/yql/providers/yt/gateway/lib/yt_helpers.cpp index f5b82c718a3..0a5251692e2 100644 --- a/yt/yql/providers/yt/gateway/lib/yt_helpers.cpp +++ b/yt/yql/providers/yt/gateway/lib/yt_helpers.cpp @@ -128,6 +128,7 @@ THashSet<TStringBuf> SERVICE_YQL_ATTRS = { TStringBuf("_yql_runner"), TStringBuf("_yql_op_id"), TStringBuf("_yql_op_title"), + TStringBuf("_yql_op_url"), TStringBuf("_yql_query_name"), }; @@ -562,6 +563,9 @@ NYT::TNode YqlOpOptionsToAttrs(const TYqlOperationOptions& opOpts) { if (auto id = opOpts.Id.GetOrElse(TString())) { attrs["_yql_op_id"] = id; } + if (auto url = opOpts.Url.GetOrElse(TString())) { + attrs["_yql_op_url"] = url; + } if (auto title = opOpts.Title.GetOrElse(TString())) { attrs["_yql_op_title"] = title; } diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_exec_ctx.h b/yt/yql/providers/yt/gateway/native/yql_yt_exec_ctx.h index d60ebb49f43..29f716a275e 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_exec_ctx.h +++ b/yt/yql/providers/yt/gateway/native/yql_yt_exec_ctx.h @@ -19,7 +19,7 @@ #include <yql/essentials/core/expr_nodes/yql_expr_nodes.h> #include <yql/essentials/core/file_storage/file_storage.h> #include <yql/essentials/minikql/mkql_function_registry.h> -#include <yql/essentials/utils/log/context.h> +#include <yql/essentials/utils/log/log.h> #include <yt/cpp/mapreduce/interface/common.h> #include <library/cpp/yson/node/node.h> @@ -163,6 +163,7 @@ public: bool DisableAnonymousClusterAccess_; bool Hidden = false; IMetricsRegistryPtr Metrics; + TOperationProgress::EOpBlockStatus BlockStatus = TOperationProgress::EOpBlockStatus::None; }; @@ -231,6 +232,19 @@ public: Session_->ProgressWriter_(progress); } + void ReportNodeBlockStatus() const { + auto publicId = Options_.PublicId(); + if (!publicId) { + return; + } + + YQL_CLOG(INFO, ProviderYt) << "Reporting " << BlockStatus << " block status for node #" << *publicId; + auto progress = TOperationProgress(TString(YtProviderName), *publicId, + TOperationProgress::EState::InProgress); + progress.BlockStatus = BlockStatus; + Session_->ProgressWriter_(progress); + } + [[nodiscard]] NThreading::TFuture<bool> LookupQueryCacheAsync() { if (QueryCacheItem) { diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp index 0dac584bc6f..fe3db65474e 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp +++ b/yt/yql/providers/yt/gateway/native/yql_yt_native.cpp @@ -1002,8 +1002,7 @@ public: if (auto outputOp = opBase.Maybe<TYtOutputOpBase>()) { execCtx->SetOutput(outputOp.Cast().Output()); } - - ReportBlockStatus(opBase, execCtx); + execCtx->ReportNodeBlockStatus(); TFuture<void> future; if (auto op = opBase.Maybe<TYtSort>()) { @@ -5964,6 +5963,10 @@ private: ctx->CodeSnippets_.emplace_back("code", ConvertToAst(*root, *exprCtx, 0, true).Root->ToString(TAstPrintFlags::ShortQuote | TAstPrintFlags::PerLine | TAstPrintFlags::AdaptArbitraryContent)); } + + if (TYtOutputOpBase::Match(root)) { + ctx->BlockStatus = DetermineBlockStatus(TYtOutputOpBase(root)); + } } return ctx; } @@ -5992,47 +5995,32 @@ private: return Nothing(); } - static void ReportBlockStatus(const TYtOpBase& op, const TExecContext<TRunOptions>::TPtr& execCtx) { - if (execCtx->Options_.PublicId().Empty()) { - return; - } - - auto opPublicId = *execCtx->Options_.PublicId(); - - TOperationProgress::EOpBlockStatus status; + static TOperationProgress::EOpBlockStatus DetermineBlockStatus(const TYtOutputOpBase& op) { if (auto map = op.Maybe<TYtMap>()) { - status = DetermineProgramBlockStatus(map.Cast().Mapper().Body().Ref()); - } else if (auto map = op.Maybe<TYtReduce>()) { - status = DetermineProgramBlockStatus(map.Cast().Reducer().Body().Ref()); - } else if (auto map = op.Maybe<TYtMapReduce>()) { - status = DetermineProgramBlockStatus(map.Cast().Reducer().Body().Ref()); - if (auto mapLambda = map.Cast().Mapper().Maybe<TCoLambda>()) { + return DetermineProgramBlockStatus(map.Cast().Mapper().Body().Ref()); + } else if (auto reduce = op.Maybe<TYtReduce>()) { + return DetermineProgramBlockStatus(reduce.Cast().Reducer().Body().Ref()); + } else if (auto mapReduce = op.Maybe<TYtMapReduce>()) { + auto status = DetermineProgramBlockStatus(mapReduce.Cast().Reducer().Body().Ref()); + if (auto mapLambda = mapReduce.Cast().Mapper().Maybe<TCoLambda>()) { status = TOperationProgress::CombineBlockStatuses(status, DetermineProgramBlockStatus(mapLambda.Cast().Body().Ref())); } + return status; } else if (auto fill = op.Maybe<TYtFill>()) { - status = DetermineProgramBlockStatus(fill.Cast().Content().Body().Ref()); + return DetermineProgramBlockStatus(fill.Cast().Content().Body().Ref()); } else if (op.Maybe<TYtSort>()) { - return; + return TOperationProgress::EOpBlockStatus::None; } else if (op.Maybe<TYtCopy>()) { - return; + return TOperationProgress::EOpBlockStatus::None; } else if (op.Maybe<TYtMerge>()) { - return; + return TOperationProgress::EOpBlockStatus::None; } else if (op.Maybe<TYtTouch>()) { - return; - } else if (op.Maybe<TYtDropTable>()) { - return; - } else if (op.Maybe<TYtStatOut>()) { - return; + return TOperationProgress::EOpBlockStatus::None; } else if (op.Maybe<TYtDqProcessWrite>()) { - return; + return TOperationProgress::EOpBlockStatus::None; } else { YQL_ENSURE(false, "unknown operation: " << op.Ref().Content()); } - - YQL_CLOG(INFO, ProviderYt) << "Reporting " << status << " block status for operation " << op.Ref().Content() << " with public id #" << opPublicId; - auto p = TOperationProgress(TString(YtProviderName), opPublicId, TOperationProgress::EState::InProgress); - p.BlockStatus = status; - execCtx->Session_->ProgressWriter_(p); } private: diff --git a/yt/yql/providers/yt/gateway/native/yql_yt_spec.cpp b/yt/yql/providers/yt/gateway/native/yql_yt_spec.cpp index 52fc088538c..5391dc9c31c 100644 --- a/yt/yql/providers/yt/gateway/native/yql_yt_spec.cpp +++ b/yt/yql/providers/yt/gateway/native/yql_yt_spec.cpp @@ -406,6 +406,15 @@ void FillSpec(NYT::TNode& spec, } } + auto probabily = TString(std::to_string(settings->_EnforceRegexpProbabilityFail.Get().GetOrElse(0))); + if (opProps.HasFlags(EYtOpProp::WithMapper)) { + spec["mapper"]["environment"]["YQL_RE2_REGEXP_PROBABILITY_FAIL"] = probabily; + } + + if (opProps.HasFlags(EYtOpProp::WithReducer)) { + spec["reducer"]["environment"]["YQL_RE2_REGEXP_PROBABILITY_FAIL"] = probabily; + } + if (settings->SuspendIfAccountLimitExceeded.Get(cluster).GetOrElse(false)) { spec["suspend_operation_if_account_limit_exceeded"] = true; } @@ -700,9 +709,10 @@ void FillUserJobSpecImpl(NYT::TUserJobSpec& spec, if (defaultMemoryLimit || fileMemUsage || llvmMemUsage || extraUsage.Memory || tmpFsSize) { const ui64 memIoBuffers = YQL_JOB_CODEC_MEM * (static_cast<size_t>(!execCtx.InputTables_.empty()) + execCtx.OutTables_.size()); + const ui64 arrowMemoryPoolReserve = (execCtx.BlockStatus != TOperationProgress::EOpBlockStatus::None ? YQL_ARROW_MEMORY_POOL_RESERVE : 0); const ui64 finalMemLimit = Max<ui64>( defaultMemoryLimit, - 128_MB + fileMemUsage + extraUsage.Memory + tmpFsSize + memIoBuffers, + 128_MB + fileMemUsage + extraUsage.Memory + tmpFsSize + memIoBuffers + arrowMemoryPoolReserve, llvmMemUsage + memIoBuffers // LLVM consumes memory only once on job start, but after IO initialization ); YQL_CLOG(DEBUG, ProviderYt) << "Job memory limit: " << finalMemLimit @@ -712,6 +722,7 @@ void FillUserJobSpecImpl(NYT::TUserJobSpec& spec, << ", extra: " << extraUsage.Memory << ", extra tmpfs: " << tmpFsSize << ", I/O buffers: " << memIoBuffers + << ", Arrow pool reserve: " << arrowMemoryPoolReserve << ")"; spec.MemoryLimit(static_cast<i64>(finalMemLimit)); } diff --git a/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp b/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp index 5612a85adf9..c4618e02879 100644 --- a/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp +++ b/yt/yql/providers/yt/provider/yql_yt_dq_integration.cpp @@ -395,7 +395,7 @@ public: return false; } - if (pragma == "pooltrees") { + if (pragma == "pooltrees" && node.ChildrenSize() >= 5) { auto pools = NPrivate::GetDefaultParser<TVector<TString>>()(TString{node.Child(4)->Content()}); for (const auto& pool : pools) { if (!POOL_TREES_WHITELIST.contains(pool)) { diff --git a/yt/yt/client/api/config.cpp b/yt/yt/client/api/config.cpp index ff1bb03903f..369d23d88dd 100644 --- a/yt/yt/client/api/config.cpp +++ b/yt/yt/client/api/config.cpp @@ -4,6 +4,8 @@ namespace NYT::NApi { +using namespace NYTree; + //////////////////////////////////////////////////////////////////////////////// void TTableMountCacheConfig::Register(TRegistrar registrar) @@ -132,6 +134,27 @@ void TJournalChunkWriterConfig::Register(TRegistrar registrar) //////////////////////////////////////////////////////////////////////////////// +void TDynamicJournalWriterConfig::Register(TRegistrar registrar) { + registrar.Parameter("validate_erasure_coding", &TThis::ValidateErasureCoding) + .Optional(); +} + +//////////////////////////////////////////////////////////////////////////////// + +TJournalWriterConfigPtr TJournalWriterConfig::ApplyDynamic( + const TDynamicJournalWriterConfigPtr& dynamicConfig) const +{ + auto config = CloneYsonStruct(MakeStrong(this)); + config->ApplyDynamicInplace(dynamicConfig); + config->Postprocess(); + return config; +} + +void TJournalWriterConfig::ApplyDynamicInplace(const TDynamicJournalWriterConfigPtr& dynamicConfig) +{ + UpdateYsonStructField(ValidateErasureCoding, dynamicConfig->ValidateErasureCoding); +} + void TJournalWriterConfig::Register(TRegistrar registrar) { registrar.Parameter("max_chunk_row_count", &TThis::MaxChunkRowCount) diff --git a/yt/yt/client/api/config.h b/yt/yt/client/api/config.h index 8666bdd7ac6..b5e0b44bd10 100644 --- a/yt/yt/client/api/config.h +++ b/yt/yt/client/api/config.h @@ -197,6 +197,20 @@ DEFINE_REFCOUNTED_TYPE(TJournalChunkWriterConfig) //////////////////////////////////////////////////////////////////////////////// +struct TDynamicJournalWriterConfig + : public virtual NYTree::TYsonStruct +{ + std::optional<bool> ValidateErasureCoding; + + REGISTER_YSON_STRUCT(TDynamicJournalWriterConfig); + + static void Register(TRegistrar registrar); +}; + +DEFINE_REFCOUNTED_TYPE(TDynamicJournalWriterConfig) + +//////////////////////////////////////////////////////////////////////////////// + struct TJournalWriterConfig : public TJournalChunkWriterConfig { @@ -219,6 +233,9 @@ struct TJournalWriterConfig std::optional<TDuration> OpenDelay; + TJournalWriterConfigPtr ApplyDynamic(const TDynamicJournalWriterConfigPtr& dynamicConfig) const; + void ApplyDynamicInplace(const TDynamicJournalWriterConfigPtr& dynamicConfig); + REGISTER_YSON_STRUCT(TJournalWriterConfig); static void Register(TRegistrar registrar); diff --git a/yt/yt/client/api/cypress_client.h b/yt/yt/client/api/cypress_client.h index 90f01674b09..89ab1702daf 100644 --- a/yt/yt/client/api/cypress_client.h +++ b/yt/yt/client/api/cypress_client.h @@ -88,8 +88,8 @@ struct TLockNodeOptions , public TPrerequisiteOptions { bool Waitable = false; - std::optional<TString> ChildKey; - std::optional<TString> AttributeKey; + std::optional<std::string> ChildKey; + std::optional<std::string> AttributeKey; }; struct TLockNodeResult diff --git a/yt/yt/client/api/operation_client.cpp b/yt/yt/client/api/operation_client.cpp index a8e025874db..d648a0333d3 100644 --- a/yt/yt/client/api/operation_client.cpp +++ b/yt/yt/client/api/operation_client.cpp @@ -94,7 +94,7 @@ void TListJobsContinuationTokenSerializer::Register(TRegistrar registrar) .DontSerializeDefault(); } -TString EncodeNewToken(TListJobsOptions&& options, int jobCount) +std::string EncodeNewToken(TListJobsOptions&& options, int jobCount) { options.Offset += jobCount; options.ContinuationToken.reset(); @@ -106,7 +106,7 @@ TString EncodeNewToken(TListJobsOptions&& options, int jobCount) return Base64Encode(optionsYson.ToString()); } -TListJobsOptions DecodeListJobsOptionsFromToken(const TString& continuationToken) +TListJobsOptions DecodeListJobsOptionsFromToken(const std::string& continuationToken) { auto optionsYson = TYsonString(Base64StrictDecode(continuationToken)); return ConvertTo<TListJobsContinuationToken>(optionsYson); diff --git a/yt/yt/client/api/operation_client.h b/yt/yt/client/api/operation_client.h index f75a310b73e..7683f127466 100644 --- a/yt/yt/client/api/operation_client.h +++ b/yt/yt/client/api/operation_client.h @@ -257,8 +257,8 @@ ASSIGN_EXTERNAL_YSON_SERIALIZER(TListJobsContinuationToken, TListJobsContinuatio //////////////////////////////////////////////////////////////////////////////// -TString EncodeNewToken(TListJobsOptions&& options, int jobCount); -TListJobsOptions DecodeListJobsOptionsFromToken(const TString& continuationToken); +std::string EncodeNewToken(TListJobsOptions&& options, int jobCount); +TListJobsOptions DecodeListJobsOptionsFromToken(const std::string& continuationToken); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/api/public.h b/yt/yt/client/api/public.h index f8aafb33322..45313f48a2c 100644 --- a/yt/yt/client/api/public.h +++ b/yt/yt/client/api/public.h @@ -26,17 +26,17 @@ namespace NYT::NApi { using TClusterTag = NObjectClient::TCellTag; // Keep in sync with NRpcProxy::NProto::EMasterReadKind. -// On cache miss request is redirected to next level cache: -// Local cache -> (node) cache -> master cache +// On cache miss request is redirected to next level as follows: +// Client-side cache -> cache -> master-side cache DEFINE_ENUM(EMasterChannelKind, + // These options cover the majority of cases. ((Leader) (0)) ((Follower) (1)) - // Use local (per-connection) cache. - ((LocalCache) (4)) - // Use cache located on nodes. - ((Cache) (2)) - // Use cache located on masters (if caching on masters is enabled). - ((MasterCache) (3)) + ((Cache) (2)) // cluster-wide cache + + // These are advanced options. Typically you don't need these. + ((MasterSideCache) (3)) // cache located on masters + ((ClientSideCache) (4)) // local (per-connection) cache ); DEFINE_ENUM(EUserWorkloadCategory, @@ -174,6 +174,7 @@ DECLARE_REFCOUNTED_STRUCT(TJournalReaderConfig) DECLARE_REFCOUNTED_STRUCT(TJournalChunkWriterConfig) DECLARE_REFCOUNTED_STRUCT(TJournalWriterConfig) +DECLARE_REFCOUNTED_STRUCT(TDynamicJournalWriterConfig) DECLARE_REFCOUNTED_STRUCT(TJournalChunkWriterOptions) @@ -208,15 +209,15 @@ inline const NYPath::TYPath ClusterNamePath("//sys/@cluster_name"); inline const NYPath::TYPath HttpProxiesPath("//sys/http_proxies"); inline const NYPath::TYPath RpcProxiesPath("//sys/rpc_proxies"); inline const NYPath::TYPath GrpcProxiesPath("//sys/grpc_proxies"); -inline const TString AliveNodeName("alive"); -inline const TString BannedAttributeName("banned"); -inline const TString RoleAttributeName("role"); -inline const TString AddressesAttributeName("addresses"); -inline const TString BalancersAttributeName("balancers"); +inline const std::string AliveNodeName("alive"); +inline const std::string BannedAttributeName("banned"); +inline const std::string RoleAttributeName("role"); +inline const std::string AddressesAttributeName("addresses"); +inline const std::string BalancersAttributeName("balancers"); inline const std::string DefaultRpcProxyRole("default"); inline const std::string DefaultHttpProxyRole("data"); -inline const TString JournalPayloadKey("payload"); -inline const TString HunkPayloadKey("payload"); +inline const std::string JournalPayloadKey("payload"); +inline const std::string HunkPayloadKey("payload"); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/api/query_tracker_client.h b/yt/yt/client/api/query_tracker_client.h index 69990a38761..c3598fc08ca 100644 --- a/yt/yt/client/api/query_tracker_client.h +++ b/yt/yt/client/api/query_tracker_client.h @@ -12,7 +12,7 @@ namespace NYT::NApi { struct TQueryTrackerOptions { - TString QueryTrackerStage = "production"; + std::string QueryTrackerStage = NQueryTrackerClient::ProductionStage; }; DEFINE_ENUM(EContentType, @@ -170,7 +170,7 @@ struct TGetQueryTrackerInfoOptions struct TGetQueryTrackerInfoResult { - TString QueryTrackerStage; + std::string QueryTrackerStage; std::string ClusterName; NYson::TYsonString SupportedFeatures; std::vector<std::string> AccessControlObjects; diff --git a/yt/yt/client/api/queue_client.h b/yt/yt/client/api/queue_client.h index 658dbd5b397..adf1adb3349 100644 --- a/yt/yt/client/api/queue_client.h +++ b/yt/yt/client/api/queue_client.h @@ -77,6 +77,7 @@ struct TListQueueConsumerRegistrationsResult struct TCreateQueueProducerSessionOptions : public TTimeoutOptions + , public TMutatingOptions { NYTree::INodePtr UserMeta; }; diff --git a/yt/yt/client/api/rpc_proxy/client_base.cpp b/yt/yt/client/api/rpc_proxy/client_base.cpp index 941b7dfb529..594987f73f2 100644 --- a/yt/yt/client/api/rpc_proxy/client_base.cpp +++ b/yt/yt/client/api/rpc_proxy/client_base.cpp @@ -59,6 +59,7 @@ using NYT::FromProto; //////////////////////////////////////////////////////////////////////////////// constexpr i64 MaxTracingTagLength = 1'000; +constexpr i64 MinQueryTailPartSize = 100; static const TString DisabledSelectQueryTracingTag = "Tag is disabled, look for enable_select_query_tracing_tag parameter"; std::string SanitizeTracingTag(TStringBuf originalTag) @@ -69,6 +70,16 @@ std::string SanitizeTracingTag(TStringBuf originalTag) return Format("%v ... TRUNCATED", originalTag.substr(0, MaxTracingTagLength)); } +std::string SanitizeTracingQuery(TStringBuf originalQuery) +{ + if (originalQuery.size() <= MaxTracingTagLength) { + return std::string(originalQuery); + } + return Format("%v...<truncated>...%v", + originalQuery.substr(0, MaxTracingTagLength - MinQueryTailPartSize), + originalQuery.substr(originalQuery.size() - MinQueryTailPartSize)); +} + void EnrichTracingForLookupRequest(NTracing::TTraceContext::TTagList& tagList, TStringBuf path, const auto& columns) { if (NTracing::IsCurrentTraceContextRecorded()) { @@ -810,7 +821,7 @@ TFuture<TDistributedWriteSessionWithCookies> TClientBase::StartDistributedWriteS TDistributedWriteSessionWithCookies sessionWithCookies; sessionWithCookies.Session = ConvertTo<TSignedDistributedWriteSessionPtr>(TYsonString(result->signed_session())), sessionWithCookies.Cookies = std::move(cookies); - return std::move(sessionWithCookies); + return sessionWithCookies; })); } @@ -1041,7 +1052,7 @@ TFuture<TSelectRowsResult> TClientBase::SelectRows( if (NTracing::IsCurrentTraceContextRecorded()) { if (config->EnableSelectQueryTracingTag) { - req->TracingTags().emplace_back("yt.query", SanitizeTracingTag(query)); + req->TracingTags().emplace_back("yt.query", SanitizeTracingQuery(query)); } else { req->TracingTags().emplace_back("yt.query", DisabledSelectQueryTracingTag); } diff --git a/yt/yt/client/api/rpc_proxy/client_impl.cpp b/yt/yt/client/api/rpc_proxy/client_impl.cpp index 61cf0e45dcf..d0caae4c51e 100644 --- a/yt/yt/client/api/rpc_proxy/client_impl.cpp +++ b/yt/yt/client/api/rpc_proxy/client_impl.cpp @@ -920,7 +920,7 @@ TFuture<std::vector<TListQueueConsumerRegistrationsResult>> TClient::ListQueueCo TFuture<TCreateQueueProducerSessionResult> TClient::CreateQueueProducerSession( const TRichYPath& producerPath, const TRichYPath& queuePath, - const NQueueClient::TQueueProducerSessionId& sessionId, + const TQueueProducerSessionId& sessionId, const TCreateQueueProducerSessionOptions& options) { auto proxy = CreateApiServiceProxy(); @@ -934,6 +934,7 @@ TFuture<TCreateQueueProducerSessionResult> TClient::CreateQueueProducerSession( if (options.UserMeta) { ToProto(req->mutable_user_meta(), ConvertToYsonString(options.UserMeta).ToString()); } + ToProto(req->mutable_mutating_options(), options); return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspCreateQueueProducerSessionPtr& rsp) { INodePtr userMeta; @@ -952,7 +953,7 @@ TFuture<TCreateQueueProducerSessionResult> TClient::CreateQueueProducerSession( TFuture<void> TClient::RemoveQueueProducerSession( const NYPath::TRichYPath& producerPath, const NYPath::TRichYPath& queuePath, - const NQueueClient::TQueueProducerSessionId& sessionId, + const TQueueProducerSessionId& sessionId, const TRemoveQueueProducerSessionOptions& options) { auto proxy = CreateApiServiceProxy(); @@ -2533,7 +2534,7 @@ TFuture<TGetQueryTrackerInfoResult> TClient::GetQueryTrackerInfo( return req->Invoke().Apply(BIND([] (const TApiServiceProxy::TRspGetQueryTrackerInfoPtr& rsp) { return TGetQueryTrackerInfoResult{ - .QueryTrackerStage = FromProto<TString>(rsp->query_tracker_stage()), + .QueryTrackerStage = FromProto<std::string>(rsp->query_tracker_stage()), .ClusterName = FromProto<std::string>(rsp->cluster_name()), .SupportedFeatures = TYsonString(rsp->supported_features()), .AccessControlObjects = FromProto<std::vector<std::string>>(rsp->access_control_objects()), diff --git a/yt/yt/client/api/security_client.h b/yt/yt/client/api/security_client.h index 24c1831cf59..87fa27b94f5 100644 --- a/yt/yt/client/api/security_client.h +++ b/yt/yt/client/api/security_client.h @@ -88,7 +88,7 @@ struct TIssueTemporaryTokenOptions struct TIssueTokenResult { - TString Token; + std::string Token; //! Cypress node corresponding to issued token. //! Deleting this node will revoke the token. NCypressClient::TNodeId NodeId; diff --git a/yt/yt/client/chunk_client/public.h b/yt/yt/client/chunk_client/public.h index 50debd140a3..532f5de59da 100644 --- a/yt/yt/client/chunk_client/public.h +++ b/yt/yt/client/chunk_client/public.h @@ -196,7 +196,7 @@ DEFINE_ENUM(EChunkAvailabilityPolicy, ((Repairable) (2)) ); -// Keep in sync with NChunkServer::ETableChunkFormat. +// Keep in sync with SerializeChunkFormatAsTableChunkFormat. DEFINE_ENUM_WITH_UNDERLYING_TYPE(EChunkFormat, i8, // Sentinels. ((Unknown) (-1)) diff --git a/yt/yt/client/chunk_client/read_limit.cpp b/yt/yt/client/chunk_client/read_limit.cpp index 3d6b7fd6c57..0b8bc86e57d 100644 --- a/yt/yt/client/chunk_client/read_limit.cpp +++ b/yt/yt/client/chunk_client/read_limit.cpp @@ -334,7 +334,7 @@ void Serialize(const TLegacyReadLimit& readLimit, IYsonConsumer* consumer) namespace { template <class T> -std::optional<T> FindReadLimitComponent(const IAttributeDictionaryPtr& attributes, const TString& key) +std::optional<T> FindReadLimitComponent(const IAttributeDictionaryPtr& attributes, const std::string& key) { try { return attributes->Find<T>(key); @@ -469,7 +469,7 @@ void Serialize(const TLegacyReadRange& readRange, NYson::IYsonConsumer* consumer namespace { template <class T> -std::optional<T> FindReadRangeComponent(const IAttributeDictionaryPtr& attributes, const TString& key) +std::optional<T> FindReadRangeComponent(const IAttributeDictionaryPtr& attributes, const std::string& key) { try { return attributes->Find<T>(key); diff --git a/yt/yt/client/driver/cypress_commands.cpp b/yt/yt/client/driver/cypress_commands.cpp index ca42597fb1d..b1f68eb469f 100644 --- a/yt/yt/client/driver/cypress_commands.cpp +++ b/yt/yt/client/driver/cypress_commands.cpp @@ -355,14 +355,14 @@ void TLockCommand::Register(TRegistrar registrar) }) .Optional(/*init*/ false); - registrar.ParameterWithUniversalAccessor<std::optional<TString>>( + registrar.ParameterWithUniversalAccessor<std::optional<std::string>>( "child_key", [] (TThis* command) -> auto& { return command->Options.ChildKey; }) .Optional(/*init*/ false); - registrar.ParameterWithUniversalAccessor<std::optional<TString>>( + registrar.ParameterWithUniversalAccessor<std::optional<std::string>>( "attribute_key", [] (TThis* command) -> auto& { return command->Options.AttributeKey; diff --git a/yt/yt/client/driver/proxy_discovery_cache.cpp b/yt/yt/client/driver/proxy_discovery_cache.cpp index e69e60f9789..83abddaf927 100644 --- a/yt/yt/client/driver/proxy_discovery_cache.cpp +++ b/yt/yt/client/driver/proxy_discovery_cache.cpp @@ -92,7 +92,7 @@ private: } TGetNodeOptions options; - options.ReadFrom = EMasterChannelKind::LocalCache; + options.ReadFrom = EMasterChannelKind::ClientSideCache; options.Attributes = {BalancersAttributeName}; TYPath path; @@ -123,7 +123,7 @@ private: TFuture<TProxyDiscoveryResponse> GetResponseByAddresses(const TProxyDiscoveryRequest& request) { TGetNodeOptions options; - options.ReadFrom = EMasterChannelKind::LocalCache; + options.ReadFrom = EMasterChannelKind::ClientSideCache; options.SuppressUpstreamSync = true; options.SuppressTransactionCoordinatorSync = true; options.Attributes = {BannedAttributeName, RoleAttributeName, AddressesAttributeName}; diff --git a/yt/yt/client/driver/query_commands.cpp b/yt/yt/client/driver/query_commands.cpp index 25aca01345f..4942a6da9f8 100644 --- a/yt/yt/client/driver/query_commands.cpp +++ b/yt/yt/client/driver/query_commands.cpp @@ -35,7 +35,7 @@ void TStartQueryCommand::Register(TRegistrar registrar) }) .Optional(/*init*/ false); - registrar.ParameterWithUniversalAccessor<TString>( + registrar.ParameterWithUniversalAccessor<std::string>( "stage", [] (TThis* command) -> auto& { return command->Options.QueryTrackerStage; @@ -104,7 +104,7 @@ void TAbortQueryCommand::Register(TRegistrar registrar) { registrar.Parameter("query_id", &TThis::QueryId); - registrar.ParameterWithUniversalAccessor<TString>( + registrar.ParameterWithUniversalAccessor<std::string>( "stage", [] (TThis* command) -> auto& { return command->Options.QueryTrackerStage; @@ -128,7 +128,7 @@ void TGetQueryResultCommand::Register(TRegistrar registrar) registrar.Parameter("query_id", &TThis::QueryId); registrar.Parameter("result_index", &TThis::ResultIndex) .Default(0); - registrar.ParameterWithUniversalAccessor<TString>( + registrar.ParameterWithUniversalAccessor<std::string>( "stage", [] (TThis* command) -> auto& { return command->Options.QueryTrackerStage; @@ -153,7 +153,7 @@ void TReadQueryResultCommand::Register(TRegistrar registrar) registrar.Parameter("result_index", &TThis::ResultIndex) .Default(0); - registrar.ParameterWithUniversalAccessor<TString>( + registrar.ParameterWithUniversalAccessor<std::string>( "stage", [] (TThis* command) -> auto& { return command->Options.QueryTrackerStage; @@ -215,7 +215,7 @@ void TGetQueryCommand::Register(TRegistrar registrar) }) .Optional(/*init*/ false); - registrar.ParameterWithUniversalAccessor<TString>( + registrar.ParameterWithUniversalAccessor<std::string>( "stage", [] (TThis* command) -> auto& { return command->Options.QueryTrackerStage; @@ -235,7 +235,7 @@ void TGetQueryCommand::DoExecute(ICommandContextPtr context) void TListQueriesCommand::Register(TRegistrar registrar) { - registrar.ParameterWithUniversalAccessor<TString>( + registrar.ParameterWithUniversalAccessor<std::string>( "stage", [] (TThis* command) -> auto& { return command->Options.QueryTrackerStage; @@ -353,7 +353,7 @@ void TAlterQueryCommand::Register(TRegistrar registrar) }) .Optional(/*init*/ false); - registrar.ParameterWithUniversalAccessor<TString>( + registrar.ParameterWithUniversalAccessor<std::string>( "stage", [] (TThis* command) -> auto& { return command->Options.QueryTrackerStage; @@ -372,7 +372,7 @@ void TAlterQueryCommand::DoExecute(ICommandContextPtr context) void TGetQueryTrackerInfoCommand::Register(TRegistrar registrar) { - registrar.ParameterWithUniversalAccessor<TString>( + registrar.ParameterWithUniversalAccessor<std::string>( "stage", [] (TThis* command) -> auto& { return command->Options.QueryTrackerStage; diff --git a/yt/yt/client/driver/shuffle_commands.cpp b/yt/yt/client/driver/shuffle_commands.cpp index 31a4d93eb7b..691870e399d 100644 --- a/yt/yt/client/driver/shuffle_commands.cpp +++ b/yt/yt/client/driver/shuffle_commands.cpp @@ -7,6 +7,7 @@ #include <yt/yt/client/formats/config.h> #include <yt/yt/client/signature/signature.h> +#include <yt/yt/client/signature/validator.h> #include <yt/yt/client/table_client/adapters.h> #include <yt/yt/client/table_client/table_output.h> @@ -14,6 +15,7 @@ namespace NYT::NDriver { +using namespace NApi; using namespace NConcurrency; using namespace NFormats; using namespace NTableClient; @@ -80,6 +82,15 @@ void TReadShuffleDataCommand::DoExecute(ICommandContextPtr context) { auto client = context->GetClient(); + const auto& signatureValidator = context->GetDriver()->GetSignatureValidator(); + auto validationSuccessful = WaitFor(signatureValidator->Validate(SignedShuffleHandle.Underlying())) + .ValueOrThrow(); + if (!validationSuccessful) { + auto shuffleHandle = ConvertTo<TShuffleHandlePtr>(TYsonStringBuf(SignedShuffleHandle.Underlying()->Payload())); + THROW_ERROR_EXCEPTION("Signature validation failed for shuffle handle") + << TErrorAttribute("shuffle_handle", shuffleHandle); + } + std::optional<std::pair<int, int>> writerIndexRange; if (WriterIndexBegin.has_value()) { writerIndexRange = std::pair(*WriterIndexBegin, *WriterIndexEnd); @@ -140,6 +151,15 @@ void TWriteShuffleDataCommand::DoExecute(ICommandContextPtr context) { auto client = context->GetClient(); + const auto& signatureValidator = context->GetDriver()->GetSignatureValidator(); + auto validationSuccessful = WaitFor(signatureValidator->Validate(SignedShuffleHandle.Underlying())) + .ValueOrThrow(); + if (!validationSuccessful) { + auto shuffleHandle = ConvertTo<TShuffleHandlePtr>(TYsonStringBuf(SignedShuffleHandle.Underlying()->Payload())); + THROW_ERROR_EXCEPTION("Signature validation failed for shuffle handle") + << TErrorAttribute("shuffle_handle", shuffleHandle); + } + Options.OverwriteExistingWriterData = OverwriteExistingWriterData; auto writer = WaitFor(context->GetClient()->CreateShuffleWriter( diff --git a/yt/yt/client/formats/config.h b/yt/yt/client/formats/config.h index 301eb2338e6..69f5e2fd90d 100644 --- a/yt/yt/client/formats/config.h +++ b/yt/yt/client/formats/config.h @@ -126,9 +126,9 @@ DEFINE_REFCOUNTED_TYPE(TDsvFormatConfigBase) struct TYamrFormatConfig : public TYamrFormatConfigBase { - TString Key; - TString Subkey; - TString Value; + std::string Key; + std::string Subkey; + std::string Value; REGISTER_YSON_STRUCT(TYamrFormatConfig); diff --git a/yt/yt/client/misc/io_tags.cpp b/yt/yt/client/misc/io_tags.cpp index 95367e7ddc8..9d5389cb9b0 100644 --- a/yt/yt/client/misc/io_tags.cpp +++ b/yt/yt/client/misc/io_tags.cpp @@ -21,8 +21,7 @@ std::string FormatIOTag(EAggregateIOTag tag) template <class T> void AddTagToBaggage(const NYTree::IAttributeDictionaryPtr& baggage, T tag, TStringBuf value) { - // TODO(babenko): switch to std::string - baggage->Set(TString(FormatIOTag(tag)), value); + baggage->Set(FormatIOTag(tag), value); } template void AddTagToBaggage<ERawIOTag>(const NYTree::IAttributeDictionaryPtr& baggage, ERawIOTag tag, TStringBuf value); diff --git a/yt/yt/client/object_client/helpers.cpp b/yt/yt/client/object_client/helpers.cpp index db59fe02b08..48243cf023a 100644 --- a/yt/yt/client/object_client/helpers.cpp +++ b/yt/yt/client/object_client/helpers.cpp @@ -25,7 +25,8 @@ bool IsSequoiaNode(NObjectClient::EObjectType type) { return type == EObjectType::SequoiaMapNode || - type == EObjectType::SequoiaLink; + type == EObjectType::SequoiaLink || + type == EObjectType::Scion; } bool IsVersionedType(EObjectType type) diff --git a/yt/yt/client/query_tracker_client/public.h b/yt/yt/client/query_tracker_client/public.h index aad588c21df..767c3995cad 100644 --- a/yt/yt/client/query_tracker_client/public.h +++ b/yt/yt/client/query_tracker_client/public.h @@ -44,4 +44,8 @@ DEFINE_STRING_SERIALIZABLE_ENUM(EQueryState, //////////////////////////////////////////////////////////////////////////////// +inline const std::string ProductionStage = "production"; + +//////////////////////////////////////////////////////////////////////////////// + } // namespace NYT::NQueryTrackerClient diff --git a/yt/yt/client/queue_client/public.h b/yt/yt/client/queue_client/public.h index 8e4ffa909c6..88efe734bfe 100644 --- a/yt/yt/client/queue_client/public.h +++ b/yt/yt/client/queue_client/public.h @@ -15,6 +15,7 @@ YT_DEFINE_ERROR_ENUM( ((ZombieEpoch) (3102)) ((InvalidRowSequenceNumbers) (3103)) ((QueueAgentRetriableError) (3104)) + ((QueueAgentObjectIsNotMapped) (3105)) ); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/scheduler/public.h b/yt/yt/client/scheduler/public.h index 11d9d40eb95..8d29145a91b 100644 --- a/yt/yt/client/scheduler/public.h +++ b/yt/yt/client/scheduler/public.h @@ -176,12 +176,13 @@ DEFINE_ENUM(EAbortReason, DEFINE_ENUM_UNKNOWN_VALUE(EAbortReason, Unknown); DEFINE_ENUM(EInterruptionReason, - ((None) (0)) - ((Preemption) (1)) - ((UserRequest) (2)) - ((JobSplit) (3)) - ((Unknown) (4)) - ((JobsDisabledOnNode) (5)) + ((None) (0)) + ((Preemption) (1)) + ((UserRequest) (2)) + ((JobSplit) (3)) + ((Unknown) (4)) + ((JobsDisabledOnNode) (5)) + ((NbdDeviceStopping) (6)) ); DEFINE_ENUM_UNKNOWN_VALUE(EInterruptionReason, Unknown); diff --git a/yt/yt/client/table_client/helpers-inl.h b/yt/yt/client/table_client/helpers-inl.h index 0fcee077aea..b65fe5f9e7b 100644 --- a/yt/yt/client/table_client/helpers-inl.h +++ b/yt/yt/client/table_client/helpers-inl.h @@ -9,6 +9,8 @@ #include <yt/yt/core/yson/protobuf_interop.h> +#include <yt/yt/core/ytree/convert.h> + #include <yt/yt/core/misc/protobuf_helpers.h> #include <yt/yt/core/concurrency/scheduler.h> @@ -131,6 +133,27 @@ struct TRowValueTypesChecker //////////////////////////////////////////////////////////////////////////////// +template <NYTree::CYsonStructDerived T> +void ToUnversionedValue( + TUnversionedValue* unversionedValue, + T value, + const TRowBufferPtr& rowBuffer, + int id, + EValueFlags flags) +{ + ToUnversionedValue(unversionedValue, NYson::ConvertToYsonString(value), rowBuffer, id, flags); +} + +template <NYTree::CYsonStructDerived T> +void FromUnversionedValue( + T* value, + TUnversionedValue unversionedValue) +{ + NYson::TYsonString ysonStringValue; + FromUnversionedValue(&ysonStringValue, unversionedValue); + *value = ConvertTo<T>(ysonStringValue); +} + template <class T> requires TEnumTraits<T>::IsEnum void ToUnversionedValue( @@ -424,7 +447,7 @@ void FromUnversionedValue( void MapToUnversionedValueImpl( TUnversionedValue* unversionedValue, - const std::function<bool(TString*, TUnversionedValue*)> producer, + const std::function<bool(std::string*, TUnversionedValue*)> producer, const TRowBufferPtr& rowBuffer, int id, EValueFlags flags); @@ -440,7 +463,7 @@ void ToUnversionedValue( auto it = map.begin(); MapToUnversionedValueImpl( unversionedValue, - [&] (TString* itemKey, TUnversionedValue* itemValue) mutable -> bool { + [&] (std::string* itemKey, TUnversionedValue* itemValue) mutable -> bool { if (it == map.end()) { return false; } @@ -467,7 +490,7 @@ void FromUnversionedValue( { map->clear(); UnversionedValueToMapImpl( - [&] (TString key) { + [&] (std::string key) { auto pair = map->emplace(FromString<TKey>(std::move(key)), TValue()); return &pair.first->second; }, diff --git a/yt/yt/client/table_client/helpers.cpp b/yt/yt/client/table_client/helpers.cpp index 50e95a59a8f..cfdc4e9ab32 100644 --- a/yt/yt/client/table_client/helpers.cpp +++ b/yt/yt/client/table_client/helpers.cpp @@ -1152,7 +1152,7 @@ void UnversionedValueToListImpl( void MapToUnversionedValueImpl( TUnversionedValue* unversionedValue, - const std::function<bool(TString*, TUnversionedValue*)> producer, + const std::function<bool(std::string*, TUnversionedValue*)> producer, const TRowBufferPtr& rowBuffer, int id, EValueFlags flags) @@ -1162,7 +1162,7 @@ void MapToUnversionedValueImpl( NYT::NYson::TYsonWriter writer(&outputStream); writer.OnBeginMap(); - TString itemKey; + std::string itemKey; TUnversionedValue itemValue; while (true) { if (!producer(&itemKey, &itemValue)) { @@ -1292,10 +1292,10 @@ void UnversionedValueToMapImpl( } private: - const std::function<google::protobuf::Message*(TString)> Appender_; + const std::function<google::protobuf::Message*(std::string)> Appender_; const TProtobufMessageType* const Type_; - std::optional<TString> Key_; + std::optional<std::string> Key_; std::unique_ptr<IYsonConsumer> Underlying_; int Depth_ = 0; @@ -1315,7 +1315,7 @@ void UnversionedValueToMapImpl( { FlushElement(); WireBytes_.clear(); - Key_ = TString(key); + Key_ = std::string(key); Underlying_ = CreateProtobufWriter(&OutputStream_, Type_); } diff --git a/yt/yt/client/table_client/helpers.h b/yt/yt/client/table_client/helpers.h index 802f2389ac9..7bb3b0b629f 100644 --- a/yt/yt/client/table_client/helpers.h +++ b/yt/yt/client/table_client/helpers.h @@ -132,6 +132,18 @@ void FromUnversionedValue(NNet::TIP6Address* value, TUnversionedValue unversione void ToUnversionedValue(TUnversionedValue* unversionedValue, const TError& value, const TRowBufferPtr& rowBuffer, int id = 0, EValueFlags flags = EValueFlags::None); void FromUnversionedValue(TError* value, TUnversionedValue unversionedValue); +template <NYTree::CYsonStructDerived T> +void ToUnversionedValue( + TUnversionedValue* unversionedValue, + T value, + const TRowBufferPtr& rowBuffer, + int id = 0, + EValueFlags flags = EValueFlags::None); +template <NYTree::CYsonStructDerived T> +void FromUnversionedValue( + T* value, + TUnversionedValue unversionedValue); + template <class T> requires TEnumTraits<T>::IsEnum void ToUnversionedValue( diff --git a/yt/yt/client/table_client/public.h b/yt/yt/client/table_client/public.h index d314028ccb7..f0b18ad7b09 100644 --- a/yt/yt/client/table_client/public.h +++ b/yt/yt/client/table_client/public.h @@ -200,6 +200,7 @@ YT_DEFINE_ERROR_ENUM( ((StringLikeValueLengthLimitExceeded)(326)) ((NameTableUpdateFailed) (327)) ((InvalidTableChunkFormat) (328)) + ((UnableToSynchronizeReplicationCard)(329)) ); DEFINE_ENUM(EControlAttribute, diff --git a/yt/yt/client/table_client/table_upload_options.cpp b/yt/yt/client/table_client/table_upload_options.cpp index 3908a272da3..401b04af088 100644 --- a/yt/yt/client/table_client/table_upload_options.cpp +++ b/yt/yt/client/table_client/table_upload_options.cpp @@ -153,9 +153,9 @@ static void ValidateAppendKeyColumns(const TSortColumns& sortColumns, const TTab } } -const std::vector<TString>& GetTableUploadOptionsAttributeKeys() +const std::vector<std::string>& GetTableUploadOptionsAttributeKeys() { - static const std::vector<TString> Result{ + static const std::vector<std::string> Result{ "schema_mode", "optimize_for", "chunk_format", diff --git a/yt/yt/client/table_client/table_upload_options.h b/yt/yt/client/table_client/table_upload_options.h index 1adc0a98d3c..f4c322abcae 100644 --- a/yt/yt/client/table_client/table_upload_options.h +++ b/yt/yt/client/table_client/table_upload_options.h @@ -70,7 +70,7 @@ struct TTableUploadOptions void Persist(const NPhoenix::TPersistenceContext& context); }; -const std::vector<TString>& GetTableUploadOptionsAttributeKeys(); +const std::vector<std::string>& GetTableUploadOptionsAttributeKeys(); TTableUploadOptions GetTableUploadOptions( const NYPath::TRichYPath& path, diff --git a/yt/yt/client/tablet_client/public.cpp b/yt/yt/client/tablet_client/public.cpp index b1bcfc8b8bc..6f272015522 100644 --- a/yt/yt/client/tablet_client/public.cpp +++ b/yt/yt/client/tablet_client/public.cpp @@ -10,14 +10,14 @@ const TStoreId NullStoreId; const TPartitionId NullPartitionId; const THunkStorageId NullHunkStorageId; -const TString TReplicationLogTable::ChangeTypeColumnName("change_type"); -const TString TReplicationLogTable::KeyColumnNamePrefix("key:"); -const TString TReplicationLogTable::ValueColumnNamePrefix("value:"); -const TString TReplicationLogTable::FlagsColumnNamePrefix("flags:"); - -const TString TUnversionedUpdateSchema::ChangeTypeColumnName("$change_type"); -const TString TUnversionedUpdateSchema::ValueColumnNamePrefix("$value:"); -const TString TUnversionedUpdateSchema::FlagsColumnNamePrefix("$flags:"); +const std::string TReplicationLogTable::ChangeTypeColumnName("change_type"); +const std::string TReplicationLogTable::KeyColumnNamePrefix("key:"); +const std::string TReplicationLogTable::ValueColumnNamePrefix("value:"); +const std::string TReplicationLogTable::FlagsColumnNamePrefix("flags:"); + +const std::string TUnversionedUpdateSchema::ChangeTypeColumnName("$change_type"); +const std::string TUnversionedUpdateSchema::ValueColumnNamePrefix("$value:"); +const std::string TUnversionedUpdateSchema::FlagsColumnNamePrefix("$flags:"); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/tablet_client/public.h b/yt/yt/client/tablet_client/public.h index 876d5e723e9..219ee4d3301 100644 --- a/yt/yt/client/tablet_client/public.h +++ b/yt/yt/client/tablet_client/public.h @@ -94,6 +94,7 @@ YT_DEFINE_ERROR_ENUM( ((TabletServantIsNotActive) (1740)) ((UniqueIndexConflict) (1741)) ((TabletReplicationEraMismatch) (1742)) + ((SetOfDynamicStoresHasChanged) (1743)) ); DEFINE_ENUM(EInMemoryMode, @@ -142,10 +143,10 @@ DEFINE_BIT_ENUM(EReplicationLogDataFlags, struct TReplicationLogTable { - static const TString ChangeTypeColumnName; - static const TString KeyColumnNamePrefix; - static const TString ValueColumnNamePrefix; - static const TString FlagsColumnNamePrefix; + static const std::string ChangeTypeColumnName; + static const std::string KeyColumnNamePrefix; + static const std::string ValueColumnNamePrefix; + static const std::string FlagsColumnNamePrefix; }; DEFINE_BIT_ENUM(EUnversionedUpdateDataFlags, @@ -160,9 +161,9 @@ constexpr EUnversionedUpdateDataFlags MaxValidUnversionedUpdateDataFlags = struct TUnversionedUpdateSchema { - static const TString ChangeTypeColumnName; - static const TString ValueColumnNamePrefix; - static const TString FlagsColumnNamePrefix; + static const std::string ChangeTypeColumnName; + static const std::string ValueColumnNamePrefix; + static const std::string FlagsColumnNamePrefix; }; DEFINE_ENUM(ETabletCellHealth, @@ -227,7 +228,7 @@ DEFINE_ENUM(ERowMergerType, ((New) (2)) ); -extern const TString CustomRuntimeDataWatermarkKey; +extern const std::string CustomRuntimeDataWatermarkKey; struct TWatermarkRuntimeDataConfig; struct TWatermarkRuntimeData; diff --git a/yt/yt/client/tablet_client/watermark_runtime_data.cpp b/yt/yt/client/tablet_client/watermark_runtime_data.cpp index 638f21e82ed..da30da8f1c4 100644 --- a/yt/yt/client/tablet_client/watermark_runtime_data.cpp +++ b/yt/yt/client/tablet_client/watermark_runtime_data.cpp @@ -4,7 +4,7 @@ namespace NYT::NTabletClient { //////////////////////////////////////////////////////////////////////////////// -const TString CustomRuntimeDataWatermarkKey("watermark"); +const std::string CustomRuntimeDataWatermarkKey("watermark"); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/client/ypath/rich.cpp b/yt/yt/client/ypath/rich.cpp index ab6e6e8825b..156d7e6b99a 100644 --- a/yt/yt/client/ypath/rich.cpp +++ b/yt/yt/client/ypath/rich.cpp @@ -115,7 +115,7 @@ void AppendAttributes(TStringBuilderBase* builder, const IAttributeDictionary& a } template <class TFunc> -auto RunAttributeAccessor(const TRichYPath& path, const TString& key, TFunc accessor) -> decltype(accessor()) +auto RunAttributeAccessor(const TRichYPath& path, const std::string& key, TFunc accessor) -> decltype(accessor()) { try { return accessor(); @@ -127,7 +127,7 @@ auto RunAttributeAccessor(const TRichYPath& path, const TString& key, TFunc acce } template <class T> -T GetAttribute(const TRichYPath& path, const TString& key, const T& defaultValue) +T GetAttribute(const TRichYPath& path, const std::string& key, const T& defaultValue) { return RunAttributeAccessor(path, key, [&] { return path.Attributes().Get(key, defaultValue); @@ -135,14 +135,14 @@ T GetAttribute(const TRichYPath& path, const TString& key, const T& defaultValue } template <class T> -typename TOptionalTraits<T>::TOptional FindAttribute(const TRichYPath& path, const TString& key) +typename TOptionalTraits<T>::TOptional FindAttribute(const TRichYPath& path, const std::string& key) { return RunAttributeAccessor(path, key, [&] { return path.Attributes().Find<T>(key); }); } -TYsonString FindAttributeYson(const TRichYPath& path, const TString& key) +TYsonString FindAttributeYson(const TRichYPath& path, const std::string& key) { return RunAttributeAccessor(path, key, [&] { return path.Attributes().FindYson(key); diff --git a/yt/yt/core/bus/tcp/connection.cpp b/yt/yt/core/bus/tcp/connection.cpp index e78888d8a2f..52ec0dfd1c2 100644 --- a/yt/yt/core/bus/tcp/connection.cpp +++ b/yt/yt/core/bus/tcp/connection.cpp @@ -1310,6 +1310,11 @@ bool TTcpConnection::OnHandshakePacketReceived() if (EncryptionMode_ == EEncryptionMode::Required || otherEncryptionMode == EEncryptionMode::Required) { if (EncryptionMode_ == EEncryptionMode::Disabled || otherEncryptionMode == EEncryptionMode::Disabled) { + if (ConnectionType_ == EConnectionType::Server) { + // Send handshake response before abort to let client deduce reason. + ProcessQueuedMessages(); + OnSocketWrite(); + } Abort(TError(NBus::EErrorCode::SslError, "TLS/SSL client/server encryption mode compatibility error") << TErrorAttribute("mode", EncryptionMode_) << TErrorAttribute("other_mode", otherEncryptionMode)); diff --git a/yt/yt/core/bus/unittests/ssl_ut.cpp b/yt/yt/core/bus/unittests/ssl_ut.cpp index 4bceb132758..6f65b24c8e1 100644 --- a/yt/yt/core/bus/unittests/ssl_ut.cpp +++ b/yt/yt/core/bus/unittests/ssl_ut.cpp @@ -450,7 +450,9 @@ TEST_F(TSslTest, RequiredAndDisabledEncryptionMode) auto client = CreateBusClient(clientConfig); auto bus = client->CreateBus(New<TEmptyBusHandler>()); - EXPECT_FALSE(bus->GetReadyFuture().Get().IsOK()); + auto error = bus->GetReadyFuture().Get(); + EXPECT_FALSE(error.IsOK()); + EXPECT_EQ(error.GetCode(), EErrorCode::SslError); server->Stop() .Get() @@ -469,7 +471,9 @@ TEST_F(TSslTest, DisabledAndRequiredEncryptionMode) auto client = CreateBusClient(clientConfig); auto bus = client->CreateBus(New<TEmptyBusHandler>()); - EXPECT_FALSE(bus->GetReadyFuture().Get().IsOK()); + auto error = bus->GetReadyFuture().Get(); + EXPECT_FALSE(error.IsOK()); + EXPECT_EQ(error.GetCode(), EErrorCode::SslError); server->Stop() .Get() @@ -545,7 +549,9 @@ TEST_F(TSslTest, CAVerificationModeFailure) auto client = CreateBusClient(clientConfig); auto bus = client->CreateBus(New<TEmptyBusHandler>()); - EXPECT_FALSE(bus->GetReadyFuture().Get().IsOK()); + auto error = bus->GetReadyFuture().Get(); + EXPECT_FALSE(error.IsOK()); + EXPECT_EQ(error.GetCode(), EErrorCode::SslError); server->Stop() .Get() @@ -771,7 +777,9 @@ TEST_F(TSslTest, DifferentCipherLists) auto client = CreateBusClient(clientConfig); auto bus = client->CreateBus(New<TEmptyBusHandler>()); - EXPECT_FALSE(bus->GetReadyFuture().Get().IsOK()); + auto error = bus->GetReadyFuture().Get(); + EXPECT_FALSE(error.IsOK()); + EXPECT_EQ(error.GetCode(), EErrorCode::SslError); server->Stop() .Get() diff --git a/yt/yt/core/concurrency/throughput_throttler.cpp b/yt/yt/core/concurrency/throughput_throttler.cpp index ae550923861..1ecb63fe0ed 100644 --- a/yt/yt/core/concurrency/throughput_throttler.cpp +++ b/yt/yt/core/concurrency/throughput_throttler.cpp @@ -758,7 +758,7 @@ public: return true; } - // TODO: implement TryAcquireAvailable the same way as TryAcquire. + // TODO(vvshlyaga): implement TryAcquireAvailable the same way as TryAcquire. i64 TryAcquireAvailable(i64 /*amount*/) override { YT_ABORT(); diff --git a/yt/yt/core/misc/async_expiring_cache.h b/yt/yt/core/misc/async_expiring_cache.h index 4fc7759cb2e..c0fd8c62135 100644 --- a/yt/yt/core/misc/async_expiring_cache.h +++ b/yt/yt/core/misc/async_expiring_cache.h @@ -18,6 +18,10 @@ namespace NYT { //////////////////////////////////////////////////////////////////////////////// +/*! + * \note + * Thread affinity: delayed executor's thread + */ template <class TKey, class TValue> class TAsyncExpiringCache : public virtual TRefCounted diff --git a/yt/yt/core/misc/memory_usage_tracker.cpp b/yt/yt/core/misc/memory_usage_tracker.cpp index e9d0e855aad..58b18307b90 100644 --- a/yt/yt/core/misc/memory_usage_tracker.cpp +++ b/yt/yt/core/misc/memory_usage_tracker.cpp @@ -163,7 +163,7 @@ TErrorOr<TMemoryUsageTrackerGuard> TMemoryUsageTrackerGuard::TryAcquire( guard.Size_ = size; guard.AcquiredSize_ = size; guard.Granularity_ = granularity; - return std::move(guard); + return guard; } void TMemoryUsageTrackerGuard::Release() @@ -258,7 +258,7 @@ TMemoryUsageTrackerGuard TMemoryUsageTrackerGuard::TransferMemory(i64 size) guard.Size_ = size; guard.AcquiredSize_ = acquiredDelta; guard.Granularity_ = Granularity_; - return std::move(guard); + return guard; } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/misc/protobuf_helpers.cpp b/yt/yt/core/misc/protobuf_helpers.cpp index 37428337c4d..2ed1448c919 100644 --- a/yt/yt/core/misc/protobuf_helpers.cpp +++ b/yt/yt/core/misc/protobuf_helpers.cpp @@ -332,7 +332,7 @@ public: return it == ExtensionTagToExtensionDescriptor_.end() ? nullptr : &it->second; } - const TProtobufExtensionDescriptor* FindDescriptorByName(const TString& name) override + const TProtobufExtensionDescriptor* FindDescriptorByName(const std::string& name) override { EnsureInitialized(); @@ -454,8 +454,7 @@ void Deserialize(TExtensionSet& extensionSet, NYTree::INodePtr node) { auto mapNode = node->AsMap(); for (const auto& [name, value] : mapNode->GetChildren()) { - // TODO(babenko): migrate to std::string - const auto* extensionDescriptor = IProtobufExtensionRegistry::Get()->FindDescriptorByName(TString(name)); + const auto* extensionDescriptor = IProtobufExtensionRegistry::Get()->FindDescriptorByName(name); // Do not parse unknown extensions. if (!extensionDescriptor) { continue; @@ -556,7 +555,7 @@ THashSet<int> GetExtensionTagSet(const NYT::NProto::TExtensionSet& source) return tags; } -std::optional<TString> FindExtensionName(int tag) +std::optional<std::string> FindExtensionName(int tag) { const auto* extensionDescriptor = IProtobufExtensionRegistry::Get()->FindDescriptorByTag(tag); if (!extensionDescriptor) { diff --git a/yt/yt/core/misc/protobuf_helpers.h b/yt/yt/core/misc/protobuf_helpers.h index eff6f223b2f..5c22765969d 100644 --- a/yt/yt/core/misc/protobuf_helpers.h +++ b/yt/yt/core/misc/protobuf_helpers.h @@ -357,7 +357,7 @@ struct TProtobufExtensionDescriptor { const google::protobuf::Descriptor* MessageDescriptor; const int Tag; - const TString Name; + const std::string Name; }; struct IProtobufExtensionRegistry @@ -378,7 +378,7 @@ struct IProtobufExtensionRegistry virtual const TProtobufExtensionDescriptor* FindDescriptorByTag(int tag) = 0; //! Finds a descriptor by name. - virtual const TProtobufExtensionDescriptor* FindDescriptorByName(const TString& name) = 0; + virtual const TProtobufExtensionDescriptor* FindDescriptorByName(const std::string& name) = 0; //! Returns the singleton instance. static IProtobufExtensionRegistry* Get(); @@ -437,7 +437,7 @@ NYT::NProto::TExtensionSet FilterProtoExtensions( //////////////////////////////////////////////////////////////////////////////// THashSet<int> GetExtensionTagSet(const NYT::NProto::TExtensionSet& source); -std::optional<TString> FindExtensionName(int tag); +std::optional<std::string> FindExtensionName(int tag); //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/misc/unittests/consistent_hashing_ut.cpp b/yt/yt/core/misc/unittests/consistent_hashing_ut.cpp index dfe692f419a..8633819cac6 100644 --- a/yt/yt/core/misc/unittests/consistent_hashing_ut.cpp +++ b/yt/yt/core/misc/unittests/consistent_hashing_ut.cpp @@ -254,7 +254,7 @@ public: std::swap(Items_.back(), Items_[deleteIndex]); Items_.pop_back(); - return std::move(deleteItem); + return deleteItem; } std::vector<TCrpItemWithToken>::iterator begin() diff --git a/yt/yt/core/rpc/bus/channel.cpp b/yt/yt/core/rpc/bus/channel.cpp index 5625e70630c..9ee42a06b1b 100644 --- a/yt/yt/core/rpc/bus/channel.cpp +++ b/yt/yt/core/rpc/bus/channel.cpp @@ -733,18 +733,21 @@ private: .Subscribe(BIND( &TSession::OnRequestSerialized, MakeStrong(this), + std::move(request), std::move(requestControl), options)); } else { try { auto requestMessage = request->Serialize(); OnRequestSerialized( - std::move(requestControl), + request, + requestControl, options, std::move(requestMessage)); } catch (const std::exception& ex) { OnRequestSerialized( - std::move(requestControl), + request, + requestControl, options, TError(ex)); } @@ -776,6 +779,7 @@ private: void OnRequestSerialized( + const IClientRequestPtr& request, const TClientRequestControlPtr& requestControl, const TSendOptions& options, TErrorOr<TSharedRefArray> requestMessageOrError) @@ -874,7 +878,7 @@ private: requestControl->ProfileRequest(requestMessage); YT_LOG_DEBUG("Request sent (RequestId: %v, Method: %v.%v, Timeout: %v, TrackingLevel: %v, " - "ChecksummedPartCount: %v, MultiplexingBand: %v, Endpoint: %v, BodySize: %v, AttachmentsSize: %v)", + "ChecksummedPartCount: %v, MultiplexingBand: %v, Endpoint: %v, BodySize: %v, AttachmentsSize: %v%v)", requestId, requestControl->GetService(), requestControl->GetMethod(), @@ -884,7 +888,8 @@ private: options.MultiplexingBand, Bus_->GetEndpointDescription(), GetMessageBodySize(requestMessage), - GetTotalMessageAttachmentSize(requestMessage)); + GetTotalMessageAttachmentSize(requestMessage), + request->GetRequestInfo() ? std::string(Format(", %v", *request->GetRequestInfo())) : std::string()); } diff --git a/yt/yt/core/rpc/client-inl.h b/yt/yt/core/rpc/client-inl.h index fb844d4a231..112642b00c4 100644 --- a/yt/yt/core/rpc/client-inl.h +++ b/yt/yt/core/rpc/client-inl.h @@ -37,6 +37,14 @@ void IClientRequest::RequireServerFeature(E featureId) //////////////////////////////////////////////////////////////////////////////// +template <class... TArgs> +void TClientRequest::SetRequestInfo(TFormatString<TArgs...> format, TArgs&&... args) +{ + RequestInfo_ = Format(format, std::forward<TArgs>(args)...); +} + +//////////////////////////////////////////////////////////////////////////////// + template <class TRequestMessage, class TResponse> TTypedClientRequest<TRequestMessage, TResponse>::TTypedClientRequest( IChannelPtr channel, diff --git a/yt/yt/core/rpc/client.cpp b/yt/yt/core/rpc/client.cpp index ff3301b5ea1..1953601581a 100644 --- a/yt/yt/core/rpc/client.cpp +++ b/yt/yt/core/rpc/client.cpp @@ -105,8 +105,7 @@ TSharedRefArray TClientRequest::Serialize() auto headerlessMessage = GetHeaderlessMessage(); if (!retry) { - auto output = CreateRequestMessage(Header_, headerlessMessage); - return std::move(output); + return CreateRequestMessage(Header_, headerlessMessage); } if (StreamingEnabled_) { @@ -116,8 +115,7 @@ TSharedRefArray TClientRequest::Serialize() auto patchedHeader = Header_; patchedHeader.set_retry(true); - auto output = CreateRequestMessage(patchedHeader, headerlessMessage); - return std::move(output); + return CreateRequestMessage(patchedHeader, headerlessMessage); } IClientRequestControlPtr TClientRequest::Send(IClientResponseHandlerPtr responseHandler) @@ -221,6 +219,11 @@ std::string TClientRequest::GetMethod() const return FromProto<std::string>(Header_.method()); } +const std::optional<std::string>& TClientRequest::GetRequestInfo() const +{ + return RequestInfo_; +} + void TClientRequest::DeclareClientFeature(int featureId) { Header_.add_declared_client_feature_ids(featureId); diff --git a/yt/yt/core/rpc/client.h b/yt/yt/core/rpc/client.h index 0fb76bd9bee..63a0b40fade 100644 --- a/yt/yt/core/rpc/client.h +++ b/yt/yt/core/rpc/client.h @@ -63,6 +63,8 @@ struct IClientRequest virtual std::string GetService() const = 0; virtual std::string GetMethod() const = 0; + virtual const std::optional<std::string>& GetRequestInfo() const = 0; + virtual void DeclareClientFeature(int featureId) = 0; virtual void RequireServerFeature(int featureId) = 0; @@ -174,6 +176,11 @@ public: std::string GetService() const override; std::string GetMethod() const override; + template <class... TArgs> + void SetRequestInfo(TFormatString<TArgs...> format, TArgs&&... args); + + const std::optional<std::string>& GetRequestInfo() const override; + using NRpc::IClientRequest::DeclareClientFeature; using NRpc::IClientRequest::RequireServerFeature; @@ -248,6 +255,7 @@ private: std::string User_; std::string UserTag_; + std::optional<std::string> RequestInfo_; TWeakPtr<IClientRequestControl> RequestControl_; diff --git a/yt/yt/core/ytree/node_detail.cpp b/yt/yt/core/ytree/node_detail.cpp index 9f273ad0a34..8f9c077b16c 100644 --- a/yt/yt/core/ytree/node_detail.cpp +++ b/yt/yt/core/ytree/node_detail.cpp @@ -206,8 +206,7 @@ void TCompositeNodeMixin::SetRecursive( ValidatePermission(EPermissionCheckScope::This, EPermission::Write); auto factory = CreateFactory(); - auto child = ConvertToNode(TYsonString(request->value()), factory.get()); - SetChild(factory.get(), "/" + path, child, request->recursive()); + SetChildValue(factory.get(), "/" + path, TYsonString(request->value()), request->recursive()); factory->Commit(); context->Reply(); @@ -366,13 +365,17 @@ void TMapNodeMixin::ListSelf( })); } -std::pair<TString, INodePtr> TMapNodeMixin::PrepareSetChild( +std::pair<TString, INodePtr> TMapNodeMixin::PrepareSetChildOrChildValue( INodeFactory* factory, const TYPath& path, - INodePtr child, + std::variant<INodePtr, NYson::TYsonString> childOrChildValue, bool recursive) { - YT_VERIFY(factory || !recursive); + if (std::holds_alternative<INodePtr>(childOrChildValue)) { + YT_VERIFY(factory || !recursive); + } else { + YT_VERIFY(factory); + } NYPath::TTokenizer tokenizer(path); if (tokenizer.Advance() == NYPath::ETokenType::EndOfStream) { @@ -407,7 +410,15 @@ std::pair<TString, INodePtr> TMapNodeMixin::PrepareSetChild( ValidateChildCount(GetPath(), currentNode->GetChildCount()); - auto newChild = lastStep ? child : factory->CreateMap(); + auto newChild = lastStep + ? Visit(childOrChildValue, + [] (INodePtr child) { + return child; + }, + [&] (const TYsonString& childValue) { + return ConvertToNode(childValue, factory); + }) + : factory->CreateMap(); if (currentNode != rootNode) { YT_VERIFY(currentNode->AddChild(key, newChild)); } else { @@ -432,6 +443,15 @@ std::pair<TString, INodePtr> TMapNodeMixin::PrepareSetChild( return {rootKey, rootChild}; } +std::pair<TString, INodePtr> TMapNodeMixin::PrepareSetChild( + INodeFactory* factory, + const TYPath& path, + INodePtr child, + bool recursive) +{ + return PrepareSetChildOrChildValue(factory, path, child, recursive); +} + void TMapNodeMixin::SetChild( INodeFactory* factory, const TYPath& path, @@ -442,6 +462,16 @@ void TMapNodeMixin::SetChild( AddChild(rootKey, rootChild); } +void TMapNodeMixin::SetChildValue( + INodeFactory* factory, + const TYPath& path, + NYson::TYsonString childValue, + bool recursive) +{ + const auto& [rootKey, rootChild] = PrepareSetChildOrChildValue(factory, path, childValue, recursive); + AddChild(rootKey, rootChild); +} + int TMapNodeMixin::GetMaxKeyLength() const { return std::numeric_limits<int>::max(); @@ -518,12 +548,14 @@ IYPathService::TResolveResult TListNodeMixin::ResolveRecursive( } } -void TListNodeMixin::SetChild( - INodeFactory* /*factory*/, +void TListNodeMixin::SetChildOrChildValue( + INodeFactory* factory, const TYPath& path, - INodePtr child, + std::variant<INodePtr, TYsonString> childOrValue, bool recursive) { + YT_VERIFY(factory || std::holds_alternative<INodePtr>(childOrValue)); + if (recursive) { THROW_ERROR_EXCEPTION("List node %v does not support \"recursive\" option", GetPath()); @@ -562,7 +594,29 @@ void TListNodeMixin::SetChild( ValidateChildCount(GetPath(), GetChildCount()); - AddChild(child, beforeIndex); + AddChild( + std::holds_alternative<INodePtr>(childOrValue) + ? std::get<INodePtr>(childOrValue) + : ConvertToNode(std::get<TYsonString>(childOrValue), factory), + beforeIndex); +} + +void TListNodeMixin::SetChild( + INodeFactory* factory, + const TYPath& path, + INodePtr child, + bool recursive) +{ + SetChildOrChildValue(factory, path, child, recursive); +} + +void TListNodeMixin::SetChildValue( + INodeFactory* factory, + const TYPath& path, + TYsonString childValue, + bool recursive) +{ + SetChildOrChildValue(factory, path, childValue, recursive); } //////////////////////////////////////////////////////////////////////////////// diff --git a/yt/yt/core/ytree/node_detail.h b/yt/yt/core/ytree/node_detail.h index 25a32747c74..cd347db01b5 100644 --- a/yt/yt/core/ytree/node_detail.h +++ b/yt/yt/core/ytree/node_detail.h @@ -93,6 +93,12 @@ protected: virtual int GetMaxChildCount() const; void ValidateChildCount(const TYPath& path, int childCount) const; + + virtual void SetChildValue( + INodeFactory* factory, + const TYPath& path, + NYson::TYsonString childValue, + bool recursive) = 0; }; //////////////////////////////////////////////////////////////////////////////// @@ -126,8 +132,20 @@ protected: virtual int GetMaxKeyLength() const; + void SetChildValue( + INodeFactory* factory, + const TYPath& path, + NYson::TYsonString childValue, + bool recursive) final; + private: void ThrowMaxKeyLengthViolated() const; + + std::pair<TString, INodePtr> PrepareSetChildOrChildValue( + INodeFactory* factory, + const TYPath& path, + std::variant<INodePtr, NYson::TYsonString> childOrChildValue, + bool recursive); }; //////////////////////////////////////////////////////////////////////////////// @@ -146,6 +164,19 @@ protected: const TYPath& path, INodePtr child, bool recursive) override; + + void SetChildValue( + INodeFactory* factory, + const TYPath& path, + NYson::TYsonString childValue, + bool recursive) final; + +private: + void SetChildOrChildValue( + INodeFactory* factory, + const TYPath& path, + std::variant<INodePtr, NYson::TYsonString> childOrChildValue, + bool recursive); }; //////////////////////////////////////////////////////////////////////////////// @@ -242,4 +273,3 @@ private: //////////////////////////////////////////////////////////////////////////////// } // namespace NYT::NYTree - diff --git a/yt/yt/core/ytree/ypath_client.cpp b/yt/yt/core/ytree/ypath_client.cpp index bd18ac308bd..c9cc57a7531 100644 --- a/yt/yt/core/ytree/ypath_client.cpp +++ b/yt/yt/core/ytree/ypath_client.cpp @@ -77,6 +77,12 @@ std::string TYPathRequest::GetService() const return FromProto<std::string>(Header_.service()); } +const std::optional<std::string>& TYPathRequest::GetRequestInfo() const +{ + static const std::optional<std::string> Empty = std::nullopt; + return Empty; +} + void TYPathRequest::DeclareClientFeature(int featureId) { Header_.add_declared_client_feature_ids(featureId); diff --git a/yt/yt/core/ytree/ypath_client.h b/yt/yt/core/ytree/ypath_client.h index c97e8e36d15..02ab95b2e40 100644 --- a/yt/yt/core/ytree/ypath_client.h +++ b/yt/yt/core/ytree/ypath_client.h @@ -34,6 +34,7 @@ public: NRpc::TRealmId GetRealmId() const override; std::string GetMethod() const override; std::string GetService() const override; + const std::optional<std::string>& GetRequestInfo() const override; using NRpc::IClientRequest::DeclareClientFeature; using NRpc::IClientRequest::RequireServerFeature; diff --git a/yt/yt/library/formats/protobuf.cpp b/yt/yt/library/formats/protobuf.cpp index 46f87d4b15d..576210ba960 100644 --- a/yt/yt/library/formats/protobuf.cpp +++ b/yt/yt/library/formats/protobuf.cpp @@ -36,16 +36,16 @@ using namespace NYT::NYTree; //////////////////////////////////////////////////////////////////////////////// -TEnumerationDescription::TEnumerationDescription(const TString& name) +TEnumerationDescription::TEnumerationDescription(const std::string& name) : Name_(name) { } -const TString& TEnumerationDescription::GetEnumerationName() const +const std::string& TEnumerationDescription::GetEnumerationName() const { return Name_; } -const TString& TEnumerationDescription::GetValueName(i32 value) const +const std::string& TEnumerationDescription::GetValueName(i32 value) const { if (auto valueName = TryGetValueName(value)) { return *valueName; @@ -55,7 +55,7 @@ const TString& TEnumerationDescription::GetValueName(i32 value) const << TErrorAttribute("value", value); } -const TString* TEnumerationDescription::TryGetValueName(i32 value) const +const std::string* TEnumerationDescription::TryGetValueName(i32 value) const { auto it = ValueToName_.find(value); if (it == ValueToName_.end()) { @@ -83,7 +83,7 @@ std::optional<i32> TEnumerationDescription::TryGetValue(TStringBuf valueName) co return it->second; } -void TEnumerationDescription::Add(TString name, i32 value) +void TEnumerationDescription::Add(std::string name, i32 value) { if (NameToValue_.find(name) != NameToValue_.end()) { THROW_ERROR_EXCEPTION("Enumeration %v already has value %v", @@ -174,7 +174,7 @@ private: //////////////////////////////////////////////////////////////////////////////// static TEnumerationDescription CreateEnumerationMap( - const TString& enumName, + const std::string& enumName, const IMapNodePtr& enumerationConfig) { TEnumerationDescription result(enumName); @@ -183,12 +183,10 @@ static TEnumerationDescription CreateEnumerationMap( const auto& valueNode = enumValue.second; switch (valueNode->GetType()) { case ENodeType::Uint64: - // TODO(babenko): migrate to std::string - result.Add(TString(name), valueNode->GetValue<ui64>()); + result.Add(name, valueNode->GetValue<ui64>()); break; case ENodeType::Int64: - // TODO(babenko): migrate to std::string - result.Add(TString(name), valueNode->GetValue<i64>()); + result.Add(name, valueNode->GetValue<i64>()); break; default: THROW_ERROR_EXCEPTION("Invalid specification of %Qv enumeration: expected type \"int64\" or \"uint64\", actual type %Qlv", @@ -386,7 +384,7 @@ void ValidateSimpleType( auto validateLogicalType = [&] (auto... expectedTypes) { if ((... && (logicalType != expectedTypes))) { - auto typeNameList = std::vector<TString>{FormatEnum(expectedTypes)...}; + auto typeNameList = std::vector<std::string>{FormatEnum(expectedTypes)...}; throwMismatchError(Format("expected logical type to be one of %v", typeNameList)); } }; @@ -1007,14 +1005,14 @@ void TProtobufFormatDescriptionBase<TType>::InitFromProtobufSchema( const auto& enumerationConfigMap = config->Enumerations; for (const auto& [name_, field] : enumerationConfigMap->GetChildren()) { // TODO(babenko): migrate to std::string - auto name = TString(name_); + auto name = std::string(name_); if (field->GetType() != ENodeType::Map) { THROW_ERROR_EXCEPTION(R"(Invalid enumeration specification type: expected "map", found %Qlv)", field->GetType()); } const auto& enumerationConfig = field->AsMap(); // TODO(babenko): migrate to std::string - EnumerationDescriptionMap_.emplace(name, CreateEnumerationMap(TString(TimestampColumnName), enumerationConfig)); + EnumerationDescriptionMap_.emplace(name, CreateEnumerationMap(std::string(TimestampColumnName), enumerationConfig)); } } @@ -1046,7 +1044,7 @@ void TProtobufFormatDescriptionBase<TType>::InitFromProtobufSchema( //////////////////////////////////////////////////////////////////////////////// template <typename TType> -TProtobufTypeBuilder<TType>::TProtobufTypeBuilder(const THashMap<TString, TEnumerationDescription>& enumerations) +TProtobufTypeBuilder<TType>::TProtobufTypeBuilder(const THashMap<std::string, TEnumerationDescription>& enumerations) : Enumerations_(enumerations) { } diff --git a/yt/yt/library/formats/protobuf.h b/yt/yt/library/formats/protobuf.h index 7f0b3e38f63..14f6aff11c7 100644 --- a/yt/yt/library/formats/protobuf.h +++ b/yt/yt/library/formats/protobuf.h @@ -14,22 +14,22 @@ namespace NYT::NFormats { class TEnumerationDescription { public: - explicit TEnumerationDescription(const TString& name); + explicit TEnumerationDescription(const std::string& name); - const TString& GetEnumerationName() const; + const std::string& GetEnumerationName() const; - const TString& GetValueName(i32 value) const; - const TString* TryGetValueName(i32 value) const; + const std::string& GetValueName(i32 value) const; + const std::string* TryGetValueName(i32 value) const; i32 GetValue(TStringBuf name) const; std::optional<i32> TryGetValue(TStringBuf name) const; - void Add(TString name, i32 value); + void Add(std::string name, i32 value); private: - THashMap<TString, i32> NameToValue_; - THashMap<i32, TString> ValueToName_; - TString Name_; + THashMap<std::string, i32, THash<TStringBuf>, TEqualTo<>> NameToValue_; + THashMap<i32, std::string> ValueToName_; + std::string Name_; }; //////////////////////////////////////////////////////////////////////////////// @@ -47,7 +47,7 @@ struct TProtobufTag struct TProtobufFieldDescriptionBase : public TProtobufTag { - TString Name; + std::string Name; // Index of field inside struct (for fields corresponding to struct fields in schema). int StructFieldIndex = 0; @@ -224,7 +224,7 @@ public: using TField = std::conditional_t<IsWriter, TProtobufWriterFieldDescription, TProtobufParserFieldDescription>; using TFieldPtr = std::unique_ptr<TField>; - TProtobufTypeBuilder(const THashMap<TString, TEnumerationDescription>& enumerations); + TProtobufTypeBuilder(const THashMap<std::string, TEnumerationDescription>& enumerations); TFieldPtr CreateField( int structFieldIndex, @@ -234,7 +234,7 @@ public: bool allowEmbedded = false); private: - const THashMap<TString, TEnumerationDescription>& Enumerations_; + const THashMap<std::string, TEnumerationDescription>& Enumerations_; private: // Traverse type config, matching it with type descriptor from schema. @@ -281,7 +281,7 @@ protected: const std::vector<NTableClient::TTableSchemaPtr>& schemas); private: - THashMap<TString, TEnumerationDescription> EnumerationDescriptionMap_; + THashMap<std::string, TEnumerationDescription> EnumerationDescriptionMap_; private: virtual void AddTable(NYT::TIntrusivePtr<TType> tableType) = 0; @@ -341,7 +341,7 @@ private: struct TTableDescription { TProtobufWriterTypePtr Type; - THashMap<TString, const TProtobufWriterFieldDescription*> Columns; + THashMap<std::string, const TProtobufWriterFieldDescription*, THash<TStringBuf>, TEqualTo<>> Columns; std::vector<TProtobufWriterEmbeddingDescription> Embeddings; // Cached data. diff --git a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto index d6aa77155eb..fe5cd1da967 100644 --- a/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto +++ b/yt/yt_proto/yt/client/api/rpc_proxy/proto/api_service.proto @@ -69,10 +69,13 @@ enum EReplicaConsistency enum EMasterReadKind { + // These options cover the majority of cases. MRK_LEADER = 0; MRK_FOLLOWER = 1; MRK_CACHE = 2; - MRK_MASTER_CACHE = 3; + + // These are advanced options. Typically you don't need these. + MRK_MASTER_SIDE_CACHE = 3; } enum ERowsetKind @@ -809,6 +812,8 @@ message TReqCreateQueueProducerSession optional bytes queue_path = 2; // RichYPath optional string session_id = 3; optional bytes user_meta = 4; // YSON + + optional TMutatingOptions mutating_options = 103; } message TRspCreateQueueProducerSession |