diff options
author | Alexander Smirnov <alex@ydb.tech> | 2024-11-22 09:21:01 +0000 |
---|---|---|
committer | Alexander Smirnov <alex@ydb.tech> | 2024-11-22 09:21:01 +0000 |
commit | a7cac42c868ca5722777ccee944217410812e72c (patch) | |
tree | 7bff7ce2a3ade72f6f15dfc634490d13628066ee /build | |
parent | a18f18d81996ca8e681bb6cabd441b52833d99bf (diff) | |
parent | 9478cfdab4217d3710b96329466825bf47111d7d (diff) | |
download | ydb-a7cac42c868ca5722777ccee944217410812e72c.tar.gz |
Merge branch 'rightlib' into mergelibs-241122-0919
Diffstat (limited to 'build')
-rw-r--r-- | build/config/tests/flake8/flake8.conf | 2 | ||||
-rw-r--r-- | build/export_generators/ide-gradle/build.gradle.kts.jinja | 41 | ||||
-rw-r--r-- | build/export_generators/ide-gradle/build.gradle.kts.proto.jinja | 80 | ||||
-rw-r--r-- | build/external_resources/flake8_py3/ya.make | 10 | ||||
-rw-r--r-- | build/mapping.conf.json | 26 | ||||
-rw-r--r-- | build/platform/yfm/ya.make | 8 | ||||
-rw-r--r-- | build/plugins/lib/nots/package_manager/base/package_json.py | 6 | ||||
-rw-r--r-- | build/plugins/lib/nots/package_manager/base/package_manager.py | 11 | ||||
-rw-r--r-- | build/plugins/lib/nots/package_manager/npm/npm_package_manager.py | 3 | ||||
-rw-r--r-- | build/plugins/lib/nots/package_manager/pnpm/package_manager.py | 60 | ||||
-rw-r--r-- | build/plugins/lib/nots/package_manager/ya.make | 1 | ||||
-rw-r--r-- | build/ya.conf.json | 54 | ||||
-rwxr-xr-x | build/ymake_conf.py | 1 |
13 files changed, 226 insertions, 77 deletions
diff --git a/build/config/tests/flake8/flake8.conf b/build/config/tests/flake8/flake8.conf index fda445a3f0..a4a60be743 100644 --- a/build/config/tests/flake8/flake8.conf +++ b/build/config/tests/flake8/flake8.conf @@ -285,6 +285,8 @@ ignore = PLR915, # too-many-boolean-expressions PLR916, + # too-many-positional-arguments + PLR917, # consider-merging-isinstance PLR701, # too-many-nested-blocks diff --git a/build/export_generators/ide-gradle/build.gradle.kts.jinja b/build/export_generators/ide-gradle/build.gradle.kts.jinja index 46bcd4cefd..798922b5b7 100644 --- a/build/export_generators/ide-gradle/build.gradle.kts.jinja +++ b/build/export_generators/ide-gradle/build.gradle.kts.jinja @@ -38,21 +38,6 @@ subprojects { {%- endif %} plugins { -{#- some plugins configuration -#} -{%- for library in target.consumer if library.classpath -%} -{#- error prone plugin configuration -#} -{%- if library.prebuilt and library.jar and (library.type != "contrib" or build_contribs) and "contrib/java/com/google/errorprone/error_prone_annotations/" in library.jar -%} - - id("net.ltgt.errorprone") version "{{ errorprone_plugin_version }}" - -{%- endif -%} -{%- endfor -%} - -{#- lombok configuration -#} -{#- TODO remove usings annotation_processors semantic -#} -{%- if ("lombok.launch.AnnotationProcessorHider$AnnotationProcessor" in target.annotation_processors) or (target.use_annotation_processor|length and target.use_annotation_processor|select('startsWith', 'contrib/java/org/projectlombok/lombok')|length) %} - id("io.freefair.lombok") version "8.6" -{%- endif -%} {%- if mainClass %} `application` {%- else %} @@ -76,6 +61,17 @@ plugins { {%- if target.with_kotlinc_plugin_serialization|length %} kotlin("plugin.serialization") version "{{ kotlin_version }}" {% endif -%} +{%- endif -%} + +{#- errorprone plugin configuration -#} +{%- if target.consumer|selectattr('jar', 'startsWith', 'contrib/java/com/google/errorprone/error_prone_annotations')|length %} + id("net.ltgt.errorprone") version "{{ errorprone_plugin_version }}" +{%- endif -%} + +{#- lombok plugin configuration -#} +{#- TODO remove usings annotation_processors semantic -#} +{%- if ("lombok.launch.AnnotationProcessorHider$AnnotationProcessor" in target.annotation_processors) or (target.use_annotation_processor|length and target.use_annotation_processor|select('startsWith', 'contrib/java/org/projectlombok/lombok')|length) %} + id("io.freefair.lombok") version "8.6" {%- endif %} } @@ -272,6 +268,7 @@ sourceSets.main.java.srcDirs += "{{ srcdir_glob[0] }}" {%- endfor -%} sourceSets { +{%- if target.runs|length %} main { {#- Default by Gradle: @@ -280,13 +277,11 @@ sourceSets { resources.srcDir("src/main/resources") #} -{%- if target.runs|length -%} {%- for run in target.runs -%} {{ OutDirs(run, ' java.srcDir("', '")') }} -{%- endfor -%} -{%- endif %} +{%- endfor %} } - +{%- endif %} test { {#- Default by Gradle: @@ -315,11 +310,9 @@ sourceSets { dependencies { {%- for library in target.consumer if library.classpath -%} -{# error prone plugin configuration #} -{%- if library.prebuilt and library.jar and (library.type != "contrib" or build_contribs) and "contrib/java/com/google/errorprone/error_prone_annotations" in library.jar -%} -{% set errorprone_version = library.jar -%} -{% set errorprone_parts = errorprone_version|replace("contrib/java/com/google/errorprone/error_prone_annotations/") -%} -{% set errorprone_parts = split(errorprone_parts, '/') %} +{%- if library.prebuilt and (library.type != "contrib" or build_contribs) and ("contrib/java/com/google/errorprone/error_prone_annotations" in library.jar) -%} +{%- set errorprone_version = library.jar|replace("contrib/java/com/google/errorprone/error_prone_annotations/", "") -%} +{%- set errorprone_parts = split(errorprone_version, '/', 2) %} errorprone("com.google.errorprone:error_prone_core:{{ errorprone_parts[0] }}") {%- endif -%} diff --git a/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja b/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja index 9b7b71f743..fe989ed989 100644 --- a/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja +++ b/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja @@ -1,4 +1,6 @@ {%- set publish = target.publish -%} +{%- set libraries = target.consumer|selectattr('type', 'eq', 'library') -%} + import com.google.protobuf.gradle.* val baseBuildDir = "{{ export_root }}/gradle.build/" @@ -7,7 +9,10 @@ subprojects { buildDir = file(baseBuildDir + project.path.replaceFirst(":", "/").replace(":", ".")) } -val buildProtoDir = File("${buildDir}", "__proto__") +val mainProtosDir = File(buildDir, "main_protos") +{%- if libraries|length %} +val mainExtractedIncludeProtosDir = File(buildDir, "extracted-include-protos/main") +{%- endif %} plugins { id("java-library") @@ -37,14 +42,23 @@ repositories { } } +val project_root = "{{ arcadia_root }}" + sourceSets { main { - java.srcDir("build/generated/source/proto") + java.srcDir("$buildDir/generated/sources/proto/main/java") +{%- if target.proto_grpc %} + java.srcDir("$buildDir/generated/sources/proto/main/grpc") +{%- endif %} + } + test { + java.srcDir("$buildDir/generated/sources/proto/test/java") +{%- if target.proto_grpc %} + java.srcDir("$buildDir/generated/sources/proto/test/grpc") +{%- endif %} } } -val project_root = "{{ arcadia_root }}" - java { withSourcesJar() withJavadocJar() @@ -92,21 +106,24 @@ dependencies { } {%- endif -%} {%- endif -%} -{%- endfor %} +{%- endfor -%} {%- if target.proto_namespace %} - protobuf(files(File(buildProtoDir, "{{ target.proto_namespace }}"))) + protobuf(files(File(mainProtosDir, "{{ target.proto_namespace }}"))) {%- else %} - protobuf(files(buildProtoDir)) + protobuf(files(mainProtosDir)) {%- endif %} } protobuf { + generatedFilesBaseDir = "$buildDir/generated/sources/proto" + protoc { // Download from repositories artifact = "com.google.protobuf:protoc:{%- if target.proto_compiler_version -%}{{ target.proto_compiler_version }}{%- else -%}3.22.5{%- endif -%}" } -{% if target.proto_grpc -%} +{%- if target.proto_grpc %} + plugins { id("grpc") { artifact = "io.grpc:protoc-gen-grpc-java:{%- if target.proto_grpc_version -%}{{ target.proto_grpc_version }}{%- else -%}1.45.0{%- endif -%}" @@ -115,48 +132,57 @@ protobuf { id("grpckt") { artifact = "io.grpc:protoc-gen-grpc-kotlin:{%- if target.proto_kotlin_grpc_version -%}{{ target.proto_kotlin_grpc_version }}{%- else -%}1.3.1{%- endif -%}:jdk8@jar" } -{% endif -%} +{%- endif %} } + generateProtoTasks { all().forEach { it.plugins { id("grpc") {%- if target.proto_kotlin_grpc %} id("grpckt") -{% endif -%} +{%- endif %} } {%- if target.proto_kotlin_grpc %} it.builtins { create("kotlin") } -{% endif -%} +{%- endif %} } } {%- endif %} } -val prepareProto = tasks.register<Copy>("prepareProto") { - from(rootDir) { -{%- for proto in target.proto_files %} - include("{{ proto }}") -{%- endfor %} -{# Mining proto-deps sources directories #} -{%- for library in target.consumer if library.prebuilt -%} -{%- if library.type == "library" %} -{%- set proto_rep = library.classpath|replace('project(":','') %} -{%- set proto_rep = proto_rep|replace('")','') %} -{%- set proto_rep = proto_rep|replace(':','/') %} -{%- set proto = proto_rep + '/**' %} +val prepareMainProtos = tasks.register<Copy>("prepareMainProtos") { + from("$project_root") { +{#- list of all current project proto files -#} +{%- for proto in target.proto_files %} include("{{ proto }}") -{%- endif -%} -{%- endfor %} +{%- endfor %} } - into(buildProtoDir) + into(mainProtosDir) } +{% if libraries|length -%} +val extractMainLibrariesProtos = tasks.register<Copy>("extractMainLibrariesProtos") { + from("$project_root") { +{#- list of all library directories -#} +{%- for library in libraries -%} +{%- set path_and_jar = rsplit(library.jar, '/', 2) %} + include("{{ path_and_jar[0] }}/**/*.proto") +{%- endfor %} + } + into(mainExtractedIncludeProtosDir) +} + +{% endif -%} afterEvaluate { - tasks.getByName("extractProto").dependsOn(prepareProto) + tasks.getByName("extractProto").dependsOn(prepareMainProtos) +{%- if libraries|length %} + tasks.getByName("extractProto").dependsOn(extractMainLibrariesProtos) +{%- endif %} } + {# To avoid problems when build project with proto #} tasks.getByName("sourcesJar").dependsOn("generateProto") diff --git a/build/external_resources/flake8_py3/ya.make b/build/external_resources/flake8_py3/ya.make index 3b66251bae..3d89508e8e 100644 --- a/build/external_resources/flake8_py3/ya.make +++ b/build/external_resources/flake8_py3/ya.make @@ -12,12 +12,12 @@ ENDIF() DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE( FLAKE8_PY3 - sbr:6822862832 FOR DARWIN-ARM64 - sbr:6822863322 FOR DARWIN + sbr:7501315930 FOR DARWIN-ARM64 + sbr:7501316202 FOR DARWIN sbr:6726869647 FOR LINUX-PPC64LE - sbr:6822864509 FOR LINUX - sbr:6822862341 FOR LINUX-AARCH64 - sbr:6822864007 FOR WIN32 + sbr:7501316756 FOR LINUX + sbr:7501315620 FOR LINUX-AARCH64 + sbr:7501316414 FOR WIN32 ) END() diff --git a/build/mapping.conf.json b/build/mapping.conf.json index 7c3df72802..473f306a32 100644 --- a/build/mapping.conf.json +++ b/build/mapping.conf.json @@ -293,24 +293,28 @@ "6726871384": "https://devtools-registry.s3.yandex.net/6726871384", "6760720465": "https://devtools-registry.s3.yandex.net/6760720465", "6822863322": "https://devtools-registry.s3.yandex.net/6822863322", + "7501316202": "https://devtools-registry.s3.yandex.net/7501316202", "6603781997": "https://devtools-registry.s3.yandex.net/6603781997", "6652223452": "https://devtools-registry.s3.yandex.net/6652223452", "6674733966": "https://devtools-registry.s3.yandex.net/6674733966", "6726867988": "https://devtools-registry.s3.yandex.net/6726867988", "6760720256": "https://devtools-registry.s3.yandex.net/6760720256", "6822862832": "https://devtools-registry.s3.yandex.net/6822862832", + "7501315930": "https://devtools-registry.s3.yandex.net/7501315930", "6603787904": "https://devtools-registry.s3.yandex.net/6603787904", "6652225714": "https://devtools-registry.s3.yandex.net/6652225714", "6674734993": "https://devtools-registry.s3.yandex.net/6674734993", "6726875537": "https://devtools-registry.s3.yandex.net/6726875537", "6760721014": "https://devtools-registry.s3.yandex.net/6760721014", "6822864509": "https://devtools-registry.s3.yandex.net/6822864509", + "7501316756": "https://devtools-registry.s3.yandex.net/7501316756", "6603780916": "https://devtools-registry.s3.yandex.net/6603780916", "6652222817": "https://devtools-registry.s3.yandex.net/6652222817", "6674733653": "https://devtools-registry.s3.yandex.net/6674733653", "6726866588": "https://devtools-registry.s3.yandex.net/6726866588", "6760720046": "https://devtools-registry.s3.yandex.net/6760720046", "6822862341": "https://devtools-registry.s3.yandex.net/6822862341", + "7501315620": "https://devtools-registry.s3.yandex.net/7501315620", "6603783079": "https://devtools-registry.s3.yandex.net/6603783079", "6652224021": "https://devtools-registry.s3.yandex.net/6652224021", "6674734285": "https://devtools-registry.s3.yandex.net/6674734285", @@ -321,6 +325,7 @@ "6726873745": "https://devtools-registry.s3.yandex.net/6726873745", "6760720811": "https://devtools-registry.s3.yandex.net/6760720811", "6822864007": "https://devtools-registry.s3.yandex.net/6822864007", + "7501316414": "https://devtools-registry.s3.yandex.net/7501316414", "3961412335": "https://devtools-registry.s3.yandex.net/3961412335", "3961411314": "https://devtools-registry.s3.yandex.net/3961411314", "3961413236": "https://devtools-registry.s3.yandex.net/3961413236", @@ -906,7 +911,10 @@ "6990860705": "https://devtools-registry.s3.yandex.net/6990860705", "6990881789": "https://devtools-registry.s3.yandex.net/6990881789", "7324461836": "https://devtools-registry.s3.yandex.net/7324461836", + "7512107516": "https://devtools-registry.s3.yandex.net/7512107516", "7193803465": "https://devtools-registry.s3.yandex.net/7193803465", + "7512042091": "https://devtools-registry.s3.yandex.net/7512042091", + "7512103615": "https://devtools-registry.s3.yandex.net/7512103615", "7324464594": "https://devtools-registry.s3.yandex.net/7324464594", "7487656957": "https://devtools-registry.s3.yandex.net/7487656957", "7458707245": "https://devtools-registry.s3.yandex.net/7458707245", @@ -1037,6 +1045,11 @@ "7451466407": "https://devtools-registry.s3.yandex.net/7451466407", "7407150115": "https://devtools-registry.s3.yandex.net/7407150115", "7451406900": "https://devtools-registry.s3.yandex.net/7451406900", + "7494722271": "https://devtools-registry.s3.yandex.net/7494722271", + "7494597813": "https://devtools-registry.s3.yandex.net/7494597813", + "7495431598": "https://devtools-registry.s3.yandex.net/7495431598", + "7495332394": "https://devtools-registry.s3.yandex.net/7495332394", + "7495957585": "https://devtools-registry.s3.yandex.net/7495957585", "6048579718": "https://devtools-registry.s3.yandex.net/6048579718", "2980468199": "https://devtools-registry.s3.yandex.net/2980468199", "5562224408": "https://devtools-registry.s3.yandex.net/5562224408" @@ -1334,24 +1347,28 @@ "6726871384": "devtools/ya/test/programs/flake8/flake8 for darwin", "6760720465": "devtools/ya/test/programs/flake8/flake8 for darwin", "6822863322": "devtools/ya/test/programs/flake8/flake8 for darwin", + "7501316202": "devtools/ya/test/programs/flake8/flake8 for darwin", "6603781997": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64", "6652223452": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64", "6674733966": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64", "6726867988": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64", "6760720256": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64", "6822862832": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64", + "7501315930": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64", "6603787904": "devtools/ya/test/programs/flake8/flake8 for linux", "6652225714": "devtools/ya/test/programs/flake8/flake8 for linux", "6674734993": "devtools/ya/test/programs/flake8/flake8 for linux", "6726875537": "devtools/ya/test/programs/flake8/flake8 for linux", "6760721014": "devtools/ya/test/programs/flake8/flake8 for linux", "6822864509": "devtools/ya/test/programs/flake8/flake8 for linux", + "7501316756": "devtools/ya/test/programs/flake8/flake8 for linux", "6603780916": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64", "6652222817": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64", "6674733653": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64", "6726866588": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64", "6760720046": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64", "6822862341": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64", + "7501315620": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64", "6603783079": "devtools/ya/test/programs/flake8/flake8 for linux-ppc64le", "6652224021": "devtools/ya/test/programs/flake8/flake8 for linux-ppc64le", "6674734285": "devtools/ya/test/programs/flake8/flake8 for linux-ppc64le", @@ -1362,6 +1379,7 @@ "6726873745": "devtools/ya/test/programs/flake8/flake8 for win32", "6760720811": "devtools/ya/test/programs/flake8/flake8 for win32", "6822864007": "devtools/ya/test/programs/flake8/flake8 for win32", + "7501316414": "devtools/ya/test/programs/flake8/flake8 for win32", "3961412335": "devtools/ya/test/programs/flake8/py2/flake8 for darwin", "3961411314": "devtools/ya/test/programs/flake8/py2/flake8 for darwin-arm64", "3961413236": "devtools/ya/test/programs/flake8/py2/flake8 for linux", @@ -1947,7 +1965,10 @@ "6990860705": "none-none-none-sandbox/backup/d386643e-58f8-43e1-8760-341d73801df8/yfm-docs.tar", "6990881789": "none-none-none-sandbox/backup/efc428e5-52a5-4a6f-8f0c-53f1d255efea/yfm-docs.tar", "7324461836": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/0541e185-8261-4b07-9149-257f03a9c8ae/yfm-docs.tar", + "7512107516": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/0ae16953-fe8d-4f1b-9a76-137110f40faf/yfm-docs.tar", "7193803465": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/17df2ad2-24bc-49e8-8909-b58685dac393/yfm-docs.tar", + "7512042091": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/203a60f0-6d4d-46bf-93eb-a2a5f998b768/yfm-docs.tar", + "7512103615": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/29fcd2aa-e0b5-481e-83c8-b7ff5e0157c9/yfm-docs.tar", "7324464594": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/32cc8c74-decd-44a8-bc8c-f8f0d7edfffe/yfm-docs.tar", "7487656957": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/4b747fb6-2d4e-4a7b-bbed-abb9a1e21761/yfm-docs.tar", "7458707245": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/4e7df89f-3762-4eba-ba73-562ccaeae548/yfm-docs.tar", @@ -2078,6 +2099,11 @@ "7451466407": "ynd-clang-18-optimized-linux-x86_64-c66062833fd14752a7ad8fbfb8f25f370b097278", "7407150115": "ynd-clang-18-optimized-mingw-w64-x86_64-56872a411472c7ec2b09f2c8372ea4c67d069a5b", "7451406900": "ynd-clang-18-optimized-mingw-w64-x86_64-c66062833fd14752a7ad8fbfb8f25f370b097278", + "7494722271": "ynd-clang-format-18-darwin-arm64-d0b9255e235e80974f9dfd0ebff3850929bed981", + "7494597813": "ynd-clang-format-18-darwin-x86_64-d0b9255e235e80974f9dfd0ebff3850929bed981", + "7495431598": "ynd-clang-format-18-linux-aarch64-d0b9255e235e80974f9dfd0ebff3850929bed981", + "7495332394": "ynd-clang-format-18-linux-x86_64-d0b9255e235e80974f9dfd0ebff3850929bed981", + "7495957585": "ynd-clang-format-18-mingw-w64-x86_64-d0b9255e235e80974f9dfd0ebff3850929bed981", "6048579718": "yt/go/ytrecipe/cmd/ytexec for linux", "2980468199": "ytexec for linux", "5562224408": "ytexec for linux" diff --git a/build/platform/yfm/ya.make b/build/platform/yfm/ya.make index 5935d3b7f3..2da4c854c2 100644 --- a/build/platform/yfm/ya.make +++ b/build/platform/yfm/ya.make @@ -6,10 +6,10 @@ ENDIF() DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE( YFM_TOOL - sbr:7487672728 FOR DARWIN-ARM64 - sbr:7487672728 FOR DARWIN - sbr:7487632536 FOR LINUX - sbr:7487656957 FOR WIN32 + sbr:7512103615 FOR DARWIN-ARM64 + sbr:7512103615 FOR DARWIN + sbr:7512042091 FOR LINUX + sbr:7512107516 FOR WIN32 ) END() diff --git a/build/plugins/lib/nots/package_manager/base/package_json.py b/build/plugins/lib/nots/package_manager/base/package_json.py index 449b386556..4fcb22e3ca 100644 --- a/build/plugins/lib/nots/package_manager/base/package_json.py +++ b/build/plugins/lib/nots/package_manager/base/package_json.py @@ -1,11 +1,12 @@ import json -import logging import os from six import iteritems +import logging from .utils import build_pj_path + logger = logging.getLogger(__name__) @@ -247,3 +248,6 @@ class PackageJson(object): messages.extend([f" - {key}" for key in missing_overrides]) return (not messages, messages) + + def get_pnpm_patched_dependencies(self) -> dict[str, str]: + return self.data.get("pnpm", {}).get("patchedDependencies", {}) diff --git a/build/plugins/lib/nots/package_manager/base/package_manager.py b/build/plugins/lib/nots/package_manager/base/package_manager.py index fbe4a16436..b780ae8edc 100644 --- a/build/plugins/lib/nots/package_manager/base/package_manager.py +++ b/build/plugins/lib/nots/package_manager/base/package_manager.py @@ -1,13 +1,13 @@ import os -import sys import subprocess - +import sys from abc import ABCMeta, abstractmethod from six import add_metaclass from .constants import NPM_REGISTRY_URL from .package_json import PackageJson from .utils import build_nm_path, build_pj_path +from .timeit import timeit class PackageManagerError(RuntimeError): @@ -126,7 +126,8 @@ class BasePackageManager(object): return [p[prefix_len:] for p in pj.get_workspace_map(ignore_self=True).keys()] - def _exec_command(self, args, include_defaults=True, script_path=None, env=None): + @timeit + def _exec_command(self, args, cwd: str, include_defaults=True, script_path=None, env=None): if not self.nodejs_bin_path: raise PackageManagerError("Unable to execute command: nodejs_bin_path is not configured") @@ -135,9 +136,7 @@ class BasePackageManager(object): + args + (self._get_default_options() if include_defaults else []) ) - p = subprocess.Popen( - cmd, cwd=self.build_path, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env - ) + p = subprocess.Popen(cmd, cwd=cwd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) stdout, stderr = p.communicate() if p.returncode != 0: diff --git a/build/plugins/lib/nots/package_manager/npm/npm_package_manager.py b/build/plugins/lib/nots/package_manager/npm/npm_package_manager.py index 1d251ee1c8..faacb19f49 100644 --- a/build/plugins/lib/nots/package_manager/npm/npm_package_manager.py +++ b/build/plugins/lib/nots/package_manager/npm/npm_package_manager.py @@ -1,6 +1,5 @@ import os - from ..base import BasePackageManager, PackageManagerError from ..base.constants import NODE_MODULES_WORKSPACE_BUNDLE_FILENAME from ..base.node_modules_bundler import bundle_node_modules @@ -157,7 +156,7 @@ class NpmPackageManager(BasePackageManager): env = os.environ.copy() env.update({"NPM_CONFIG_CACHE": os.path.join(self.build_path, ".npm-cache")}) - self._exec_command(install_cmd, env=env) + self._exec_command(install_cmd, cwd=self.build_path, env=env) def _prepare_workspace(self): lf = self.load_lockfile(build_pre_lockfile_path(self.build_path)) diff --git a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py index b40cafeb34..891bd96f6e 100644 --- a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py +++ b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py @@ -5,8 +5,9 @@ from .lockfile import PnpmLockfile from .utils import build_lockfile_path, build_pre_lockfile_path, build_ws_config_path from .workspace import PnpmWorkspace from ..base import BasePackageManager, PackageManagerError -from ..base.constants import NODE_MODULES_WORKSPACE_BUNDLE_FILENAME +from ..base.constants import NODE_MODULES_WORKSPACE_BUNDLE_FILENAME, PACKAGE_JSON_FILENAME, PNPM_LOCKFILE_FILENAME from ..base.node_modules_bundler import bundle_node_modules +from ..base.package_json import PackageJson from ..base.timeit import timeit from ..base.utils import ( b_rooted, @@ -47,6 +48,40 @@ class PnpmPackageManager(BasePackageManager): return os.path.join(home_dir(), ".cache", "pnpm-store") @timeit + def _create_local_node_modules(self, nm_store_path: str, store_dir: str, virtual_store_dir: str): + """ + Creates ~/.nots/nm_store/$MODDIR/node_modules folder (with installed packages and .pnpm/virtual-store) + + Should be used after build for local development ($SOURCE_DIR/node_modules should be a symlink to this folder). + + But now it is also a workaround to provide valid node_modules structure in the parent folder of virtual-store + It is needed for fixing custom module resolvers (like in tsc, webpack, etc...), which are trying to find modules in the parents directories + """ + + # provide files required for `pnpm install` + pj = PackageJson.load(os.path.join(self.build_path, PACKAGE_JSON_FILENAME)) + required_files = [ + PACKAGE_JSON_FILENAME, + PNPM_LOCKFILE_FILENAME, + *list(pj.bins_iter()), + *pj.get_pnpm_patched_dependencies().values(), + ] + + for f in required_files: + src = os.path.join(self.build_path, f) + if os.path.exists(src): + dst = os.path.join(nm_store_path, f) + try: + os.remove(dst) + except FileNotFoundError: + pass + + os.makedirs(os.path.dirname(dst), exist_ok=True) + shutil.copy(src, dst) + + self._run_pnpm_install(store_dir, virtual_store_dir, nm_store_path) + + @timeit def create_node_modules(self, yatool_prebuilder_path=None, local_cli=False, bundle=True): """ Creates node_modules directory according to the lockfile. @@ -68,7 +103,10 @@ class PnpmPackageManager(BasePackageManager): # Use single virtual-store location in ~/.nots/nm_store/$MODDIR/node_modules/.pnpm/virtual-store virtual_store_dir = os.path.join(build_nm_path(nm_store_path), self._VSTORE_NM_PATH) - self._run_pnpm_install(store_dir, virtual_store_dir) + self._create_local_node_modules(nm_store_path, store_dir, virtual_store_dir) + + self._run_pnpm_install(store_dir, virtual_store_dir, self.build_path) + self._run_apply_addons_if_need(yatool_prebuilder_path, virtual_store_dir) self._replace_internal_lockfile_with_original(virtual_store_dir) @@ -81,7 +119,7 @@ class PnpmPackageManager(BasePackageManager): ) @timeit - def _run_pnpm_install(self, store_dir: str, virtual_store_dir: str): + def _run_pnpm_install(self, store_dir: str, virtual_store_dir: str, cwd: str): install_cmd = [ "install", "--frozen-lockfile", @@ -100,8 +138,9 @@ class PnpmPackageManager(BasePackageManager): virtual_store_dir, ] - self._exec_command(install_cmd) + self._exec_command(install_cmd, cwd=cwd) + @timeit def calc_prepare_deps_inouts_and_resources( self, store_path: str, has_deps: bool ) -> tuple[list[str], list[str], list[str]]: @@ -122,6 +161,7 @@ class PnpmPackageManager(BasePackageManager): return ins, outs, resources + @timeit def calc_node_modules_inouts(self, local_cli: bool, has_deps: bool) -> tuple[list[str], list[str]]: """ Returns input and output paths for command that creates `node_modules` bundle. @@ -139,6 +179,7 @@ class PnpmPackageManager(BasePackageManager): return ins, outs + @timeit def extract_packages_meta_from_lockfiles(self, lf_paths): """ :type lf_paths: iterable of BaseLockfile @@ -167,6 +208,7 @@ class PnpmPackageManager(BasePackageManager): return PnpmWorkspace.load(build_ws_config_path(self.build_path)) + @timeit def build_workspace(self, tarballs_store: str): """ :rtype: PnpmWorkspace @@ -182,6 +224,7 @@ class PnpmPackageManager(BasePackageManager): return ws + @timeit def _build_merged_pre_lockfile(self, tarballs_store, dep_paths): """ :type dep_paths: list of str @@ -199,6 +242,7 @@ class PnpmPackageManager(BasePackageManager): lf.write() + @timeit def _build_merged_workspace_config(self, ws, dep_paths): """ NOTE: This method mutates `ws`. @@ -223,10 +267,12 @@ class PnpmPackageManager(BasePackageManager): "--virtual-store", virtual_store_dir, ], + cwd=self.build_path, include_defaults=False, script_path=os.path.join(yatool_prebuilder_path, "build", "bin", "prebuilder.js"), ) + @timeit def _replace_internal_lockfile_with_original(self, virtual_store_dir): original_lf_path = build_lockfile_path(self.sources_path) vs_lf_path = os.path.join(virtual_store_dir, "lock.yaml") @@ -236,14 +282,15 @@ class PnpmPackageManager(BasePackageManager): @timeit def _copy_pnpm_patches(self): pj = self.load_package_json_from_dir(self.sources_path) - patchedDependencies: dict[str, str] = pj.data.get("pnpm", {}).get("patchedDependencies", {}) + patched_dependencies: dict[str, str] = pj.data.get("pnpm", {}).get("patchedDependencies", {}) - for p in patchedDependencies.values(): + for p in patched_dependencies.values(): patch_source_path = os.path.join(self.sources_path, p) patch_build_path = os.path.join(self.build_path, p) os.makedirs(os.path.dirname(patch_build_path), exist_ok=True) shutil.copyfile(patch_source_path, patch_build_path) + @timeit def _get_default_options(self): return super(PnpmPackageManager, self)._get_default_options() + [ "--stream", @@ -252,5 +299,6 @@ class PnpmPackageManager(BasePackageManager): "--no-color", ] + @timeit def _get_debug_log_path(self): return self._nm_path(".pnpm-debug.log") diff --git a/build/plugins/lib/nots/package_manager/ya.make b/build/plugins/lib/nots/package_manager/ya.make index 08a791eb7e..6853c7b50a 100644 --- a/build/plugins/lib/nots/package_manager/ya.make +++ b/build/plugins/lib/nots/package_manager/ya.make @@ -4,7 +4,6 @@ PY3_LIBRARY() STYLE_PYTHON() - PY_SRCS( __init__.py ) diff --git a/build/ya.conf.json b/build/ya.conf.json index 995c61a0fc..35dfed0031 100644 --- a/build/ya.conf.json +++ b/build/ya.conf.json @@ -86,6 +86,14 @@ }, "formula": "build/platform/clang/clang-format/clang-format16.json" }, + "clang-format-18": { + "executable": { + "clang-format": [ + "clang-format" + ] + }, + "formula": "build/platform/clang/clang-format/clang-format18.json" + }, "clang16": { "executable": { "c++": [ @@ -484,6 +492,49 @@ } } }, + "clang-format-18": { + "platforms": [ + { + "default": true, + "host": { + "arch": "x86_64", + "os": "LINUX" + } + }, + { + "default": true, + "host": { + "arch": "aarch64", + "os": "LINUX" + } + }, + { + "default": true, + "host": { + "os": "WIN" + } + }, + { + "default": true, + "host": { + "os": "DARWIN" + } + }, + { + "default": true, + "host": { + "arch": "arm64", + "os": "DARWIN" + } + } + ], + "tools": { + "clang-format-18": { + "bottle": "clang-format-18", + "executable": "clang-format" + } + } + }, "clang16": { "env": { "CPATH": [ @@ -2302,6 +2353,9 @@ "clang-format": { "description": "Run Clang-Format source code formatter" }, + "clang-format-18": { + "description": "Run Clang-Format source code formatter (18 version)" + }, "clang-rename": { "description": "Run Clang-Rename refactoring tool" }, diff --git a/build/ymake_conf.py b/build/ymake_conf.py index 771c2b2902..77bf04e0a9 100755 --- a/build/ymake_conf.py +++ b/build/ymake_conf.py @@ -1551,7 +1551,6 @@ class GnuCompiler(Compiler): '-Wno-deprecated-enum-float-conversion', '-Wno-deprecated-volatile', '-Wno-pessimizing-move', - '-Wno-return-std-move', '-Wno-undefined-var-template', ] |