aboutsummaryrefslogtreecommitdiffstats
path: root/build
diff options
context:
space:
mode:
authorAlexander Smirnov <alex@ydb.tech>2024-11-22 09:21:01 +0000
committerAlexander Smirnov <alex@ydb.tech>2024-11-22 09:21:01 +0000
commita7cac42c868ca5722777ccee944217410812e72c (patch)
tree7bff7ce2a3ade72f6f15dfc634490d13628066ee /build
parenta18f18d81996ca8e681bb6cabd441b52833d99bf (diff)
parent9478cfdab4217d3710b96329466825bf47111d7d (diff)
downloadydb-a7cac42c868ca5722777ccee944217410812e72c.tar.gz
Merge branch 'rightlib' into mergelibs-241122-0919
Diffstat (limited to 'build')
-rw-r--r--build/config/tests/flake8/flake8.conf2
-rw-r--r--build/export_generators/ide-gradle/build.gradle.kts.jinja41
-rw-r--r--build/export_generators/ide-gradle/build.gradle.kts.proto.jinja80
-rw-r--r--build/external_resources/flake8_py3/ya.make10
-rw-r--r--build/mapping.conf.json26
-rw-r--r--build/platform/yfm/ya.make8
-rw-r--r--build/plugins/lib/nots/package_manager/base/package_json.py6
-rw-r--r--build/plugins/lib/nots/package_manager/base/package_manager.py11
-rw-r--r--build/plugins/lib/nots/package_manager/npm/npm_package_manager.py3
-rw-r--r--build/plugins/lib/nots/package_manager/pnpm/package_manager.py60
-rw-r--r--build/plugins/lib/nots/package_manager/ya.make1
-rw-r--r--build/ya.conf.json54
-rwxr-xr-xbuild/ymake_conf.py1
13 files changed, 226 insertions, 77 deletions
diff --git a/build/config/tests/flake8/flake8.conf b/build/config/tests/flake8/flake8.conf
index fda445a3f0..a4a60be743 100644
--- a/build/config/tests/flake8/flake8.conf
+++ b/build/config/tests/flake8/flake8.conf
@@ -285,6 +285,8 @@ ignore =
PLR915,
# too-many-boolean-expressions
PLR916,
+ # too-many-positional-arguments
+ PLR917,
# consider-merging-isinstance
PLR701,
# too-many-nested-blocks
diff --git a/build/export_generators/ide-gradle/build.gradle.kts.jinja b/build/export_generators/ide-gradle/build.gradle.kts.jinja
index 46bcd4cefd..798922b5b7 100644
--- a/build/export_generators/ide-gradle/build.gradle.kts.jinja
+++ b/build/export_generators/ide-gradle/build.gradle.kts.jinja
@@ -38,21 +38,6 @@ subprojects {
{%- endif %}
plugins {
-{#- some plugins configuration -#}
-{%- for library in target.consumer if library.classpath -%}
-{#- error prone plugin configuration -#}
-{%- if library.prebuilt and library.jar and (library.type != "contrib" or build_contribs) and "contrib/java/com/google/errorprone/error_prone_annotations/" in library.jar -%}
-
- id("net.ltgt.errorprone") version "{{ errorprone_plugin_version }}"
-
-{%- endif -%}
-{%- endfor -%}
-
-{#- lombok configuration -#}
-{#- TODO remove usings annotation_processors semantic -#}
-{%- if ("lombok.launch.AnnotationProcessorHider$AnnotationProcessor" in target.annotation_processors) or (target.use_annotation_processor|length and target.use_annotation_processor|select('startsWith', 'contrib/java/org/projectlombok/lombok')|length) %}
- id("io.freefair.lombok") version "8.6"
-{%- endif -%}
{%- if mainClass %}
`application`
{%- else %}
@@ -76,6 +61,17 @@ plugins {
{%- if target.with_kotlinc_plugin_serialization|length %}
kotlin("plugin.serialization") version "{{ kotlin_version }}"
{% endif -%}
+{%- endif -%}
+
+{#- errorprone plugin configuration -#}
+{%- if target.consumer|selectattr('jar', 'startsWith', 'contrib/java/com/google/errorprone/error_prone_annotations')|length %}
+ id("net.ltgt.errorprone") version "{{ errorprone_plugin_version }}"
+{%- endif -%}
+
+{#- lombok plugin configuration -#}
+{#- TODO remove usings annotation_processors semantic -#}
+{%- if ("lombok.launch.AnnotationProcessorHider$AnnotationProcessor" in target.annotation_processors) or (target.use_annotation_processor|length and target.use_annotation_processor|select('startsWith', 'contrib/java/org/projectlombok/lombok')|length) %}
+ id("io.freefair.lombok") version "8.6"
{%- endif %}
}
@@ -272,6 +268,7 @@ sourceSets.main.java.srcDirs += "{{ srcdir_glob[0] }}"
{%- endfor -%}
sourceSets {
+{%- if target.runs|length %}
main {
{#-
Default by Gradle:
@@ -280,13 +277,11 @@ sourceSets {
resources.srcDir("src/main/resources")
#}
-{%- if target.runs|length -%}
{%- for run in target.runs -%}
{{ OutDirs(run, ' java.srcDir("', '")') }}
-{%- endfor -%}
-{%- endif %}
+{%- endfor %}
}
-
+{%- endif %}
test {
{#-
Default by Gradle:
@@ -315,11 +310,9 @@ sourceSets {
dependencies {
{%- for library in target.consumer if library.classpath -%}
-{# error prone plugin configuration #}
-{%- if library.prebuilt and library.jar and (library.type != "contrib" or build_contribs) and "contrib/java/com/google/errorprone/error_prone_annotations" in library.jar -%}
-{% set errorprone_version = library.jar -%}
-{% set errorprone_parts = errorprone_version|replace("contrib/java/com/google/errorprone/error_prone_annotations/") -%}
-{% set errorprone_parts = split(errorprone_parts, '/') %}
+{%- if library.prebuilt and (library.type != "contrib" or build_contribs) and ("contrib/java/com/google/errorprone/error_prone_annotations" in library.jar) -%}
+{%- set errorprone_version = library.jar|replace("contrib/java/com/google/errorprone/error_prone_annotations/", "") -%}
+{%- set errorprone_parts = split(errorprone_version, '/', 2) %}
errorprone("com.google.errorprone:error_prone_core:{{ errorprone_parts[0] }}")
{%- endif -%}
diff --git a/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja b/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja
index 9b7b71f743..fe989ed989 100644
--- a/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja
+++ b/build/export_generators/ide-gradle/build.gradle.kts.proto.jinja
@@ -1,4 +1,6 @@
{%- set publish = target.publish -%}
+{%- set libraries = target.consumer|selectattr('type', 'eq', 'library') -%}
+
import com.google.protobuf.gradle.*
val baseBuildDir = "{{ export_root }}/gradle.build/"
@@ -7,7 +9,10 @@ subprojects {
buildDir = file(baseBuildDir + project.path.replaceFirst(":", "/").replace(":", "."))
}
-val buildProtoDir = File("${buildDir}", "__proto__")
+val mainProtosDir = File(buildDir, "main_protos")
+{%- if libraries|length %}
+val mainExtractedIncludeProtosDir = File(buildDir, "extracted-include-protos/main")
+{%- endif %}
plugins {
id("java-library")
@@ -37,14 +42,23 @@ repositories {
}
}
+val project_root = "{{ arcadia_root }}"
+
sourceSets {
main {
- java.srcDir("build/generated/source/proto")
+ java.srcDir("$buildDir/generated/sources/proto/main/java")
+{%- if target.proto_grpc %}
+ java.srcDir("$buildDir/generated/sources/proto/main/grpc")
+{%- endif %}
+ }
+ test {
+ java.srcDir("$buildDir/generated/sources/proto/test/java")
+{%- if target.proto_grpc %}
+ java.srcDir("$buildDir/generated/sources/proto/test/grpc")
+{%- endif %}
}
}
-val project_root = "{{ arcadia_root }}"
-
java {
withSourcesJar()
withJavadocJar()
@@ -92,21 +106,24 @@ dependencies {
}
{%- endif -%}
{%- endif -%}
-{%- endfor %}
+{%- endfor -%}
{%- if target.proto_namespace %}
- protobuf(files(File(buildProtoDir, "{{ target.proto_namespace }}")))
+ protobuf(files(File(mainProtosDir, "{{ target.proto_namespace }}")))
{%- else %}
- protobuf(files(buildProtoDir))
+ protobuf(files(mainProtosDir))
{%- endif %}
}
protobuf {
+ generatedFilesBaseDir = "$buildDir/generated/sources/proto"
+
protoc {
// Download from repositories
artifact = "com.google.protobuf:protoc:{%- if target.proto_compiler_version -%}{{ target.proto_compiler_version }}{%- else -%}3.22.5{%- endif -%}"
}
-{% if target.proto_grpc -%}
+{%- if target.proto_grpc %}
+
plugins {
id("grpc") {
artifact = "io.grpc:protoc-gen-grpc-java:{%- if target.proto_grpc_version -%}{{ target.proto_grpc_version }}{%- else -%}1.45.0{%- endif -%}"
@@ -115,48 +132,57 @@ protobuf {
id("grpckt") {
artifact = "io.grpc:protoc-gen-grpc-kotlin:{%- if target.proto_kotlin_grpc_version -%}{{ target.proto_kotlin_grpc_version }}{%- else -%}1.3.1{%- endif -%}:jdk8@jar"
}
-{% endif -%}
+{%- endif %}
}
+
generateProtoTasks {
all().forEach {
it.plugins {
id("grpc")
{%- if target.proto_kotlin_grpc %}
id("grpckt")
-{% endif -%}
+{%- endif %}
}
{%- if target.proto_kotlin_grpc %}
it.builtins {
create("kotlin")
}
-{% endif -%}
+{%- endif %}
}
}
{%- endif %}
}
-val prepareProto = tasks.register<Copy>("prepareProto") {
- from(rootDir) {
-{%- for proto in target.proto_files %}
- include("{{ proto }}")
-{%- endfor %}
-{# Mining proto-deps sources directories #}
-{%- for library in target.consumer if library.prebuilt -%}
-{%- if library.type == "library" %}
-{%- set proto_rep = library.classpath|replace('project(":','') %}
-{%- set proto_rep = proto_rep|replace('")','') %}
-{%- set proto_rep = proto_rep|replace(':','/') %}
-{%- set proto = proto_rep + '/**' %}
+val prepareMainProtos = tasks.register<Copy>("prepareMainProtos") {
+ from("$project_root") {
+{#- list of all current project proto files -#}
+{%- for proto in target.proto_files %}
include("{{ proto }}")
-{%- endif -%}
-{%- endfor %}
+{%- endfor %}
}
- into(buildProtoDir)
+ into(mainProtosDir)
}
+{% if libraries|length -%}
+val extractMainLibrariesProtos = tasks.register<Copy>("extractMainLibrariesProtos") {
+ from("$project_root") {
+{#- list of all library directories -#}
+{%- for library in libraries -%}
+{%- set path_and_jar = rsplit(library.jar, '/', 2) %}
+ include("{{ path_and_jar[0] }}/**/*.proto")
+{%- endfor %}
+ }
+ into(mainExtractedIncludeProtosDir)
+}
+
+{% endif -%}
afterEvaluate {
- tasks.getByName("extractProto").dependsOn(prepareProto)
+ tasks.getByName("extractProto").dependsOn(prepareMainProtos)
+{%- if libraries|length %}
+ tasks.getByName("extractProto").dependsOn(extractMainLibrariesProtos)
+{%- endif %}
}
+
{# To avoid problems when build project with proto #}
tasks.getByName("sourcesJar").dependsOn("generateProto")
diff --git a/build/external_resources/flake8_py3/ya.make b/build/external_resources/flake8_py3/ya.make
index 3b66251bae..3d89508e8e 100644
--- a/build/external_resources/flake8_py3/ya.make
+++ b/build/external_resources/flake8_py3/ya.make
@@ -12,12 +12,12 @@ ENDIF()
DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE(
FLAKE8_PY3
- sbr:6822862832 FOR DARWIN-ARM64
- sbr:6822863322 FOR DARWIN
+ sbr:7501315930 FOR DARWIN-ARM64
+ sbr:7501316202 FOR DARWIN
sbr:6726869647 FOR LINUX-PPC64LE
- sbr:6822864509 FOR LINUX
- sbr:6822862341 FOR LINUX-AARCH64
- sbr:6822864007 FOR WIN32
+ sbr:7501316756 FOR LINUX
+ sbr:7501315620 FOR LINUX-AARCH64
+ sbr:7501316414 FOR WIN32
)
END()
diff --git a/build/mapping.conf.json b/build/mapping.conf.json
index 7c3df72802..473f306a32 100644
--- a/build/mapping.conf.json
+++ b/build/mapping.conf.json
@@ -293,24 +293,28 @@
"6726871384": "https://devtools-registry.s3.yandex.net/6726871384",
"6760720465": "https://devtools-registry.s3.yandex.net/6760720465",
"6822863322": "https://devtools-registry.s3.yandex.net/6822863322",
+ "7501316202": "https://devtools-registry.s3.yandex.net/7501316202",
"6603781997": "https://devtools-registry.s3.yandex.net/6603781997",
"6652223452": "https://devtools-registry.s3.yandex.net/6652223452",
"6674733966": "https://devtools-registry.s3.yandex.net/6674733966",
"6726867988": "https://devtools-registry.s3.yandex.net/6726867988",
"6760720256": "https://devtools-registry.s3.yandex.net/6760720256",
"6822862832": "https://devtools-registry.s3.yandex.net/6822862832",
+ "7501315930": "https://devtools-registry.s3.yandex.net/7501315930",
"6603787904": "https://devtools-registry.s3.yandex.net/6603787904",
"6652225714": "https://devtools-registry.s3.yandex.net/6652225714",
"6674734993": "https://devtools-registry.s3.yandex.net/6674734993",
"6726875537": "https://devtools-registry.s3.yandex.net/6726875537",
"6760721014": "https://devtools-registry.s3.yandex.net/6760721014",
"6822864509": "https://devtools-registry.s3.yandex.net/6822864509",
+ "7501316756": "https://devtools-registry.s3.yandex.net/7501316756",
"6603780916": "https://devtools-registry.s3.yandex.net/6603780916",
"6652222817": "https://devtools-registry.s3.yandex.net/6652222817",
"6674733653": "https://devtools-registry.s3.yandex.net/6674733653",
"6726866588": "https://devtools-registry.s3.yandex.net/6726866588",
"6760720046": "https://devtools-registry.s3.yandex.net/6760720046",
"6822862341": "https://devtools-registry.s3.yandex.net/6822862341",
+ "7501315620": "https://devtools-registry.s3.yandex.net/7501315620",
"6603783079": "https://devtools-registry.s3.yandex.net/6603783079",
"6652224021": "https://devtools-registry.s3.yandex.net/6652224021",
"6674734285": "https://devtools-registry.s3.yandex.net/6674734285",
@@ -321,6 +325,7 @@
"6726873745": "https://devtools-registry.s3.yandex.net/6726873745",
"6760720811": "https://devtools-registry.s3.yandex.net/6760720811",
"6822864007": "https://devtools-registry.s3.yandex.net/6822864007",
+ "7501316414": "https://devtools-registry.s3.yandex.net/7501316414",
"3961412335": "https://devtools-registry.s3.yandex.net/3961412335",
"3961411314": "https://devtools-registry.s3.yandex.net/3961411314",
"3961413236": "https://devtools-registry.s3.yandex.net/3961413236",
@@ -906,7 +911,10 @@
"6990860705": "https://devtools-registry.s3.yandex.net/6990860705",
"6990881789": "https://devtools-registry.s3.yandex.net/6990881789",
"7324461836": "https://devtools-registry.s3.yandex.net/7324461836",
+ "7512107516": "https://devtools-registry.s3.yandex.net/7512107516",
"7193803465": "https://devtools-registry.s3.yandex.net/7193803465",
+ "7512042091": "https://devtools-registry.s3.yandex.net/7512042091",
+ "7512103615": "https://devtools-registry.s3.yandex.net/7512103615",
"7324464594": "https://devtools-registry.s3.yandex.net/7324464594",
"7487656957": "https://devtools-registry.s3.yandex.net/7487656957",
"7458707245": "https://devtools-registry.s3.yandex.net/7458707245",
@@ -1037,6 +1045,11 @@
"7451466407": "https://devtools-registry.s3.yandex.net/7451466407",
"7407150115": "https://devtools-registry.s3.yandex.net/7407150115",
"7451406900": "https://devtools-registry.s3.yandex.net/7451406900",
+ "7494722271": "https://devtools-registry.s3.yandex.net/7494722271",
+ "7494597813": "https://devtools-registry.s3.yandex.net/7494597813",
+ "7495431598": "https://devtools-registry.s3.yandex.net/7495431598",
+ "7495332394": "https://devtools-registry.s3.yandex.net/7495332394",
+ "7495957585": "https://devtools-registry.s3.yandex.net/7495957585",
"6048579718": "https://devtools-registry.s3.yandex.net/6048579718",
"2980468199": "https://devtools-registry.s3.yandex.net/2980468199",
"5562224408": "https://devtools-registry.s3.yandex.net/5562224408"
@@ -1334,24 +1347,28 @@
"6726871384": "devtools/ya/test/programs/flake8/flake8 for darwin",
"6760720465": "devtools/ya/test/programs/flake8/flake8 for darwin",
"6822863322": "devtools/ya/test/programs/flake8/flake8 for darwin",
+ "7501316202": "devtools/ya/test/programs/flake8/flake8 for darwin",
"6603781997": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64",
"6652223452": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64",
"6674733966": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64",
"6726867988": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64",
"6760720256": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64",
"6822862832": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64",
+ "7501315930": "devtools/ya/test/programs/flake8/flake8 for darwin-arm64",
"6603787904": "devtools/ya/test/programs/flake8/flake8 for linux",
"6652225714": "devtools/ya/test/programs/flake8/flake8 for linux",
"6674734993": "devtools/ya/test/programs/flake8/flake8 for linux",
"6726875537": "devtools/ya/test/programs/flake8/flake8 for linux",
"6760721014": "devtools/ya/test/programs/flake8/flake8 for linux",
"6822864509": "devtools/ya/test/programs/flake8/flake8 for linux",
+ "7501316756": "devtools/ya/test/programs/flake8/flake8 for linux",
"6603780916": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64",
"6652222817": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64",
"6674733653": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64",
"6726866588": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64",
"6760720046": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64",
"6822862341": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64",
+ "7501315620": "devtools/ya/test/programs/flake8/flake8 for linux-aarch64",
"6603783079": "devtools/ya/test/programs/flake8/flake8 for linux-ppc64le",
"6652224021": "devtools/ya/test/programs/flake8/flake8 for linux-ppc64le",
"6674734285": "devtools/ya/test/programs/flake8/flake8 for linux-ppc64le",
@@ -1362,6 +1379,7 @@
"6726873745": "devtools/ya/test/programs/flake8/flake8 for win32",
"6760720811": "devtools/ya/test/programs/flake8/flake8 for win32",
"6822864007": "devtools/ya/test/programs/flake8/flake8 for win32",
+ "7501316414": "devtools/ya/test/programs/flake8/flake8 for win32",
"3961412335": "devtools/ya/test/programs/flake8/py2/flake8 for darwin",
"3961411314": "devtools/ya/test/programs/flake8/py2/flake8 for darwin-arm64",
"3961413236": "devtools/ya/test/programs/flake8/py2/flake8 for linux",
@@ -1947,7 +1965,10 @@
"6990860705": "none-none-none-sandbox/backup/d386643e-58f8-43e1-8760-341d73801df8/yfm-docs.tar",
"6990881789": "none-none-none-sandbox/backup/efc428e5-52a5-4a6f-8f0c-53f1d255efea/yfm-docs.tar",
"7324461836": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/0541e185-8261-4b07-9149-257f03a9c8ae/yfm-docs.tar",
+ "7512107516": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/0ae16953-fe8d-4f1b-9a76-137110f40faf/yfm-docs.tar",
"7193803465": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/17df2ad2-24bc-49e8-8909-b58685dac393/yfm-docs.tar",
+ "7512042091": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/203a60f0-6d4d-46bf-93eb-a2a5f998b768/yfm-docs.tar",
+ "7512103615": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/29fcd2aa-e0b5-481e-83c8-b7ff5e0157c9/yfm-docs.tar",
"7324464594": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/32cc8c74-decd-44a8-bc8c-f8f0d7edfffe/yfm-docs.tar",
"7487656957": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/4b747fb6-2d4e-4a7b-bbed-abb9a1e21761/yfm-docs.tar",
"7458707245": "none-none-none-service_resources/TASKLET_EXECUTABLE/backup/4e7df89f-3762-4eba-ba73-562ccaeae548/yfm-docs.tar",
@@ -2078,6 +2099,11 @@
"7451466407": "ynd-clang-18-optimized-linux-x86_64-c66062833fd14752a7ad8fbfb8f25f370b097278",
"7407150115": "ynd-clang-18-optimized-mingw-w64-x86_64-56872a411472c7ec2b09f2c8372ea4c67d069a5b",
"7451406900": "ynd-clang-18-optimized-mingw-w64-x86_64-c66062833fd14752a7ad8fbfb8f25f370b097278",
+ "7494722271": "ynd-clang-format-18-darwin-arm64-d0b9255e235e80974f9dfd0ebff3850929bed981",
+ "7494597813": "ynd-clang-format-18-darwin-x86_64-d0b9255e235e80974f9dfd0ebff3850929bed981",
+ "7495431598": "ynd-clang-format-18-linux-aarch64-d0b9255e235e80974f9dfd0ebff3850929bed981",
+ "7495332394": "ynd-clang-format-18-linux-x86_64-d0b9255e235e80974f9dfd0ebff3850929bed981",
+ "7495957585": "ynd-clang-format-18-mingw-w64-x86_64-d0b9255e235e80974f9dfd0ebff3850929bed981",
"6048579718": "yt/go/ytrecipe/cmd/ytexec for linux",
"2980468199": "ytexec for linux",
"5562224408": "ytexec for linux"
diff --git a/build/platform/yfm/ya.make b/build/platform/yfm/ya.make
index 5935d3b7f3..2da4c854c2 100644
--- a/build/platform/yfm/ya.make
+++ b/build/platform/yfm/ya.make
@@ -6,10 +6,10 @@ ENDIF()
DECLARE_EXTERNAL_HOST_RESOURCES_BUNDLE(
YFM_TOOL
- sbr:7487672728 FOR DARWIN-ARM64
- sbr:7487672728 FOR DARWIN
- sbr:7487632536 FOR LINUX
- sbr:7487656957 FOR WIN32
+ sbr:7512103615 FOR DARWIN-ARM64
+ sbr:7512103615 FOR DARWIN
+ sbr:7512042091 FOR LINUX
+ sbr:7512107516 FOR WIN32
)
END()
diff --git a/build/plugins/lib/nots/package_manager/base/package_json.py b/build/plugins/lib/nots/package_manager/base/package_json.py
index 449b386556..4fcb22e3ca 100644
--- a/build/plugins/lib/nots/package_manager/base/package_json.py
+++ b/build/plugins/lib/nots/package_manager/base/package_json.py
@@ -1,11 +1,12 @@
import json
-import logging
import os
from six import iteritems
+import logging
from .utils import build_pj_path
+
logger = logging.getLogger(__name__)
@@ -247,3 +248,6 @@ class PackageJson(object):
messages.extend([f" - {key}" for key in missing_overrides])
return (not messages, messages)
+
+ def get_pnpm_patched_dependencies(self) -> dict[str, str]:
+ return self.data.get("pnpm", {}).get("patchedDependencies", {})
diff --git a/build/plugins/lib/nots/package_manager/base/package_manager.py b/build/plugins/lib/nots/package_manager/base/package_manager.py
index fbe4a16436..b780ae8edc 100644
--- a/build/plugins/lib/nots/package_manager/base/package_manager.py
+++ b/build/plugins/lib/nots/package_manager/base/package_manager.py
@@ -1,13 +1,13 @@
import os
-import sys
import subprocess
-
+import sys
from abc import ABCMeta, abstractmethod
from six import add_metaclass
from .constants import NPM_REGISTRY_URL
from .package_json import PackageJson
from .utils import build_nm_path, build_pj_path
+from .timeit import timeit
class PackageManagerError(RuntimeError):
@@ -126,7 +126,8 @@ class BasePackageManager(object):
return [p[prefix_len:] for p in pj.get_workspace_map(ignore_self=True).keys()]
- def _exec_command(self, args, include_defaults=True, script_path=None, env=None):
+ @timeit
+ def _exec_command(self, args, cwd: str, include_defaults=True, script_path=None, env=None):
if not self.nodejs_bin_path:
raise PackageManagerError("Unable to execute command: nodejs_bin_path is not configured")
@@ -135,9 +136,7 @@ class BasePackageManager(object):
+ args
+ (self._get_default_options() if include_defaults else [])
)
- p = subprocess.Popen(
- cmd, cwd=self.build_path, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env
- )
+ p = subprocess.Popen(cmd, cwd=cwd, stdin=None, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
stdout, stderr = p.communicate()
if p.returncode != 0:
diff --git a/build/plugins/lib/nots/package_manager/npm/npm_package_manager.py b/build/plugins/lib/nots/package_manager/npm/npm_package_manager.py
index 1d251ee1c8..faacb19f49 100644
--- a/build/plugins/lib/nots/package_manager/npm/npm_package_manager.py
+++ b/build/plugins/lib/nots/package_manager/npm/npm_package_manager.py
@@ -1,6 +1,5 @@
import os
-
from ..base import BasePackageManager, PackageManagerError
from ..base.constants import NODE_MODULES_WORKSPACE_BUNDLE_FILENAME
from ..base.node_modules_bundler import bundle_node_modules
@@ -157,7 +156,7 @@ class NpmPackageManager(BasePackageManager):
env = os.environ.copy()
env.update({"NPM_CONFIG_CACHE": os.path.join(self.build_path, ".npm-cache")})
- self._exec_command(install_cmd, env=env)
+ self._exec_command(install_cmd, cwd=self.build_path, env=env)
def _prepare_workspace(self):
lf = self.load_lockfile(build_pre_lockfile_path(self.build_path))
diff --git a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
index b40cafeb34..891bd96f6e 100644
--- a/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
+++ b/build/plugins/lib/nots/package_manager/pnpm/package_manager.py
@@ -5,8 +5,9 @@ from .lockfile import PnpmLockfile
from .utils import build_lockfile_path, build_pre_lockfile_path, build_ws_config_path
from .workspace import PnpmWorkspace
from ..base import BasePackageManager, PackageManagerError
-from ..base.constants import NODE_MODULES_WORKSPACE_BUNDLE_FILENAME
+from ..base.constants import NODE_MODULES_WORKSPACE_BUNDLE_FILENAME, PACKAGE_JSON_FILENAME, PNPM_LOCKFILE_FILENAME
from ..base.node_modules_bundler import bundle_node_modules
+from ..base.package_json import PackageJson
from ..base.timeit import timeit
from ..base.utils import (
b_rooted,
@@ -47,6 +48,40 @@ class PnpmPackageManager(BasePackageManager):
return os.path.join(home_dir(), ".cache", "pnpm-store")
@timeit
+ def _create_local_node_modules(self, nm_store_path: str, store_dir: str, virtual_store_dir: str):
+ """
+ Creates ~/.nots/nm_store/$MODDIR/node_modules folder (with installed packages and .pnpm/virtual-store)
+
+ Should be used after build for local development ($SOURCE_DIR/node_modules should be a symlink to this folder).
+
+ But now it is also a workaround to provide valid node_modules structure in the parent folder of virtual-store
+ It is needed for fixing custom module resolvers (like in tsc, webpack, etc...), which are trying to find modules in the parents directories
+ """
+
+ # provide files required for `pnpm install`
+ pj = PackageJson.load(os.path.join(self.build_path, PACKAGE_JSON_FILENAME))
+ required_files = [
+ PACKAGE_JSON_FILENAME,
+ PNPM_LOCKFILE_FILENAME,
+ *list(pj.bins_iter()),
+ *pj.get_pnpm_patched_dependencies().values(),
+ ]
+
+ for f in required_files:
+ src = os.path.join(self.build_path, f)
+ if os.path.exists(src):
+ dst = os.path.join(nm_store_path, f)
+ try:
+ os.remove(dst)
+ except FileNotFoundError:
+ pass
+
+ os.makedirs(os.path.dirname(dst), exist_ok=True)
+ shutil.copy(src, dst)
+
+ self._run_pnpm_install(store_dir, virtual_store_dir, nm_store_path)
+
+ @timeit
def create_node_modules(self, yatool_prebuilder_path=None, local_cli=False, bundle=True):
"""
Creates node_modules directory according to the lockfile.
@@ -68,7 +103,10 @@ class PnpmPackageManager(BasePackageManager):
# Use single virtual-store location in ~/.nots/nm_store/$MODDIR/node_modules/.pnpm/virtual-store
virtual_store_dir = os.path.join(build_nm_path(nm_store_path), self._VSTORE_NM_PATH)
- self._run_pnpm_install(store_dir, virtual_store_dir)
+ self._create_local_node_modules(nm_store_path, store_dir, virtual_store_dir)
+
+ self._run_pnpm_install(store_dir, virtual_store_dir, self.build_path)
+
self._run_apply_addons_if_need(yatool_prebuilder_path, virtual_store_dir)
self._replace_internal_lockfile_with_original(virtual_store_dir)
@@ -81,7 +119,7 @@ class PnpmPackageManager(BasePackageManager):
)
@timeit
- def _run_pnpm_install(self, store_dir: str, virtual_store_dir: str):
+ def _run_pnpm_install(self, store_dir: str, virtual_store_dir: str, cwd: str):
install_cmd = [
"install",
"--frozen-lockfile",
@@ -100,8 +138,9 @@ class PnpmPackageManager(BasePackageManager):
virtual_store_dir,
]
- self._exec_command(install_cmd)
+ self._exec_command(install_cmd, cwd=cwd)
+ @timeit
def calc_prepare_deps_inouts_and_resources(
self, store_path: str, has_deps: bool
) -> tuple[list[str], list[str], list[str]]:
@@ -122,6 +161,7 @@ class PnpmPackageManager(BasePackageManager):
return ins, outs, resources
+ @timeit
def calc_node_modules_inouts(self, local_cli: bool, has_deps: bool) -> tuple[list[str], list[str]]:
"""
Returns input and output paths for command that creates `node_modules` bundle.
@@ -139,6 +179,7 @@ class PnpmPackageManager(BasePackageManager):
return ins, outs
+ @timeit
def extract_packages_meta_from_lockfiles(self, lf_paths):
"""
:type lf_paths: iterable of BaseLockfile
@@ -167,6 +208,7 @@ class PnpmPackageManager(BasePackageManager):
return PnpmWorkspace.load(build_ws_config_path(self.build_path))
+ @timeit
def build_workspace(self, tarballs_store: str):
"""
:rtype: PnpmWorkspace
@@ -182,6 +224,7 @@ class PnpmPackageManager(BasePackageManager):
return ws
+ @timeit
def _build_merged_pre_lockfile(self, tarballs_store, dep_paths):
"""
:type dep_paths: list of str
@@ -199,6 +242,7 @@ class PnpmPackageManager(BasePackageManager):
lf.write()
+ @timeit
def _build_merged_workspace_config(self, ws, dep_paths):
"""
NOTE: This method mutates `ws`.
@@ -223,10 +267,12 @@ class PnpmPackageManager(BasePackageManager):
"--virtual-store",
virtual_store_dir,
],
+ cwd=self.build_path,
include_defaults=False,
script_path=os.path.join(yatool_prebuilder_path, "build", "bin", "prebuilder.js"),
)
+ @timeit
def _replace_internal_lockfile_with_original(self, virtual_store_dir):
original_lf_path = build_lockfile_path(self.sources_path)
vs_lf_path = os.path.join(virtual_store_dir, "lock.yaml")
@@ -236,14 +282,15 @@ class PnpmPackageManager(BasePackageManager):
@timeit
def _copy_pnpm_patches(self):
pj = self.load_package_json_from_dir(self.sources_path)
- patchedDependencies: dict[str, str] = pj.data.get("pnpm", {}).get("patchedDependencies", {})
+ patched_dependencies: dict[str, str] = pj.data.get("pnpm", {}).get("patchedDependencies", {})
- for p in patchedDependencies.values():
+ for p in patched_dependencies.values():
patch_source_path = os.path.join(self.sources_path, p)
patch_build_path = os.path.join(self.build_path, p)
os.makedirs(os.path.dirname(patch_build_path), exist_ok=True)
shutil.copyfile(patch_source_path, patch_build_path)
+ @timeit
def _get_default_options(self):
return super(PnpmPackageManager, self)._get_default_options() + [
"--stream",
@@ -252,5 +299,6 @@ class PnpmPackageManager(BasePackageManager):
"--no-color",
]
+ @timeit
def _get_debug_log_path(self):
return self._nm_path(".pnpm-debug.log")
diff --git a/build/plugins/lib/nots/package_manager/ya.make b/build/plugins/lib/nots/package_manager/ya.make
index 08a791eb7e..6853c7b50a 100644
--- a/build/plugins/lib/nots/package_manager/ya.make
+++ b/build/plugins/lib/nots/package_manager/ya.make
@@ -4,7 +4,6 @@ PY3_LIBRARY()
STYLE_PYTHON()
-
PY_SRCS(
__init__.py
)
diff --git a/build/ya.conf.json b/build/ya.conf.json
index 995c61a0fc..35dfed0031 100644
--- a/build/ya.conf.json
+++ b/build/ya.conf.json
@@ -86,6 +86,14 @@
},
"formula": "build/platform/clang/clang-format/clang-format16.json"
},
+ "clang-format-18": {
+ "executable": {
+ "clang-format": [
+ "clang-format"
+ ]
+ },
+ "formula": "build/platform/clang/clang-format/clang-format18.json"
+ },
"clang16": {
"executable": {
"c++": [
@@ -484,6 +492,49 @@
}
}
},
+ "clang-format-18": {
+ "platforms": [
+ {
+ "default": true,
+ "host": {
+ "arch": "x86_64",
+ "os": "LINUX"
+ }
+ },
+ {
+ "default": true,
+ "host": {
+ "arch": "aarch64",
+ "os": "LINUX"
+ }
+ },
+ {
+ "default": true,
+ "host": {
+ "os": "WIN"
+ }
+ },
+ {
+ "default": true,
+ "host": {
+ "os": "DARWIN"
+ }
+ },
+ {
+ "default": true,
+ "host": {
+ "arch": "arm64",
+ "os": "DARWIN"
+ }
+ }
+ ],
+ "tools": {
+ "clang-format-18": {
+ "bottle": "clang-format-18",
+ "executable": "clang-format"
+ }
+ }
+ },
"clang16": {
"env": {
"CPATH": [
@@ -2302,6 +2353,9 @@
"clang-format": {
"description": "Run Clang-Format source code formatter"
},
+ "clang-format-18": {
+ "description": "Run Clang-Format source code formatter (18 version)"
+ },
"clang-rename": {
"description": "Run Clang-Rename refactoring tool"
},
diff --git a/build/ymake_conf.py b/build/ymake_conf.py
index 771c2b2902..77bf04e0a9 100755
--- a/build/ymake_conf.py
+++ b/build/ymake_conf.py
@@ -1551,7 +1551,6 @@ class GnuCompiler(Compiler):
'-Wno-deprecated-enum-float-conversion',
'-Wno-deprecated-volatile',
'-Wno-pessimizing-move',
- '-Wno-return-std-move',
'-Wno-undefined-var-template',
]