fix(RH2D):新增RH2D支持urma mock编译-yuanrong-datasystem-AtomGit

M.bazelrc+7-0

	@@ -43,11 +43,18 @@ build:urma --copt=-DURMA_OVER_UB
43	# Provides a local shm+TCP transport so urma_manager.cpp can be unit-tested.	43	# Provides a local shm+TCP transport so urma_manager.cpp can be unit-tested.
44	# Mutually exclusive with build:urma (the actual SDK backend).	44	# Mutually exclusive with build:urma (the actual SDK backend).
45	build:urma_mock --define=enable_urma_mock=true	45	build:urma_mock --define=enable_urma_mock=true
		46	+build:urma_mock --copt=-DUSE_URMA
		47	+build:urma_mock --copt=-DUSE_URMA_MOCK
46		48
47	build:pipeline_h2d --config=urma	49	build:pipeline_h2d --config=urma
48	build:pipeline_h2d --define=enable_pipeline_h2d=true	50	build:pipeline_h2d --define=enable_pipeline_h2d=true
49	build:pipeline_h2d --copt=-DBUILD_PIPLN_H2D	51	build:pipeline_h2d --copt=-DBUILD_PIPLN_H2D
50		52
		53	+build:pipeline_h2d_mock --config=urma_mock
		54	+build:pipeline_h2d_mock --define=enable_pipeline_h2d=true
		55	+build:pipeline_h2d_mock --define=mock_mode=true
		56	+build:pipeline_h2d_mock --copt=-DBUILD_PIPLN_H2D
		57	+
51	build:asan --copt=-fsanitize=address	58	build:asan --copt=-fsanitize=address
52	build:asan --copt=-fno-omit-frame-pointer	59	build:asan --copt=-fno-omit-frame-pointer
53	build:asan --copt=-g3	60	build:asan --copt=-g3

MBUILD.bazel+1-4

	@@ -40,10 +40,7 @@ config_setting(
40		40
41	config_setting(	41	config_setting(
42	name = "enable_pipeline_h2d",	42	name = "enable_pipeline_h2d",
43	- define_values = {	43	+ define_values = {"enable_pipeline_h2d": "true"},
44	- "enable_pipeline_h2d": "true",
45	- "enable_urma": "true",
46	- },
47	visibility = ["//visibility:public"],	44	visibility = ["//visibility:public"],
48	)	45	)
49		46

Mbazel/ds_deps.bzl+3-3

	@@ -243,10 +243,10 @@ def setup_mlcachedirect():
243	maybe(	243	maybe(
244	http_archive,	244	http_archive,
245	name = "mlcachedirect",	245	name = "mlcachedirect",
246	- sha256 = "392b21d7d73f3fb402eab14144c83a12e54b2ade605dfaee100e70a5eba820e2",	246	+ sha256 = "457d42f7d6b2aac0cbee131aeadd41563032f33171a511f0492cdc9d7adfcae7",
247	- strip_prefix = "MLCacheDirect-0.1.6",	247	+ strip_prefix = "MLCacheDirect-0.1.7",
248	urls = [	248	urls = [
249	- "https://github.com/openeuler-mirror/MLCacheDirect/archive/refs/tags/v0.1.6.tar.gz",	249	+ "https://github.com/openeuler-mirror/MLCacheDirect/archive/refs/tags/v0.1.7.tar.gz",
250	],	250	],
251	)	251	)
252		252

Mbazel/sdk/root.BUILD.bazel+1-4

	@@ -12,10 +12,7 @@ config_setting(
12		12
13	config_setting(	13	config_setting(
14	name = "enable_pipeline_h2d",	14	name = "enable_pipeline_h2d",
15	- define_values = {	15	+ define_values = {"enable_pipeline_h2d": "true"},
16	- "enable_pipeline_h2d": "true",
17	- "enable_urma": "true",
18	- },
19	)	16	)
20		17
21	config_setting(	18	config_setting(

Mcmake/dependency.cmake+3-2

	@@ -33,8 +33,8 @@ if (BUILD_WITH_URMA)
33	endif()	33	endif()
34		34
35	if (BUILD_PIPLN_H2D)	35	if (BUILD_PIPLN_H2D)
36	- if (NOT BUILD_WITH_URMA)	36	+ if (NOT BUILD_WITH_URMA AND NOT BUILD_WITH_URMA_MOCK)
37	- message(FATAL_ERROR "BUILD_PIPLN_H2D should be enabled with BUILD_WITH_URMA")	37	+ message(FATAL_ERROR "BUILD_PIPLN_H2D requires BUILD_WITH_URMA or BUILD_WITH_URMA_MOCK")
38	endif()	38	endif()
39	include(${CMAKE_SOURCE_DIR}/cmake/external_libs/pipeline_rh2d.cmake)	39	include(${CMAKE_SOURCE_DIR}/cmake/external_libs/pipeline_rh2d.cmake)
40	endif()	40	endif()
	@@ -44,6 +44,7 @@ if (BUILD_WITH_URMA_MOCK)
44	message(FATAL_ERROR "BUILD_WITH_URMA_MOCK and BUILD_WITH_URMA are mutually exclusive (mock backend is for dev/CI without liburma.so)")	44	message(FATAL_ERROR "BUILD_WITH_URMA_MOCK and BUILD_WITH_URMA are mutually exclusive (mock backend is for dev/CI without liburma.so)")
45	endif()	45	endif()
46	message(STATUS "Build with urma mock (local shm+UDS backend, no SDK needed)")	46	message(STATUS "Build with urma mock (local shm+UDS backend, no SDK needed)")
		47	+ add_compile_definitions(USE_URMA USE_URMA_MOCK)
47	endif()	48	endif()
48	if (BUILD_WITH_RDMA)	49	if (BUILD_WITH_RDMA)
49	include(${CMAKE_SOURCE_DIR}/cmake/external_libs/ucx.cmake)	50	include(${CMAKE_SOURCE_DIR}/cmake/external_libs/ucx.cmake)

Mcmake/external_libs/mlcachedirect.cmake+17-3

	@@ -14,8 +14,8 @@
14		14
15	# Description: Build MLCacheDirect os_transport as a static CMake target for RH2D.	15	# Description: Build MLCacheDirect os_transport as a static CMake target for RH2D.
16		16
17	-set(MLCacheDirect_VERSION 0.1.6)	17	+set(MLCacheDirect_VERSION 0.1.7)
18	-set(MLCacheDirect_SHA256 392b21d7d73f3fb402eab14144c83a12e54b2ade605dfaee100e70a5eba820e2)	18	+set(MLCacheDirect_SHA256 457d42f7d6b2aac0cbee131aeadd41563032f33171a511f0492cdc9d7adfcae7)
19		19
20	if (MLCACHEDIRECT_URL)	20	if (MLCACHEDIRECT_URL)
21	set(MLCacheDirect_URL "${MLCACHEDIRECT_URL}")	21	set(MLCacheDirect_URL "${MLCACHEDIRECT_URL}")
	@@ -38,6 +38,10 @@ set(MLCacheDirect_OS_TRANSPORT_SRCS
38	${MLCacheDirect_SOURCE_DIR}/src/os_transport_log.c	38	${MLCacheDirect_SOURCE_DIR}/src/os_transport_log.c
39	)	39	)
40		40
		41	+if (BUILD_WITH_URMA_MOCK)
		42	+ list(APPEND MLCacheDirect_OS_TRANSPORT_SRCS ${MLCacheDirect_SOURCE_DIR}/mock/urma_mock_backend.c)
		43	+endif()
		44	+
41	set(MLCacheDirect_PREFIXED_INCLUDE_DIR ${CMAKE_BINARY_DIR}/third_party/mlcachedirect_prefixed_include)	45	set(MLCacheDirect_PREFIXED_INCLUDE_DIR ${CMAKE_BINARY_DIR}/third_party/mlcachedirect_prefixed_include)
42	file(MAKE_DIRECTORY ${MLCacheDirect_PREFIXED_INCLUDE_DIR}/os-transport)	46	file(MAKE_DIRECTORY ${MLCacheDirect_PREFIXED_INCLUDE_DIR}/os-transport)
43	file(COPY ${MLCacheDirect_SOURCE_DIR}/include/	47	file(COPY ${MLCacheDirect_SOURCE_DIR}/include/
	@@ -55,12 +59,22 @@ target_compile_options(mlcachedirect_os_transport_static PRIVATE
55	-Wno-implicit-function-declaration	59	-Wno-implicit-function-declaration
56	)	60	)
57	target_compile_definitions(mlcachedirect_os_transport_static PRIVATE _POSIX_C_SOURCE=200809L)	61	target_compile_definitions(mlcachedirect_os_transport_static PRIVATE _POSIX_C_SOURCE=200809L)
		62	+if (BUILD_WITH_URMA_MOCK)
		63	+ target_compile_definitions(mlcachedirect_os_transport_static PUBLIC USE_URMA_MOCK)
		64	+endif()
58	target_include_directories(mlcachedirect_os_transport_static	65	target_include_directories(mlcachedirect_os_transport_static
59	PUBLIC	66	PUBLIC
60	${MLCacheDirect_SOURCE_DIR}/include	67	${MLCacheDirect_SOURCE_DIR}/include
61	${MLCacheDirect_PREFIXED_INCLUDE_DIR}	68	${MLCacheDirect_PREFIXED_INCLUDE_DIR}
62	)	69	)
63	-target_link_libraries(mlcachedirect_os_transport_static PUBLIC Threads::Threads ${URMA_LIBRARY})	70	+if (BUILD_WITH_URMA_MOCK)
		71	+ target_include_directories(mlcachedirect_os_transport_static PUBLIC
		72	+ ${MLCacheDirect_SOURCE_DIR}/mock
		73	+ )
		74	+ target_link_libraries(mlcachedirect_os_transport_static PUBLIC Threads::Threads)
		75	+else()
		76	+ target_link_libraries(mlcachedirect_os_transport_static PUBLIC Threads::Threads ${URMA_LIBRARY})
		77	+endif()
64		78
65	set(OS_TRANSPORT_INCLUDE_DIR ${MLCacheDirect_PREFIXED_INCLUDE_DIR})	79	set(OS_TRANSPORT_INCLUDE_DIR ${MLCacheDirect_PREFIXED_INCLUDE_DIR})
66	set(MLCacheDirect_INCLUDE_DIR ${MLCacheDirect_SOURCE_DIR}/include)	80	set(MLCacheDirect_INCLUDE_DIR ${MLCacheDirect_SOURCE_DIR}/include)

Mscripts/build_bazel.sh+6-2

	@@ -80,7 +80,11 @@ function _bazel_build_configs() {
80		80
81	# Pipeline H2D	81	# Pipeline H2D
82	if is_on "${BUILD_PIPLN_H2D}"; then	82	if is_on "${BUILD_PIPLN_H2D}"; then
83	- echo "--config=pipeline_h2d"	83	+ if is_on "${BUILD_WITH_URMA_MOCK}"; then
		84	+ echo "--config=pipeline_h2d_mock"
		85	+ else
		86	+ echo "--config=pipeline_h2d"
		87	+ fi
84	fi	88	fi
85		89
86	# Perf	90	# Perf
	@@ -94,7 +98,7 @@ function _bazel_build_configs() {
94	fi	98	fi
95		99
96	# URMA mock backend (mutually exclusive with BUILD_WITH_URMA, enforced in cmake path)	100	# URMA mock backend (mutually exclusive with BUILD_WITH_URMA, enforced in cmake path)
97	- if is_on "${BUILD_WITH_URMA_MOCK}"; then	101	+ if is_on "${BUILD_WITH_URMA_MOCK}" && ! is_on "${BUILD_PIPLN_H2D}"; then
98	echo "--config=urma_mock"	102	echo "--config=urma_mock"
99	fi	103	fi
100		104

Mscripts/build_common.sh+3-2

	@@ -119,8 +119,9 @@ function has_urma_runtime_lib() {
119	}	119	}
120		120
121	function normalize_build_options() {	121	function normalize_build_options() {
122	- if is_on "${BUILD_PIPLN_H2D}" && [[ "${BUILD_SYSTEM}" == "cmake" ]] && ! is_on "${BUILD_WITH_URMA}"; then	122	+ if is_on "${BUILD_PIPLN_H2D}" && [[ "${BUILD_SYSTEM}" == "cmake" ]] && ! is_on "${BUILD_WITH_URMA}" \
123	- echo -e "-- [INFO] Pipeline H2D requires URMA. Enabling BUILD_WITH_URMA because -T on was specified."	123	+ && ! is_on "${BUILD_WITH_URMA_MOCK}"; then
		124	+ echo -e "-- [INFO] Pipeline H2D requires an URMA backend. Enabling BUILD_WITH_URMA."
124	BUILD_WITH_URMA="on"	125	BUILD_WITH_URMA="on"
125	fi	126	fi
126	if [[ "${BUILD_SYSTEM}" == "cmake" && "${RUN_TESTS}" != "off" ]] && is_on "${BUILD_WITH_URMA}" \	127	if [[ "${BUILD_SYSTEM}" == "cmake" && "${RUN_TESTS}" != "off" ]] && is_on "${BUILD_WITH_URMA}" \

Msrc/datasystem/common/os_transport_pipeline/BUILD.bazel+10-0

	@@ -62,6 +62,11 @@ ds_cc_library(
62	"@local_cuda//:cuda_headers",	62	"@local_cuda//:cuda_headers",
63	],	63	],
64	"//conditions:default": [],	64	"//conditions:default": [],
		65	+ }) + select({
		66	+ "//:enable_urma_mock": [
		67	+ "//src/datasystem/common/urma_mock:urma_abi_compat",
		68	+ ],
		69	+ "//conditions:default": [],
65	}),	70	}),
66	)	71	)
67		72
	@@ -94,5 +99,10 @@ ds_cc_library(
94	"@local_cuda//:cuda_headers",	99	"@local_cuda//:cuda_headers",
95	],	100	],
96	"//conditions:default": [],	101	"//conditions:default": [],
		102	+ }) + select({
		103	+ "//:enable_urma_mock": [
		104	+ "//src/datasystem/common/urma_mock:urma_abi_compat",
		105	+ ],
		106	+ "//conditions:default": [],
97	}),	107	}),
98	)	108	)

Msrc/datasystem/common/os_transport_pipeline/chunk_manager.cpp+4-0

	@@ -28,7 +28,11 @@
28	#include <utility>	28	#include <utility>
29	#include <unistd.h>	29	#include <unistd.h>
30	#include <vector>	30	#include <vector>
		31	+#ifdef USE_URMA_MOCK
		32	+#include "datasystem/common/urma_mock/abi/urma_abi_compat.h"
		33	+#else
31	#include <ub/umdk/urma/urma_api.h>	34	#include <ub/umdk/urma/urma_api.h>
		35	+#endif
32		36
33	// maybe not need	37	// maybe not need
34	#include <iostream>	38	#include <iostream>

Msrc/datasystem/common/os_transport_pipeline/chunk_manager.h+4-0

	@@ -26,7 +26,11 @@
26	#include <shared_mutex>	26	#include <shared_mutex>
27	#include <string>	27	#include <string>
28	#include <unordered_map>	28	#include <unordered_map>
		29	+#ifdef USE_URMA_MOCK
		30	+#include "datasystem/common/urma_mock/abi/urma_abi_compat.h"
		31	+#else
29	#include <ub/umdk/urma/urma_api.h>	32	#include <ub/umdk/urma/urma_api.h>
		33	+#endif
30		34
31	#include "datasystem/common/os_transport_pipeline/os_transport_pipeline_types.h"	35	#include "datasystem/common/os_transport_pipeline/os_transport_pipeline_types.h"
32	#include "datasystem/common/os_transport_pipeline/pipeline_notify_queue.h"	36	#include "datasystem/common/os_transport_pipeline/pipeline_notify_queue.h"

Msrc/datasystem/common/os_transport_pipeline/os_transport_pipeline_common_api.h+5-1

	@@ -25,7 +25,11 @@
25		25
26	#ifdef BUILD_PIPLN_H2D	26	#ifdef BUILD_PIPLN_H2D
27	#include <atomic>	27	#include <atomic>
		28	+#ifdef USE_URMA_MOCK
		29	+#include "datasystem/common/urma_mock/abi/urma_abi_compat.h"
		30	+#else
28	#include <ub/umdk/urma/urma_api.h>	31	#include <ub/umdk/urma/urma_api.h>
		32	+#endif
29	#include "datasystem/common/flags/common_flags.h"	33	#include "datasystem/common/flags/common_flags.h"
30	#include "datasystem/common/os_transport_pipeline/chunk_manager.h"	34	#include "datasystem/common/os_transport_pipeline/chunk_manager.h"
31	inline std::atomic<bool> &ClientPipelineRH2DEnabled()	35	inline std::atomic<bool> &ClientPipelineRH2DEnabled()
	@@ -88,4 +92,4 @@ struct H2DChunkManager {
88	#endif	92	#endif
89		93
90		94
91	-#endif	95	+#endif

Msrc/datasystem/common/os_transport_pipeline/os_transport_pipeline_types.h+4-0

	@@ -25,8 +25,12 @@
25	#include <cstring>	25	#include <cstring>
26	#include <sstream>	26	#include <sstream>
27	#ifdef BUILD_PIPLN_H2D	27	#ifdef BUILD_PIPLN_H2D
		28	+#ifdef USE_URMA_MOCK
		29	+#include "datasystem/common/urma_mock/abi/urma_abi_compat.h"
		30	+#else
28	#include <ub/umdk/urma/urma_api.h>	31	#include <ub/umdk/urma/urma_api.h>
29	#endif	32	#endif
		33	+#endif
30		34
31	#define CHUNKTAG_REQID_START_INDEX 0	35	#define CHUNKTAG_REQID_START_INDEX 0
32	#define CHUNKTAG_REQID_LEN 60	36	#define CHUNKTAG_REQID_LEN 60

Msrc/datasystem/common/urma_mock/abi/urma_abi_compat.h+3-0

	@@ -24,6 +24,9 @@
24	#ifndef DATASYSTEM_COMMON_URMA_MOCK_URMA_ABI_COMPAT_H	24	#ifndef DATASYSTEM_COMMON_URMA_MOCK_URMA_ABI_COMPAT_H
25	#define DATASYSTEM_COMMON_URMA_MOCK_URMA_ABI_COMPAT_H	25	#define DATASYSTEM_COMMON_URMA_MOCK_URMA_ABI_COMPAT_H
26		26
		27	+// MLCacheDirect extends the same mock ABI; prevent its base header from redefining shared URMA types.
		28	+#define URMA_ABI_COMPAT_H
		29	+
27	// This header mirrors the URMA C ABI. Keep C-style names, typedefs, macros, unions, magic values, and pointer-based	30	// This header mirrors the URMA C ABI. Keep C-style names, typedefs, macros, unions, magic values, and pointer-based
28	// signatures aligned with the external ABI instead of applying project C++ style rewrites.	31	// signatures aligned with the external ABI instead of applying project C++ style rewrites.
29	// NOLINTBEGIN	32	// NOLINTBEGIN