已合并
[feat] 完善 ACLRTC `ResourceRegistry` 的资源发现、动态库加载和文件物化流程 #5167
sjtulxh创建于 18 天前
[feat] 完善 ACLRTC `ResourceRegistry` 的资源发现、动态库加载和文件物化流程 #5167
已合并
共 3 个文件变更+341-102
| @@ -48,6 +48,7 @@ uint32_t gDlsymCalls = 0U; | |||
| 48 | uint32_t gDlcloseCalls = 0U; | 48 | uint32_t gDlcloseCalls = 0U; |
| 49 | bool gDlsymReportsError = false; | 49 | bool gDlsymReportsError = false; |
| 50 | uint32_t gDlerrorCalls = 0U; | 50 | uint32_t gDlerrorCalls = 0U; |
| 51 | +std::string gDlopenFailurePath; | ||
| 51 | 52 | ||
| 52 | const AcCompileResourceBundleHeader* TestBundleGetter() { return gBundleHeader; } | 53 | const AcCompileResourceBundleHeader* TestBundleGetter() { return gBundleHeader; } |
| 53 | 54 | ||
| @@ -63,6 +64,15 @@ void* DlopenFailure(const char*, int) | |||
| 63 | return nullptr; | 64 | return nullptr; |
| 64 | } | 65 | } |
| 65 | 66 | ||
| 67 | +void* DlopenConfigured(const char* path, int) | ||
| 68 | +{ | ||
| 69 | + ++gDlopenCalls; | ||
| 70 | + if (path != nullptr && gDlopenFailurePath == path) { | ||
| 71 | + return nullptr; | ||
| 72 | + } | ||
| 73 | + return reinterpret_cast<void*>(static_cast<uintptr_t>(0x1234U)); | ||
| 74 | +} | ||
| 75 | + | ||
| 66 | void* DlsymBundle(void*, const char*) | 76 | void* DlsymBundle(void*, const char*) |
| 67 | { | 77 | { |
| 68 | ++gDlsymCalls; | 78 | ++gDlsymCalls; |
| @@ -240,6 +250,7 @@ protected: | |||
| 240 | gDlcloseCalls = 0U; | 250 | gDlcloseCalls = 0U; |
| 241 | gDlsymReportsError = false; | 251 | gDlsymReportsError = false; |
| 242 | gDlerrorCalls = 0U; | 252 | gDlerrorCalls = 0U; |
| 253 | + gDlopenFailurePath.clear(); | ||
| 243 | } | 254 | } |
| 244 | 255 | ||
| 245 | void MockSuccessfulLoader() | 256 | void MockSuccessfulLoader() |
| @@ -262,16 +273,32 @@ TEST_F(ResourceRegistryTest, AutomaticSearchRootsParsesCustomVendorAndBuiltInPat | |||
| 262 | opp / "vendors/config.ini", | 273 | opp / "vendors/config.ini", |
| 263 | "ignored=value\n load_priority = vendor_a, ../escape, vendor_b, vendor_a, , /absolute\n"); | 274 | "ignored=value\n load_priority = vendor_a, ../escape, vendor_b, vendor_a, , /absolute\n"); |
| 264 | SetEnvironment("ASCEND_CUSTOM_OPP_PATH", " /custom/one:/custom/two:/custom/one:: "); | 275 | SetEnvironment("ASCEND_CUSTOM_OPP_PATH", " /custom/one:/custom/two:/custom/one:: "); |
| 265 | - SetEnvironment("ASCEND_OPP_PATH", opp.string()); | 276 | + SetEnvironment("ASCEND_OPP_PATH", (opp / ".." / opp.filename()).string()); |
| 266 | 277 | ||
| 267 | const AutomaticRoots roots = ResourceRegistry::AutomaticSearchRoots(); | 278 | const AutomaticRoots roots = ResourceRegistry::AutomaticSearchRoots(); |
| 279 | + const fs::path canonicalOpp = fs::canonical(opp); | ||
| 268 | ASSERT_EQ(roots.custom.size(), 4U); | 280 | ASSERT_EQ(roots.custom.size(), 4U); |
| 269 | EXPECT_EQ(roots.custom[0], "/custom/one/op_impl/ai_core/tbe/kernel/jit"); | 281 | EXPECT_EQ(roots.custom[0], "/custom/one/op_impl/ai_core/tbe/kernel/jit"); |
| 270 | EXPECT_EQ(roots.custom[1], "/custom/two/op_impl/ai_core/tbe/kernel/jit"); | 282 | EXPECT_EQ(roots.custom[1], "/custom/two/op_impl/ai_core/tbe/kernel/jit"); |
| 271 | - EXPECT_EQ(roots.custom[2], (opp / "vendors/vendor_a/op_impl/ai_core/tbe/kernel/jit").string()); | 283 | + EXPECT_EQ(roots.custom[2], (canonicalOpp / "vendors/vendor_a/op_impl/ai_core/tbe/kernel/jit").string()); |
| 272 | - EXPECT_EQ(roots.custom[3], (opp / "vendors/vendor_b/op_impl/ai_core/tbe/kernel/jit").string()); | 284 | + EXPECT_EQ(roots.custom[3], (canonicalOpp / "vendors/vendor_b/op_impl/ai_core/tbe/kernel/jit").string()); |
| 273 | ASSERT_EQ(roots.builtIn.size(), 1U); | 285 | ASSERT_EQ(roots.builtIn.size(), 1U); |
| 274 | - EXPECT_EQ(roots.builtIn[0], (opp / "built-in/op_impl/ai_core/tbe/kernel/jit").string()); | 286 | + EXPECT_EQ(roots.builtIn[0], (canonicalOpp / "built-in/op_impl/ai_core/tbe/kernel/jit").string()); |
| 287 | + | ||
| 288 | + SetEnvironment("ASCEND_OPP_PATH", Path("missing-opp").string()); | ||
| 289 | + const AutomaticRoots missingOpp = ResourceRegistry::AutomaticSearchRoots(); | ||
| 290 | + EXPECT_EQ(missingOpp.custom.size(), 2U); | ||
| 291 | + EXPECT_TRUE(missingOpp.builtIn.empty()); | ||
| 292 | + | ||
| 293 | + const fs::path oppWithoutVendorConfig = Path("opp-without-vendor-config"); | ||
| 294 | + MakeDirectory(oppWithoutVendorConfig); | ||
| 295 | + SetEnvironment("ASCEND_OPP_PATH", oppWithoutVendorConfig.string()); | ||
| 296 | + const AutomaticRoots missingVendorConfig = ResourceRegistry::AutomaticSearchRoots(); | ||
| 297 | + EXPECT_EQ(missingVendorConfig.custom.size(), 2U); | ||
| 298 | + ASSERT_EQ(missingVendorConfig.builtIn.size(), 1U); | ||
| 299 | + EXPECT_EQ( | ||
| 300 | + missingVendorConfig.builtIn[0], | ||
| 301 | + (fs::canonical(oppWithoutVendorConfig) / "built-in/op_impl/ai_core/tbe/kernel/jit").string()); | ||
| 275 | 302 | ||
| 276 | UnsetEnvironment("ASCEND_OPP_PATH"); | 303 | UnsetEnvironment("ASCEND_OPP_PATH"); |
| 277 | const AutomaticRoots noOpp = ResourceRegistry::AutomaticSearchRoots(); | 304 | const AutomaticRoots noOpp = ResourceRegistry::AutomaticSearchRoots(); |
| @@ -332,14 +359,15 @@ TEST_F(ResourceRegistryTest, CollectAndDiscoverLibrariesHandleExplicitAndAutomat | |||
| 332 | EXPECT_EQ(libraries[0].sourceType, ResourceSourceType::External); | 359 | EXPECT_EQ(libraries[0].sourceType, ResourceSourceType::External); |
| 333 | 360 | ||
| 334 | libraries.clear(); | 361 | libraries.clear(); |
| 335 | - EXPECT_EQ(ResourceRegistry::DiscoverLibraries(explicitRoot.string().c_str(), libraries), ResourceStatus::Success); | 362 | + EXPECT_EQ( |
| 336 | - ASSERT_EQ(libraries.size(), 1U); | 363 | + ResourceRegistry::DiscoverLibraries(explicitRoot.string().c_str(), libraries), ResourceStatus::InvalidResource); |
| 337 | - EXPECT_EQ(libraries[0].path, fs::canonical(top).string()); | 364 | + EXPECT_TRUE(libraries.empty()); |
| 338 | 365 | ||
| 339 | const fs::path empty = Path("empty"); | 366 | const fs::path empty = Path("empty"); |
| 340 | MakeDirectory(empty); | 367 | MakeDirectory(empty); |
| 341 | libraries.clear(); | 368 | libraries.clear(); |
| 342 | - EXPECT_EQ(ResourceRegistry::DiscoverLibraries(empty.string().c_str(), libraries), ResourceStatus::NotFound); | 369 | + EXPECT_EQ(ResourceRegistry::DiscoverLibraries(empty.string().c_str(), libraries), ResourceStatus::InvalidResource); |
| 370 | + EXPECT_TRUE(libraries.empty()); | ||
| 343 | EXPECT_EQ( | 371 | EXPECT_EQ( |
| 344 | ResourceRegistry::DiscoverLibraries(Path("absent").string().c_str(), libraries), | 372 | ResourceRegistry::DiscoverLibraries(Path("absent").string().c_str(), libraries), |
| 345 | ResourceStatus::InvalidResource); | 373 | ResourceStatus::InvalidResource); |
| @@ -774,23 +802,37 @@ TEST_F(ResourceRegistryTest, LibraryDeleterHandlesCloseFailure) | |||
| 774 | 802 | ||
| 775 | TEST_F(ResourceRegistryTest, LoadLibraryReportsOpenFailure) | 803 | TEST_F(ResourceRegistryTest, LoadLibraryReportsOpenFailure) |
| 776 | { | 804 | { |
| 805 | + const fs::path so = Path("open-failure.so"); | ||
| 806 | + WriteFile(so); | ||
| 777 | MOCKER(dlopen).expects(once()).will(invoke(DlopenFailure)); | 807 | MOCKER(dlopen).expects(once()).will(invoke(DlopenFailure)); |
| 778 | MOCKER(dlerror).stubs().will(invoke(DlerrorFailure)); | 808 | MOCKER(dlerror).stubs().will(invoke(DlerrorFailure)); |
| 779 | StageState stage; | 809 | StageState stage; |
| 780 | EXPECT_EQ( | 810 | EXPECT_EQ( |
| 781 | - ResourceRegistry::LoadLibrary({"missing.so", ResourceSourceType::External}, stage), ResourceStatus::LoadError); | 811 | + ResourceRegistry::LoadLibrary({so.string(), ResourceSourceType::External}, stage), ResourceStatus::LoadError); |
| 782 | EXPECT_EQ(gDlopenCalls, 1U); | 812 | EXPECT_EQ(gDlopenCalls, 1U); |
| 783 | } | 813 | } |
| 784 | 814 | ||
| 815 | +TEST_F(ResourceRegistryTest, LoadLibraryRejectsUnresolvablePathBeforeDlopen) | ||
| 816 | +{ | ||
| 817 | + MOCKER(dlopen).stubs().will(invoke(DlopenFailure)); | ||
| 818 | + StageState stage; | ||
| 819 | + EXPECT_EQ( | ||
| 820 | + ResourceRegistry::LoadLibrary({Path("missing.so").string(), ResourceSourceType::External}, stage), | ||
| 821 | + ResourceStatus::InvalidResource); | ||
| 822 | + EXPECT_EQ(gDlopenCalls, 0U); | ||
| 823 | +} | ||
| 824 | + | ||
| 785 | TEST_F(ResourceRegistryTest, LoadLibraryReportsBundleFailureAndClosesHandle) | 825 | TEST_F(ResourceRegistryTest, LoadLibraryReportsBundleFailureAndClosesHandle) |
| 786 | { | 826 | { |
| 827 | + const fs::path so = Path("bundle-failure.so"); | ||
| 828 | + WriteFile(so); | ||
| 787 | MOCKER(dlopen).expects(once()).will(invoke(DlopenSuccess)); | 829 | MOCKER(dlopen).expects(once()).will(invoke(DlopenSuccess)); |
| 788 | MOCKER(dlsym).stubs().will(invoke(DlsymMissing)); | 830 | MOCKER(dlsym).stubs().will(invoke(DlsymMissing)); |
| 789 | MOCKER(dlerror).stubs().will(invoke(DlerrorControlled)); | 831 | MOCKER(dlerror).stubs().will(invoke(DlerrorControlled)); |
| 790 | MOCKER(dlclose).expects(once()).will(invoke(DlcloseSuccess)); | 832 | MOCKER(dlclose).expects(once()).will(invoke(DlcloseSuccess)); |
| 791 | StageState stage; | 833 | StageState stage; |
| 792 | EXPECT_EQ( | 834 | EXPECT_EQ( |
| 793 | - ResourceRegistry::LoadLibrary({"invalid.so", ResourceSourceType::Custom}, stage), ResourceStatus::LoadError); | 835 | + ResourceRegistry::LoadLibrary({so.string(), ResourceSourceType::Custom}, stage), ResourceStatus::LoadError); |
| 794 | EXPECT_EQ(gDlcloseCalls, 1U); | 836 | EXPECT_EQ(gDlcloseCalls, 1U); |
| 795 | } | 837 | } |
| 796 | 838 | ||
| @@ -811,6 +853,9 @@ TEST_F(ResourceRegistryTest, LoadLibraryReportsManifestFailure) | |||
| 811 | TEST_F(ResourceRegistryTest, LoadLibraryLoadsValidManifest) | 853 | TEST_F(ResourceRegistryTest, LoadLibraryLoadsValidManifest) |
| 812 | { | 854 | { |
| 813 | const SourceLayout layout = CreateLayout("load-library-success"); | 855 | const SourceLayout layout = CreateLayout("load-library-success"); |
| 856 | + const fs::path soPath(layout.soPath); | ||
| 857 | + const std::string nonCanonicalPath = | ||
| 858 | + (soPath.parent_path() / ".." / soPath.parent_path().filename() / soPath.filename()).string(); | ||
| 814 | const std::string json = R"({"resource_id":"loaded","source_file":"source.cpp"})"; | 859 | const std::string json = R"({"resource_id":"loaded","source_file":"source.cpp"})"; |
| 815 | AcCompileResourceManifest manifest{StringView(json), nullptr, 0U}; | 860 | AcCompileResourceManifest manifest{StringView(json), nullptr, 0U}; |
| 816 | AcCompileResourceBundle bundle = MakeBundle(&manifest, 1U); | 861 | AcCompileResourceBundle bundle = MakeBundle(&manifest, 1U); |
| @@ -818,24 +863,133 @@ TEST_F(ResourceRegistryTest, LoadLibraryLoadsValidManifest) | |||
| 818 | MockSuccessfulLoader(); | 863 | MockSuccessfulLoader(); |
| 819 | StageState stage; | 864 | StageState stage; |
| 820 | EXPECT_EQ( | 865 | EXPECT_EQ( |
| 821 | - ResourceRegistry::LoadLibrary({layout.soPath, ResourceSourceType::Custom}, stage), ResourceStatus::Success); | 866 | + ResourceRegistry::LoadLibrary({nonCanonicalPath, ResourceSourceType::Custom}, stage), ResourceStatus::Success); |
| 822 | - EXPECT_EQ(stage.custom.resources.size(), 1U); | 867 | + ASSERT_EQ(stage.custom.resources.size(), 1U); |
| 868 | + EXPECT_EQ(stage.custom.resources.at("loaded")->sourceSoPath, fs::canonical(soPath).string()); | ||
| 823 | EXPECT_EQ(gDlcloseCalls, 1U); | 869 | EXPECT_EQ(gDlcloseCalls, 1U); |
| 824 | } | 870 | } |
| 825 | 871 | ||
| 826 | -TEST_F(ResourceRegistryTest, LoadLibrariesStopsAtFirstFailureAndMarksDiscoveredCategory) | 872 | +TEST_F(ResourceRegistryTest, LoadLibrariesContinuesAfterAllFailuresAndReturnsFirstFailure) |
| 827 | { | 873 | { |
| 874 | + const fs::path first = Path("first.so"); | ||
| 875 | + const fs::path second = Path("second.so"); | ||
| 876 | + WriteFile(first); | ||
| 877 | + WriteFile(second); | ||
| 828 | MOCKER(dlopen).stubs().will(invoke(DlopenFailure)); | 878 | MOCKER(dlopen).stubs().will(invoke(DlopenFailure)); |
| 829 | MOCKER(dlerror).stubs().will(invoke(DlerrorFailure)); | 879 | MOCKER(dlerror).stubs().will(invoke(DlerrorFailure)); |
| 830 | StageState stage; | 880 | StageState stage; |
| 831 | const std::vector<LibrarySpec> libraries = { | 881 | const std::vector<LibrarySpec> libraries = { |
| 832 | - {"first.so", ResourceSourceType::External}, | 882 | + {first.string(), ResourceSourceType::External}, |
| 833 | - {"second.so", ResourceSourceType::BuiltIn}, | 883 | + {second.string(), ResourceSourceType::BuiltIn}, |
| 834 | }; | 884 | }; |
| 835 | EXPECT_EQ(ResourceRegistry::LoadLibraries(libraries, stage), ResourceStatus::LoadError); | 885 | EXPECT_EQ(ResourceRegistry::LoadLibraries(libraries, stage), ResourceStatus::LoadError); |
| 886 | + EXPECT_EQ(gDlopenCalls, 2U); | ||
| 887 | + EXPECT_TRUE(stage.external.discovered); | ||
| 888 | + EXPECT_TRUE(stage.builtIn.discovered); | ||
| 889 | + EXPECT_TRUE(stage.external.resources.empty()); | ||
| 890 | + EXPECT_TRUE(stage.builtIn.resources.empty()); | ||
| 891 | +} | ||
| 892 | + | ||
| 893 | +TEST_F(ResourceRegistryTest, LoadLibrariesSkipsFailedSoAndKeepsSuccessfulSo) | ||
| 894 | +{ | ||
| 895 | + const std::string json = R"({"resource_id":"continued"})"; | ||
| 896 | + AcCompileResourceManifest manifest{StringView(json), nullptr, 0U}; | ||
| 897 | + AcCompileResourceBundle bundle = MakeBundle(&manifest, 1U); | ||
| 898 | + gBundleHeader = &bundle.header; | ||
| 899 | + const fs::path first = Path("first.so"); | ||
| 900 | + const fs::path second = Path("second.so"); | ||
| 901 | + WriteFile(first); | ||
| 902 | + WriteFile(second); | ||
| 903 | + gDlopenFailurePath = first.string(); | ||
| 904 | + MOCKER(dlopen).stubs().will(invoke(DlopenConfigured)); | ||
| 905 | + MOCKER(dlsym).stubs().will(invoke(DlsymBundle)); | ||
| 906 | + MOCKER(dlerror).stubs().will(invoke(DlerrorControlled)); | ||
| 907 | + MOCKER(dlclose).stubs().will(invoke(DlcloseSuccess)); | ||
| 908 | + | ||
| 909 | + StageState stage; | ||
| 910 | + const std::vector<LibrarySpec> libraries = { | ||
| 911 | + {gDlopenFailurePath, ResourceSourceType::External}, | ||
| 912 | + {second.string(), ResourceSourceType::BuiltIn}, | ||
| 913 | + }; | ||
| 914 | + EXPECT_EQ(ResourceRegistry::LoadLibraries(libraries, stage), ResourceStatus::Success); | ||
| 915 | + EXPECT_EQ(gDlopenCalls, 2U); | ||
| 916 | + EXPECT_EQ(gDlcloseCalls, 1U); | ||
| 917 | + EXPECT_TRUE(stage.external.discovered); | ||
| 918 | + EXPECT_TRUE(stage.external.resources.empty()); | ||
| 919 | + EXPECT_TRUE(stage.builtIn.discovered); | ||
| 920 | + EXPECT_EQ(stage.builtIn.resources.count("continued"), 1U); | ||
| 921 | +} | ||
| 922 | + | ||
| 923 | +TEST_F(ResourceRegistryTest, LoadLibrariesDiscardsPartialManifestsFromFailedSo) | ||
| 924 | +{ | ||
| 925 | + const std::string validJson = R"({"resource_id":"partial"})"; | ||
| 926 | + const std::string invalidJson = "{"; | ||
| 927 | + AcCompileResourceManifest manifests[] = { | ||
| 928 | + {StringView(validJson), nullptr, 0U}, | ||
| 929 | + {StringView(invalidJson), nullptr, 0U}, | ||
| 930 | + }; | ||
| 931 | + AcCompileResourceBundle bundle = MakeBundle(manifests, 2U); | ||
| 932 | + gBundleHeader = &bundle.header; | ||
| 933 | + MockSuccessfulLoader(); | ||
| 934 | + const fs::path so = Path("partial.so"); | ||
| 935 | + WriteFile(so); | ||
| 936 | + | ||
| 937 | + StageState stage; | ||
| 938 | + EXPECT_EQ( | ||
| 939 | + ResourceRegistry::LoadLibraries({{so.string(), ResourceSourceType::External}}, stage), | ||
| 940 | + ResourceStatus::InvalidResource); | ||
| 941 | + EXPECT_TRUE(stage.external.discovered); | ||
| 942 | + EXPECT_TRUE(stage.external.resources.empty()); | ||
| 943 | + EXPECT_EQ(stage.external.bytes, 0U); | ||
| 944 | + EXPECT_EQ(stage.external.files, 0U); | ||
| 945 | +} | ||
| 946 | + | ||
| 947 | +TEST_F(ResourceRegistryTest, LoadLibrariesSkipsResourcesConflictingWithAnEarlierSo) | ||
| 948 | +{ | ||
| 949 | + const std::string json = R"({"resource_id":"conflict"})"; | ||
| 950 | + AcCompileResourceManifest manifest{StringView(json), nullptr, 0U}; | ||
| 951 | + AcCompileResourceBundle bundle = MakeBundle(&manifest, 1U); | ||
| 952 | + gBundleHeader = &bundle.header; | ||
| 953 | + MockSuccessfulLoader(); | ||
| 954 | + const fs::path first = Path("conflict-first.so"); | ||
| 955 | + const fs::path second = Path("conflict-second.so"); | ||
| 956 | + WriteFile(first); | ||
| 957 | + WriteFile(second); | ||
| 958 | + | ||
| 959 | + StageState stage; | ||
| 960 | + EXPECT_EQ( | ||
| 961 | + ResourceRegistry::LoadLibraries( | ||
| 962 | + {{first.string(), ResourceSourceType::External}, {second.string(), ResourceSourceType::External}}, stage), | ||
| 963 | + ResourceStatus::Success); | ||
| 964 | + EXPECT_EQ(gDlopenCalls, 2U); | ||
| 965 | + ASSERT_EQ(stage.external.resources.size(), 1U); | ||
| 966 | + EXPECT_EQ(stage.external.resources.at("conflict")->sourceSoPath, fs::canonical(first).string()); | ||
| 967 | +} | ||
| 968 | + | ||
| 969 | +TEST_F(ResourceRegistryTest, LoadLibrariesSkipsResourcesThatExceedCumulativeLimits) | ||
| 970 | +{ | ||
| 971 | + const std::string json = R"({"resource_id":"limited"})"; | ||
| 972 | + const std::string name = "payload.bin"; | ||
| 973 | + const std::string path = "payload.bin"; | ||
| 974 | + const std::vector<uint8_t> payload = {1U}; | ||
| 975 | + AcCompileResourceFile file{StringView(name), StringView(path), payload.data(), payload.size()}; | ||
| 976 | + AcCompileResourceManifest manifest{StringView(json), &file, 1U}; | ||
| 977 | + AcCompileResourceBundle bundle = MakeBundle(&manifest, 1U); | ||
| 978 | + gBundleHeader = &bundle.header; | ||
| 979 | + MockSuccessfulLoader(); | ||
| 980 | + const fs::path so = Path("limit.so"); | ||
| 981 | + WriteFile(so); | ||
| 982 | + | ||
| 983 | + StageState stage; | ||
| 984 | + stage.external.bytes = TEST_MAX_REGISTRY_RESOURCE_SIZE; | ||
| 985 | + EXPECT_EQ( | ||
| 986 | + ResourceRegistry::LoadLibraries({{so.string(), ResourceSourceType::External}}, stage), | ||
| 987 | + ResourceStatus::InvalidResource); | ||
| 836 | EXPECT_EQ(gDlopenCalls, 1U); | 988 | EXPECT_EQ(gDlopenCalls, 1U); |
| 837 | EXPECT_TRUE(stage.external.discovered); | 989 | EXPECT_TRUE(stage.external.discovered); |
| 838 | - EXPECT_FALSE(stage.builtIn.discovered); | 990 | + EXPECT_TRUE(stage.external.resources.empty()); |
| 991 | + EXPECT_EQ(stage.external.bytes, TEST_MAX_REGISTRY_RESOURCE_SIZE); | ||
| 992 | + EXPECT_EQ(stage.external.files, 0U); | ||
| 839 | } | 993 | } |
| 840 | 994 | ||
| 841 | TEST_F(ResourceRegistryTest, LoadLibrariesLoadsValidList) | 995 | TEST_F(ResourceRegistryTest, LoadLibrariesLoadsValidList) |
| @@ -931,10 +1085,22 @@ TEST_F(ResourceRegistryTest, WriteMaterializedFilesWritesNestedAndEmptyPayloadsA | |||
| 931 | {"empty.bin", "empty.bin", {}}, | 1085 | {"empty.bin", "empty.bin", {}}, |
| 932 | }; | 1086 | }; |
| 933 | const fs::path output = Path("materialized"); | 1087 | const fs::path output = Path("materialized"); |
| 934 | - EXPECT_EQ(ResourceRegistry::WriteMaterializedFiles(files, output.string()), ResourceStatus::Success); | 1088 | + const std::string nonCanonicalOutput = (output / ".." / output.filename()).string(); |
| 1089 | + EXPECT_EQ(ResourceRegistry::WriteMaterializedFiles(files, nonCanonicalOutput), ResourceStatus::Success); | ||
| 935 | EXPECT_EQ(ReadFile(output / "nested/data.bin"), std::string("\1\2\3", 3U)); | 1090 | EXPECT_EQ(ReadFile(output / "nested/data.bin"), std::string("\1\2\3", 3U)); |
| 936 | EXPECT_TRUE(ReadFile(output / "empty.bin").empty()); | 1091 | EXPECT_TRUE(ReadFile(output / "empty.bin").empty()); |
| 937 | 1092 | ||
| 1093 | + const fs::path redirectedRoot = Path("redirected-root"); | ||
| 1094 | + const fs::path redirectedTarget = Path("redirected-target"); | ||
| 1095 | + MakeDirectory(redirectedRoot); | ||
| 1096 | + MakeDirectory(redirectedTarget); | ||
| 1097 | + boost::system::error_code error; | ||
| 1098 | + fs::create_directory_symlink(redirectedTarget, redirectedRoot / "redirect", error); | ||
| 1099 | + ASSERT_FALSE(error); | ||
| 1100 | + const std::vector<ResourceFileData> redirected = {{"data.bin", "redirect/data.bin", {1U}}}; | ||
| 1101 | + EXPECT_EQ(ResourceRegistry::WriteMaterializedFiles(redirected, redirectedRoot.string()), ResourceStatus::IoError); | ||
| 1102 | + EXPECT_FALSE(fs::exists(redirectedTarget / "data.bin")); | ||
| 1103 | + | ||
| 938 | const fs::path blockedRoot = Path("blocked-root"); | 1104 | const fs::path blockedRoot = Path("blocked-root"); |
| 939 | WriteFile(blockedRoot, "file"); | 1105 | WriteFile(blockedRoot, "file"); |
| 940 | EXPECT_EQ(ResourceRegistry::WriteMaterializedFiles(files, blockedRoot.string()), ResourceStatus::IoError); | 1106 | EXPECT_EQ(ResourceRegistry::WriteMaterializedFiles(files, blockedRoot.string()), ResourceStatus::IoError); |
| @@ -1082,7 +1248,7 @@ TEST_F(ResourceRegistryTest, LoadReportsDiscoveryAndDynamicLoaderFailures) | |||
| 1082 | const fs::path empty = Path("load-empty"); | 1248 | const fs::path empty = Path("load-empty"); |
| 1083 | MakeDirectory(empty); | 1249 | MakeDirectory(empty); |
| 1084 | EXPECT_EQ(registry_->Load(Path("load-missing").string().c_str()), ResourceStatus::InvalidResource); | 1250 | EXPECT_EQ(registry_->Load(Path("load-missing").string().c_str()), ResourceStatus::InvalidResource); |
| 1085 | - EXPECT_EQ(registry_->Load(empty.string().c_str()), ResourceStatus::NotFound); | 1251 | + EXPECT_EQ(registry_->Load(empty.string().c_str()), ResourceStatus::InvalidResource); |
| 1086 | 1252 | ||
| 1087 | const fs::path so = Path("load/libresource_compile_database.so"); | 1253 | const fs::path so = Path("load/libresource_compile_database.so"); |
| 1088 | WriteFile(so); | 1254 | WriteFile(so); |
| @@ -466,26 +466,24 @@ ResourceStatus PrepareMaterializationRoot( | |||
| 466 | return ResourceStatus::Success; | 466 | return ResourceStatus::Success; |
| 467 | } | 467 | } |
| 468 | 468 | ||
| 469 | -ResourceStatus ResolveExplicitDiscoveryPath(const char* path, bool& isFile, std::string& canonical) | 469 | +ResourceStatus ResolveExplicitDiscoveryPath(const char* path, std::string& canonical) |
| 470 | { | 470 | { |
| 471 | - isFile = FileUtils::IsRegularFile(path); | ||
| 472 | - if (isFile) { | ||
| 473 | - if (FileUtils::IsSymlink(path)) { | ||
| 474 | - ASCENDLOGE("Explicit compile resource SO must not be a symbolic link: path=%s expected=regular file", path); | ||
| 475 | - return ResourceStatus::InvalidResource; | ||
| 476 | - } | ||
| 477 | - if (!FileUtils::ResolveCanonicalPath(path, canonical)) { | ||
| 478 | - ASCENDLOGE("Failed to resolve explicit compile resource SO: path=%s", path); | ||
| 479 | - return ResourceStatus::InvalidResource; | ||
| 480 | - } | ||
| 481 | - return ResourceStatus::Success; | ||
| 482 | - } | ||
| 483 | if (FileUtils::IsSymlink(path)) { | 471 | if (FileUtils::IsSymlink(path)) { |
| 484 | - ASCENDLOGE("Explicit compile resource directory must not be a symbolic link: path=%s expected=directory", path); | 472 | + ASCENDLOGE("Explicit compile resource SO must not be a symbolic link: path=%s expected=regular file", path); |
| 485 | return ResourceStatus::InvalidResource; | 473 | return ResourceStatus::InvalidResource; |
| 486 | } | 474 | } |
| 487 | - if (!FileUtils::IsDirectory(path)) { | 475 | + if (!FileUtils::IsRegularFile(path)) { |
| 488 | - ASCENDLOGE("Explicit compile resource path is neither a regular file nor a directory: path=%s", path); | 476 | + if (FileUtils::IsDirectory(path)) { |
| 477 | + ASCENDLOGE( | ||
| 478 | + "Explicit compile resource path must be a regular SO file; directory input is unsupported: path=%s", | ||
| 479 | + path); | ||
| 480 | + } else { | ||
| 481 | + ASCENDLOGE("Explicit compile resource path is not a regular SO file: path=%s", path); | ||
| 482 | + } | ||
| 483 | + return ResourceStatus::InvalidResource; | ||
| 484 | + } | ||
| 485 | + if (!FileUtils::ResolveCanonicalPath(path, canonical)) { | ||
| 486 | + ASCENDLOGE("Failed to resolve explicit compile resource SO: path=%s", path); | ||
| 489 | return ResourceStatus::InvalidResource; | 487 | return ResourceStatus::InvalidResource; |
| 490 | } | 488 | } |
| 491 | return ResourceStatus::Success; | 489 | return ResourceStatus::Success; |
| @@ -614,6 +612,37 @@ bool CheckRegistryLimits(const StagedResources& staged, ResourceSourceType sourc | |||
| 614 | return true; | 612 | return true; |
| 615 | } | 613 | } |
| 616 | 614 | ||
| 615 | +ResourceStatus MergeStagedLibrary(const LibrarySpec& spec, StagedResources& incoming, StagedResources& staged) | ||
| 616 | +{ | ||
| 617 | + for (const auto& item : incoming.resources) { | ||
| 618 | + const auto existing = staged.resources.find(item.first); | ||
| 619 | + if (existing != staged.resources.end()) { | ||
| 620 | + ASCENDLOGW( | ||
| 621 | + "Skipping compile resource SO because its resource conflicts with an earlier SO: resource_id=%s " | ||
| 622 | + "source_type=%s incoming_so=%s existing_so=%s", | ||
| 623 | + item.first.c_str(), SourceTypeName(spec.sourceType), spec.path.c_str(), | ||
| 624 | + existing->second->sourceSoPath.c_str()); | ||
| 625 | + return ResourceStatus::Conflict; | ||
| 626 | + } | ||
| 627 | + } | ||
| 628 | + uint64_t bytes = staged.bytes; | ||
| 629 | + uint64_t files = staged.files; | ||
| 630 | + if (!TryAddWithinLimit(incoming.bytes, MAX_REGISTRY_RESOURCE_SIZE, bytes) || | ||
| 631 | + !TryAddWithinLimit(incoming.files, MAX_REGISTRY_FILE_COUNT, files)) { | ||
| 632 | + ASCENDLOGW( | ||
| 633 | + "Skipping compile resource SO because cumulative staged resources exceed the registry limit: " | ||
| 634 | + "source_type=%s so=%s current_bytes=%" PRIu64 " incoming_bytes=%" PRIu64 " current_files=%" PRIu64 | ||
| 635 | + " incoming_files=%" PRIu64, | ||
| 636 | + SourceTypeName(spec.sourceType), spec.path.c_str(), staged.bytes, incoming.bytes, staged.files, | ||
| 637 | + incoming.files); | ||
| 638 | + return ResourceStatus::InvalidResource; | ||
| 639 | + } | ||
| 640 | + staged.resources.merge(incoming.resources); | ||
| 641 | + staged.bytes = bytes; | ||
| 642 | + staged.files = files; | ||
| 643 | + return ResourceStatus::Success; | ||
| 644 | +} | ||
| 645 | + | ||
| 617 | } // namespace | 646 | } // namespace |
| 618 | 647 | ||
| 619 | AutomaticRoots ResourceRegistry::AutomaticSearchRoots() | 648 | AutomaticRoots ResourceRegistry::AutomaticSearchRoots() |
| @@ -630,32 +659,46 @@ AutomaticRoots ResourceRegistry::AutomaticSearchRoots() | |||
| 630 | roots.custom.size()); | 659 | roots.custom.size()); |
| 631 | return roots; | 660 | return roots; |
| 632 | } | 661 | } |
| 633 | - const std::string oppRoot(environment); | 662 | + std::string oppRoot; |
| 634 | - const std::string vendorConfig = FileUtils::JoinPath(oppRoot, "vendors/config.ini"); | 663 | + if (!FileUtils::ResolveCanonicalPath(environment, oppRoot)) { |
| 635 | - std::ifstream input(vendorConfig); | ||
| 636 | - if (!input.is_open()) { | ||
| 637 | ASCENDLOGW( | 664 | ASCENDLOGW( |
| 638 | - "Unable to read optional compile resource vendor configuration: path=%s reason=open failed; " | 665 | + "Unable to normalize configured OPP root: path=%s; vendor and built-in discovery will be skipped", |
| 666 | + environment); | ||
| 667 | + return roots; | ||
| 668 | + } | ||
| 669 | + const std::string vendorConfig = FileUtils::JoinPath(oppRoot, "vendors/config.ini"); | ||
| 670 | + std::string canonicalVendorConfig; | ||
| 671 | + if (!FileUtils::ResolveCanonicalPath(vendorConfig, canonicalVendorConfig)) { | ||
| 672 | + ASCENDLOGW( | ||
| 673 | + "Unable to normalize optional compile resource vendor configuration: path=%s; " | ||
| 639 | "built-in discovery will continue", | 674 | "built-in discovery will continue", |
| 640 | vendorConfig.c_str()); | 675 | vendorConfig.c_str()); |
| 641 | - } | 676 | + } else { |
| 642 | - std::string line; | 677 | + std::ifstream input(canonicalVendorConfig); |
| 643 | - while (std::getline(input, line)) { | 678 | + if (!input.is_open()) { |
| 644 | - const size_t separator = line.find('='); | 679 | + ASCENDLOGW( |
| 645 | - if (separator == std::string::npos || Trim(line.substr(0U, separator)) != "load_priority") { | 680 | + "Unable to read optional compile resource vendor configuration: path=%s reason=open failed; " |
| 646 | - continue; | 681 | + "built-in discovery will continue", |
| 682 | + canonicalVendorConfig.c_str()); | ||
| 647 | } | 683 | } |
| 648 | - for (const std::string& vendor : SplitList(line.substr(separator + 1U).c_str(), ',')) { | 684 | + std::string line; |
| 649 | - if (IsPlainName(vendor)) { | 685 | + while (std::getline(input, line)) { |
| 650 | - roots.custom.push_back( | 686 | + const size_t separator = line.find('='); |
| 651 | - FileUtils::JoinPath(FileUtils::JoinPath(oppRoot, "vendors/" + vendor), RELATIVE_JIT_ROOT)); | 687 | + if (separator == std::string::npos || Trim(line.substr(0U, separator)) != "load_priority") { |
| 652 | - } else { | 688 | + continue; |
| 653 | - ASCENDLOGW( | ||
| 654 | - "Ignoring unsafe compile resource vendor name: path=%s vendor=%s expected=plain directory name", | ||
| 655 | - vendorConfig.c_str(), vendor.c_str()); | ||
| 656 | } | 689 | } |
| 690 | + for (const std::string& vendor : SplitList(line.substr(separator + 1U).c_str(), ',')) { | ||
| 691 | + if (IsPlainName(vendor)) { | ||
| 692 | + roots.custom.push_back( | ||
| 693 | + FileUtils::JoinPath(FileUtils::JoinPath(oppRoot, "vendors/" + vendor), RELATIVE_JIT_ROOT)); | ||
| 694 | + } else { | ||
| 695 | + ASCENDLOGW( | ||
| 696 | + "Ignoring unsafe compile resource vendor name: path=%s vendor=%s expected=plain directory name", | ||
| 697 | + canonicalVendorConfig.c_str(), vendor.c_str()); | ||
| 698 | + } | ||
| 699 | + } | ||
| 700 | + break; | ||
| 657 | } | 701 | } |
| 658 | - break; | ||
| 659 | } | 702 | } |
| 660 | roots.builtIn.push_back(FileUtils::JoinPath(FileUtils::JoinPath(oppRoot, "built-in"), RELATIVE_JIT_ROOT)); | 703 | roots.builtIn.push_back(FileUtils::JoinPath(FileUtils::JoinPath(oppRoot, "built-in"), RELATIVE_JIT_ROOT)); |
| 661 | ASCENDLOGD( | 704 | ASCENDLOGD( |
| @@ -758,32 +801,13 @@ ResourceStatus ResourceRegistry::DiscoverLibraries(const char* directory, std::v | |||
| 758 | "Discovering compile resource libraries: mode=%s path=%s", automatic ? "automatic" : "explicit", | 801 | "Discovering compile resource libraries: mode=%s path=%s", automatic ? "automatic" : "explicit", |
| 759 | automatic ? "<environment>" : directory); | 802 | automatic ? "<environment>" : directory); |
| 760 | if (!automatic) { | 803 | if (!automatic) { |
| 761 | - bool isFile = false; | ||
| 762 | std::string canonical; | 804 | std::string canonical; |
| 763 | - const ResourceStatus status = ResolveExplicitDiscoveryPath(directory, isFile, canonical); | 805 | + const ResourceStatus status = ResolveExplicitDiscoveryPath(directory, canonical); |
| 764 | if (status != ResourceStatus::Success) { | 806 | if (status != ResourceStatus::Success) { |
| 765 | return status; | 807 | return status; |
| 766 | } | 808 | } |
| 767 | - if (isFile) { | 809 | + libraries.push_back({canonical, ResourceSourceType::External}); |
| 768 | - libraries.push_back({canonical, ResourceSourceType::External}); | 810 | + ASCENDLOGI("Discovered explicit compile resource SO: path=%s", canonical.c_str()); |
| 769 | - ASCENDLOGI("Discovered explicit compile resource SO: path=%s", canonical.c_str()); | ||
| 770 | - return ResourceStatus::Success; | ||
| 771 | - } | ||
| 772 | - std::set<std::string> found; | ||
| 773 | - if (!CollectLibraries(directory, ResourceSourceType::External, false, found)) { | ||
| 774 | - return ResourceStatus::IoError; | ||
| 775 | - } | ||
| 776 | - for (const std::string& path : found) { | ||
| 777 | - libraries.push_back({path, ResourceSourceType::External}); | ||
| 778 | - } | ||
| 779 | - if (libraries.empty()) { | ||
| 780 | - ASCENDLOGW( | ||
| 781 | - "No compile resource libraries found in explicit directory: path=%s pattern=lib*%s recursive=false", | ||
| 782 | - directory, LIBRARY_NAME_SUFFIX); | ||
| 783 | - return ResourceStatus::NotFound; | ||
| 784 | - } | ||
| 785 | - ASCENDLOGI( | ||
| 786 | - "Discovered compile resource libraries: mode=explicit path=%s count=%zu", directory, libraries.size()); | ||
| 787 | return ResourceStatus::Success; | 811 | return ResourceStatus::Success; |
| 788 | } | 812 | } |
| 789 | const AutomaticRoots roots = AutomaticSearchRoots(); | 813 | const AutomaticRoots roots = AutomaticSearchRoots(); |
| @@ -1097,52 +1121,83 @@ ResourceStatus ResourceRegistry::LoadManifest( | |||
| 1097 | 1121 | ||
| 1098 | ResourceStatus ResourceRegistry::LoadLibrary(const LibrarySpec& spec, StageState& stage) | 1122 | ResourceStatus ResourceRegistry::LoadLibrary(const LibrarySpec& spec, StageState& stage) |
| 1099 | { | 1123 | { |
| 1100 | - ASCENDLOGI("Loading compile resource SO: source_type=%s so=%s", SourceTypeName(spec.sourceType), spec.path.c_str()); | 1124 | + LibrarySpec canonicalSpec = spec; |
| 1101 | - LibraryHandle library(dlopen(spec.path.c_str(), RTLD_NOW | RTLD_LOCAL)); | 1125 | + if (!FileUtils::ResolveCanonicalPath(spec.path, canonicalSpec.path)) { |
| 1126 | + ASCENDLOGE( | ||
| 1127 | + "Failed to normalize compile resource SO path before loading: source_type=%s so=%s", | ||
| 1128 | + SourceTypeName(spec.sourceType), spec.path.c_str()); | ||
| 1129 | + return ResourceStatus::InvalidResource; | ||
| 1130 | + } | ||
| 1131 | + ASCENDLOGI( | ||
| 1132 | + "Loading compile resource SO: source_type=%s so=%s", SourceTypeName(canonicalSpec.sourceType), | ||
| 1133 | + canonicalSpec.path.c_str()); | ||
| 1134 | + LibraryHandle library(dlopen(canonicalSpec.path.c_str(), RTLD_NOW | RTLD_LOCAL)); | ||
| 1102 | if (!library) { | 1135 | if (!library) { |
| 1103 | const char* error = dlerror(); | 1136 | const char* error = dlerror(); |
| 1104 | ASCENDLOGE( | 1137 | ASCENDLOGE( |
| 1105 | - "Failed to open compile resource SO: source_type=%s so=%s error=%s", SourceTypeName(spec.sourceType), | 1138 | + "Failed to open compile resource SO: source_type=%s so=%s error=%s", |
| 1106 | - spec.path.c_str(), error == nullptr ? "unknown" : error); | 1139 | + SourceTypeName(canonicalSpec.sourceType), canonicalSpec.path.c_str(), error == nullptr ? "unknown" : error); |
| 1107 | return ResourceStatus::LoadError; | 1140 | return ResourceStatus::LoadError; |
| 1108 | } | 1141 | } |
| 1109 | const AcCompileResourceBundle* bundle = nullptr; | 1142 | const AcCompileResourceBundle* bundle = nullptr; |
| 1110 | - ResourceStatus status = GetBundle(library, spec, bundle); | 1143 | + ResourceStatus status = GetBundle(library, canonicalSpec, bundle); |
| 1111 | if (status != ResourceStatus::Success) { | 1144 | if (status != ResourceStatus::Success) { |
| 1112 | return status; | 1145 | return status; |
| 1113 | } | 1146 | } |
| 1114 | for (uint64_t manifestIndex = 0U; manifestIndex < bundle->manifestCount; ++manifestIndex) { | 1147 | for (uint64_t manifestIndex = 0U; manifestIndex < bundle->manifestCount; ++manifestIndex) { |
| 1115 | - status = LoadManifest(bundle->manifests[manifestIndex], spec, manifestIndex, stage); | 1148 | + status = LoadManifest(bundle->manifests[manifestIndex], canonicalSpec, manifestIndex, stage); |
| 1116 | if (status != ResourceStatus::Success) { | 1149 | if (status != ResourceStatus::Success) { |
| 1117 | ASCENDLOGE( | 1150 | ASCENDLOGE( |
| 1118 | "Stopped loading compile resource SO at manifest: source_type=%s so=%s manifest=%" PRIu64 | 1151 | "Stopped loading compile resource SO at manifest: source_type=%s so=%s manifest=%" PRIu64 |
| 1119 | " status=%s(%d)", | 1152 | " status=%s(%d)", |
| 1120 | - SourceTypeName(spec.sourceType), spec.path.c_str(), manifestIndex, ResourceStatusName(status), | 1153 | + SourceTypeName(canonicalSpec.sourceType), canonicalSpec.path.c_str(), manifestIndex, |
| 1121 | - static_cast<int>(status)); | 1154 | + ResourceStatusName(status), static_cast<int>(status)); |
| 1122 | return status; | 1155 | return status; |
| 1123 | } | 1156 | } |
| 1124 | } | 1157 | } |
| 1125 | ASCENDLOGI( | 1158 | ASCENDLOGI( |
| 1126 | - "Loaded compile resource SO: source_type=%s so=%s manifests=%" PRIu64, SourceTypeName(spec.sourceType), | 1159 | + "Loaded compile resource SO: source_type=%s so=%s manifests=%" PRIu64, SourceTypeName(canonicalSpec.sourceType), |
| 1127 | - spec.path.c_str(), bundle->manifestCount); | 1160 | + canonicalSpec.path.c_str(), bundle->manifestCount); |
| 1128 | return ResourceStatus::Success; | 1161 | return ResourceStatus::Success; |
| 1129 | } | 1162 | } |
| 1130 | 1163 | ||
| 1131 | ResourceStatus ResourceRegistry::LoadLibraries(const std::vector<LibrarySpec>& libraries, StageState& stage) | 1164 | ResourceStatus ResourceRegistry::LoadLibraries(const std::vector<LibrarySpec>& libraries, StageState& stage) |
| 1132 | { | 1165 | { |
| 1133 | ASCENDLOGI("Loading compile resource SO list: count=%zu", libraries.size()); | 1166 | ASCENDLOGI("Loading compile resource SO list: count=%zu", libraries.size()); |
| 1167 | + size_t loaded = 0U; | ||
| 1168 | + size_t skipped = 0U; | ||
| 1169 | + ResourceStatus firstFailure = ResourceStatus::Success; | ||
| 1134 | for (const LibrarySpec& library : libraries) { | 1170 | for (const LibrarySpec& library : libraries) { |
| 1135 | - SelectStagedResources(stage, library.sourceType).discovered = true; | 1171 | + StagedResources& staged = SelectStagedResources(stage, library.sourceType); |
| 1136 | - const ResourceStatus status = LoadLibrary(library, stage); | 1172 | + staged.discovered = true; |
| 1173 | + StageState libraryStage; | ||
| 1174 | + StagedResources& incoming = SelectStagedResources(libraryStage, library.sourceType); | ||
| 1175 | + incoming.discovered = true; | ||
| 1176 | + ResourceStatus status = LoadLibrary(library, libraryStage); | ||
| 1177 | + if (status == ResourceStatus::Success) { | ||
| 1178 | + status = MergeStagedLibrary(library, incoming, staged); | ||
| 1179 | + } | ||
| 1137 | if (status != ResourceStatus::Success) { | 1180 | if (status != ResourceStatus::Success) { |
| 1138 | - ASCENDLOGE( | 1181 | + if (firstFailure == ResourceStatus::Success) { |
| 1139 | - "Stopped loading compile resource SO list: source_type=%s so=%s status=%s(%d)", | 1182 | + firstFailure = status; |
| 1183 | + } | ||
| 1184 | + ++skipped; | ||
| 1185 | + ASCENDLOGW( | ||
| 1186 | + "Skipping failed compile resource SO and continuing with the next SO: source_type=%s so=%s " | ||
| 1187 | + "status=%s(%d)", | ||
| 1140 | SourceTypeName(library.sourceType), library.path.c_str(), ResourceStatusName(status), | 1188 | SourceTypeName(library.sourceType), library.path.c_str(), ResourceStatusName(status), |
| 1141 | static_cast<int>(status)); | 1189 | static_cast<int>(status)); |
| 1142 | - return status; | 1190 | + continue; |
| 1143 | } | 1191 | } |
| 1192 | + ++loaded; | ||
| 1144 | } | 1193 | } |
| 1145 | - ASCENDLOGI("Loaded compile resource SO list: count=%zu", libraries.size()); | 1194 | + if (loaded == 0U && !libraries.empty()) { |
| 1195 | + ASCENDLOGW( | ||
| 1196 | + "No compile resource SO was loaded successfully: total=%zu skipped=%zu first_failure=%s(%d)", | ||
| 1197 | + libraries.size(), skipped, ResourceStatusName(firstFailure), static_cast<int>(firstFailure)); | ||
| 1198 | + return firstFailure; | ||
| 1199 | + } | ||
| 1200 | + ASCENDLOGI("Loaded compile resource SO list: total=%zu loaded=%zu skipped=%zu", libraries.size(), loaded, skipped); | ||
| 1146 | return ResourceStatus::Success; | 1201 | return ResourceStatus::Success; |
| 1147 | } | 1202 | } |
| 1148 | 1203 | ||
| @@ -1160,13 +1215,30 @@ ResourceStatus ResourceRegistry::WriteMaterializedFiles( | |||
| 1160 | root.c_str(), file.relativePath.c_str(), parent.c_str()); | 1215 | root.c_str(), file.relativePath.c_str(), parent.c_str()); |
| 1161 | return ResourceStatus::IoError; | 1216 | return ResourceStatus::IoError; |
| 1162 | } | 1217 | } |
| 1218 | + std::string canonicalRoot; | ||
| 1219 | + std::string canonicalParent; | ||
| 1220 | + if (!FileUtils::ResolveCanonicalPath(root, canonicalRoot) || | ||
| 1221 | + !FileUtils::ResolveCanonicalPath(parent, canonicalParent)) { | ||
| 1222 | + ASCENDLOGE( | ||
| 1223 | + "Failed to normalize materialized compile resource output path: root=%s relative_path=%s parent=%s", | ||
| 1224 | + root.c_str(), file.relativePath.c_str(), parent.c_str()); | ||
| 1225 | + return ResourceStatus::IoError; | ||
| 1226 | + } | ||
| 1227 | + if (!FileUtils::IsPathWithin(canonicalParent, canonicalRoot)) { | ||
| 1228 | + ASCENDLOGE( | ||
| 1229 | + "Rejected materialized compile resource output outside its root: root=%s resolved_root=%s " | ||
| 1230 | + "relative_path=%s resolved_parent=%s", | ||
| 1231 | + root.c_str(), canonicalRoot.c_str(), file.relativePath.c_str(), canonicalParent.c_str()); | ||
| 1232 | + return ResourceStatus::IoError; | ||
| 1233 | + } | ||
| 1234 | + const std::string canonicalPath = FileUtils::JoinPath(canonicalParent, FileUtils::FileName(file.relativePath)); | ||
| 1163 | errno = 0; | 1235 | errno = 0; |
| 1164 | - std::ofstream output(path.c_str(), std::ios::binary | std::ios::trunc); | 1236 | + std::ofstream output(canonicalPath.c_str(), std::ios::binary | std::ios::trunc); |
| 1165 | if (!output.is_open()) { | 1237 | if (!output.is_open()) { |
| 1166 | const int openError = errno; | 1238 | const int openError = errno; |
| 1167 | ASCENDLOGE( | 1239 | ASCENDLOGE( |
| 1168 | "Failed to open materialized compile resource file: root=%s relative_path=%s path=%s error=%s", | 1240 | "Failed to open materialized compile resource file: root=%s relative_path=%s path=%s error=%s", |
| 1169 | - root.c_str(), file.relativePath.c_str(), path.c_str(), | 1241 | + root.c_str(), file.relativePath.c_str(), canonicalPath.c_str(), |
| 1170 | openError == 0 ? "stream open failed" : std::strerror(openError)); | 1242 | openError == 0 ? "stream open failed" : std::strerror(openError)); |
| 1171 | return ResourceStatus::IoError; | 1243 | return ResourceStatus::IoError; |
| 1172 | } | 1244 | } |
| @@ -1178,7 +1250,7 @@ ResourceStatus ResourceRegistry::WriteMaterializedFiles( | |||
| 1178 | ASCENDLOGE( | 1250 | ASCENDLOGE( |
| 1179 | "Failed to finalize materialized compile resource file: root=%s relative_path=%s path=%s " | 1251 | "Failed to finalize materialized compile resource file: root=%s relative_path=%s path=%s " |
| 1180 | "reason=write, flush, or close failed", | 1252 | "reason=write, flush, or close failed", |
| 1181 | - root.c_str(), file.relativePath.c_str(), path.c_str()); | 1253 | + root.c_str(), file.relativePath.c_str(), canonicalPath.c_str()); |
| 1182 | return ResourceStatus::IoError; | 1254 | return ResourceStatus::IoError; |
| 1183 | } | 1255 | } |
| 1184 | } | 1256 | } |
| @@ -1215,7 +1287,7 @@ ResourceEntry* ResourceRegistry::FindResource(const std::string& resourceId) noe | |||
| 1215 | return builtIn == builtInResources_.end() ? nullptr : builtIn->second.get(); | 1287 | return builtIn == builtInResources_.end() ? nullptr : builtIn->second.get(); |
| 1216 | } | 1288 | } |
| 1217 | 1289 | ||
| 1218 | -bool ResourceRegistry::HasCommitConflict(const ResourceStore& incoming, const ResourceStore& committed) | 1290 | +bool ResourceRegistry::HasCommitConflict(const ResourceStore& incoming, const ResourceStore& committed) const |
| 1219 | { | 1291 | { |
| 1220 | for (const auto& item : incoming) { | 1292 | for (const auto& item : incoming) { |
| 1221 | const auto existing = committed.find(item.first); | 1293 | const auto existing = committed.find(item.first); |
| @@ -1239,7 +1311,8 @@ ResourceStatus ResourceRegistry::Commit(StageState& stage) | |||
| 1239 | "Committing compile resources: external=%zu custom=%zu built_in=%zu", stage.external.resources.size(), | 1311 | "Committing compile resources: external=%zu custom=%zu built_in=%zu", stage.external.resources.size(), |
| 1240 | stage.custom.resources.size(), stage.builtIn.resources.size()); | 1312 | stage.custom.resources.size(), stage.builtIn.resources.size()); |
| 1241 | ResourceStatus status = ResourceStatus::Success; | 1313 | ResourceStatus status = ResourceStatus::Success; |
| 1242 | - auto checkCategory = [&](StagedResources& staged, ResourceStore& committed, ResourceSourceType sourceType) { | 1314 | + auto checkCategory = [this, &status]( |
| 1315 | + StagedResources& staged, ResourceStore& committed, ResourceSourceType sourceType) { | ||
| 1243 | if (!staged.discovered && !staged.conflict) { | 1316 | if (!staged.discovered && !staged.conflict) { |
| 1244 | return; | 1317 | return; |
| 1245 | } | 1318 | } |
| @@ -1266,7 +1339,7 @@ ResourceStatus ResourceRegistry::Commit(StageState& stage) | |||
| 1266 | !CheckRegistryLimits(stage.builtIn, ResourceSourceType::BuiltIn, bytes, files)) { | 1339 | !CheckRegistryLimits(stage.builtIn, ResourceSourceType::BuiltIn, bytes, files)) { |
| 1267 | return ResourceStatus::InvalidResource; | 1340 | return ResourceStatus::InvalidResource; |
| 1268 | } | 1341 | } |
| 1269 | - auto commitCategory = [&](StagedResources& staged, ResourceStore& committed) { | 1342 | + auto commitCategory = [](StagedResources& staged, ResourceStore& committed) { |
| 1270 | if (!staged.discovered || staged.conflict) { | 1343 | if (!staged.discovered || staged.conflict) { |
| 1271 | return; | 1344 | return; |
| 1272 | } | 1345 | } |
| @@ -95,9 +95,9 @@ public: | |||
| 95 | static ResourceRegistry& Instance(); | 95 | static ResourceRegistry& Instance(); |
| 96 | 96 | ||
| 97 | /** | 97 | /** |
| 98 | - * @brief Loads compile resources from an explicit path or the configured OPP search paths. | 98 | + * @brief Loads compile resources from an explicit shared object or the configured OPP search paths. |
| 99 | - * @param[in] directory Resource shared object or directory to load. Pass nullptr or an empty string to use | 99 | + * @param[in] directory Resource shared object file to load. Pass nullptr or an empty string to use automatic OPP |
| 100 | - * automatic OPP discovery. | 100 | + * discovery. Directory input is not supported for explicit loading. |
| 101 | * @return ResourceStatus::Success on success; otherwise, a status describing the discovery, validation, loading, | 101 | * @return ResourceStatus::Success on success; otherwise, a status describing the discovery, validation, loading, |
| 102 | * I/O, or conflict failure. | 102 | * I/O, or conflict failure. |
| 103 | */ | 103 | */ |
| @@ -153,7 +153,7 @@ private: | |||
| 153 | ResourceEntry* FindResource(const std::string& resourceId) noexcept; | 153 | ResourceEntry* FindResource(const std::string& resourceId) noexcept; |
| 154 | ResourceStatus Materialize(const std::string& resourceId, const ResourceEntry& entry, ResourceData& resource); | 154 | ResourceStatus Materialize(const std::string& resourceId, const ResourceEntry& entry, ResourceData& resource); |
| 155 | ResourceStatus Commit(StageState& stage); | 155 | ResourceStatus Commit(StageState& stage); |
| 156 | - bool HasCommitConflict(const ResourceStore& incoming, const ResourceStore& committed); | 156 | + bool HasCommitConflict(const ResourceStore& incoming, const ResourceStore& committed) const; |
| 157 | 157 | ||
| 158 | std::string temporaryRoot_; | 158 | std::string temporaryRoot_; |
| 159 | ResourceStore externalResources_; | 159 | ResourceStore externalResources_; |