8000 [2/N] Use std::filesystem by cyyever · Pull Request #152586 · pytorch/pytorch · GitHub
[go: up one dir, main page]
Skip to content

[2/N] Use std::filesystem #152586

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

[2/N] Use std::filesystem #152586

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cdd239a
Use std::filesystem
cyyever Apr 30, 2025
20972d0
Fix headers
cyyever May 6, 2025
15352cc
Improve fs::create_directories check
cyyever May 7, 2025
d2f2e86
Fix headers
cyyever May 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion torch/csrc/distributed/c10d/UCCTracing.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ void ProcessGroupUCCLogger::flushComms(int rank, int world_size) {
std::filesystem::path trace_filename =
fullpath / fmt::format("rank{}.json", rank);
std::error_code ec{};
if (!std::filesystem::create_directories(fullpath, ec)) {
if (!std::filesystem::create_directories(fullpath, ec) &&
!std::filesystem::is_directory(fullpath)) {
LOG(INFO) << getLogPrefix() << "[INFO] failed to mkdir " << fullpath
<< " with error " << ec.message();
return;
Expand Down
18 changes: 1 addition & 17 deletions torch/csrc/inductor/aoti_runner/model_container_runner.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,7 @@
#include <torch/csrc/inductor/aoti_torch/oss_proxy_executor.h>
#include <torch/csrc/inductor/aoti_torch/tensor_converter.h>

#ifndef _WIN32
#include <sys/stat.h>
#else
#include <filesystem>
namespace fs = std::filesystem;
#endif

namespace {
bool file_exists(std::string& path) {
#ifdef _WIN32
return fs::exists(path);
#else
struct stat rc {};
return lstat(path.c_str(), &rc) == 0;
#endif
}
} // namespace

namespace torch::inductor {

Expand Down Expand Up @@ -110,7 +94,7 @@ consider rebuild your model with the latest AOTInductor.");
size_t lastindex = model_so_path.find_last_of('.');
std::string json_filename = model_so_path.substr(0, lastindex) + ".json";

if (file_exists(json_filename)) {
if (std::filesystem::exists(json_filename)) {
proxy_executor_ = std::make_unique<torch::aot_inductor::OSSProxyExecutor>(
json_filename, device_str == "cpu");
proxy_executor_handle_ =
Expand Down
3 changes: 2 additions & 1 deletion
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include <c10/xpu/XPUFunctions.h>
#include <level_zero/ze_api.h>
#include <sycl/sycl.hpp>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <string>
Expand Down Expand Up @@ -78,7 +79,7 @@ static ze_module_handle_t loadModule(std::string& spv_path) {
auto l0_context =
sycl::get_native<sycl::backend::ext_oneapi_level_zero>(sycl_context);

std::ifstream IFS(spv_path.c_str(), std::ios::binary);
std::ifstream IFS(spv_path, std::ios::binary);
std::ostringstream OSS;
OSS << IFS.rdbuf();
std::string data(OSS.str());
Expand Down
16 changes: 4 additions & 12 deletions torch/csrc/inductor/aoti_torch/oss_proxy_executor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include <nlohmann/json.hpp>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <vector>
Expand All @@ -18,12 +19,6 @@ bool has_key(
return map.find(key) != map.end();
}

#ifdef _WIN32
const std::string k_separator = "\\";
#else
const std::string k_separator = "/";
#endif

} // namespace

namespace torch::aot_inductor {
Expand Down Expand Up @@ -597,10 +592,8 @@ OSSProxyExecutor::OSSProxyExecutor(
// Load custom objects from custom_objs_config.json file
// Get the constants json path from the extern_kernel_nodes .json file

size_t lastSlash = json_path.find_last_of("/\\");
std::string folder_path = json_path.substr(0, lastSlash);
std::string custom_objs_json_path =
folder_path + k_separator + "custom_objs_config.json";
auto folder_path = std::filesystem::path(json_path).parent_path();
auto custom_objs_json_path = folder_path / "custom_objs_config.json";
LOG(INFO) << "Loading custom_objs_config .json file from "
<< custom_objs_json_path;

Expand All @@ -615,8 +608,7 @@ OSSProxyExecutor::OSSProxyExecutor(
custom_objs_json_file >> custom_objs_json;
// Load custom objects from binary torchbind file
for (auto& [customObjName, file_name] : custom_objs_json.items()) {
std::string customObjPath =
folder_path + k_separator + file_name.get<std::string>();
auto customObjPath = folder_path / file_name.get<std::string>();
LOG(INFO) << "Loading custom object to FbProxyExecutor from: "
<< customObjPath;

Expand Down
64 changes: 10 additions & 54 deletions torch/csrc/inductor/aoti_torch/shim_common.cpp
View file
Open in desktop
< 9E88 td class="blob-code blob-code-deletion js-file-line"> if (mkdir(path.c_str(), 0777) == -1) {
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <cstdint>
#include <cstdio>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <vector>
Expand Down Expand Up @@ -55,54 +56,7 @@

#endif

#ifndef _WIN32
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <climits>

#else
#include <filesystem>
namespace fs = std::filesystem;
#endif

// HACK for failed builds in ARVR, where it cannot find these symbols within
// std::experimental::filesystem
namespace {
std::string get_current_path() {
#ifdef _WIN32
return fs::current_path().string();
#else
// NOLINTNEXTLINE(*array*)
char currentPath[PATH_MAX]{};
if (getcwd(currentPath, sizeof(currentPath)) != nullptr) {
return std::string(currentPath);
} else {
throw std::runtime_error("Failed to get current path");
}
#endif
}

bool file_exists(std::string& path) {
#ifdef _WIN32
return fs::exists(path);
#else
struct stat rc {};
return lstat(path.c_str(), &rc) == 0;
#endif
}

bool create_directories(const std::string& path) {
#ifdef _WIN32
return fs::create_directories(path);
#else
throw std::runtime_error("Failed to create directory");
}
return true;
#endif
}
} // namespace

using namespace torch::aot_inductor;

Expand Down Expand Up @@ -1132,21 +1086,23 @@ void aoti_torch_save_tensor_handle(
at::Tensor* t = tensor_handle_to_tensor_pointer(self);
#ifndef C10_MOBILE
// Save tensor to tmp .pt file for tensors and can be torch.load'ed later
std::string cwd = get_current_path();
std::string tmp_folder = cwd + "/tmp/aoti_torch/";
if (!file_exists(tmp_folder)) {
auto cwd = std::filesystem::current_path();
auto tmp_folder = cwd / "tmp" / "aoti_torch";
if (!std::filesystem::exists(tmp_folder)) {
std::cout
<< "aoti_torch_save_tensor_handle: Path does not exist, creating it..."
<< tmp_folder << '\n';

if (!create_directories(tmp_folder)) {
std::error_code ec{};
if (std::filesystem::create_directories(tmp_folder, ec) &&
!std::filesystem::is_directory(tmp_folder)) {
std::cout << "aoti_torch_save_tensor_handle: Error creating directory: "
<< tmp_folder << '\n';
<< tmp_folder << " error:" << ec.message() << '\n';
return;
}
}
std::string tensor_filepath_to_save = tmp_folder + launch_prefix + "_" +
kernel_name + "_" + tensor_name + "_" + t->device().str() + ".pt";
std::string tensor_filepath_to_save = tmp_folder.string() + launch_prefix +
"_" + kernel_name + "_" + tensor_name + "_" + t->device().str() + ".pt";

auto bytes = torch::jit::pickle_save(c10::IValue(*t));
std::ofstream fout(tensor_filepath_to_save, std::ios::out | std::ios::binary);
Expand Down
Loading
0