d6259d6a54
When tests are run in parallel, they will all compete to update and set up the systemplate directory, which has a handful of files that need to be up-to-date. This is a source of errors. Normally, these files are linked (hard- or soft- link, whichever succeeds). With linking, we only need to worry about the initial setup, as the files will never be out-of-date from then on. However, when linking fails, we need to copy the files, and update them (by copying over fresh versions of the files, if necessary) every time a new kit is forked. Copying over is tricky, as it's not atomic. To make it atomic, we copy the files to the destination directory under a temporary (random) name, and then rename to the final name (which is atomic, including replacing the target file, if it exists). No such race exists in production, where there is (or should be) but one instance of loolwsd (which does the initial setup) and forkit (which updates systemplate before forking new kit instances). This is an issue with parallel tests only. Change-Id: I6ba1514d00a84da7397d28efeb6378619711d52f Reviewed-on: https://gerrit.libreoffice.org/c/online/+/97785 Tested-by: Jenkins Tested-by: Jenkins CollaboraOffice <jenkinscollaboraoffice@gmail.com> Reviewed-by: Ashod Nakashian <ashnakash@gmail.com>
487 lines
14 KiB
C++
487 lines
14 KiB
C++
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */
|
|
/*
|
|
* This file is part of the LibreOffice project.
|
|
*
|
|
* This Source Code Form is subject to the terms of the Mozilla Public
|
|
* License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
*/
|
|
|
|
#include <config.h>
|
|
|
|
#include "FileUtil.hpp"
|
|
|
|
#include <dirent.h>
|
|
#include <ftw.h>
|
|
#include <sys/time.h>
|
|
#ifdef __linux
|
|
#include <sys/vfs.h>
|
|
#elif defined IOS
|
|
#import <Foundation/Foundation.h>
|
|
#endif
|
|
|
|
#include <chrono>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <fstream>
|
|
#include <mutex>
|
|
#include <string>
|
|
|
|
#if HAVE_STD_FILESYSTEM
|
|
# if HAVE_STD_FILESYSTEM_EXPERIMENTAL
|
|
# include <experimental/filesystem>
|
|
namespace filesystem = ::std::experimental::filesystem;
|
|
# else
|
|
# include <filesystem>
|
|
namespace filesystem = ::std::filesystem;
|
|
# endif
|
|
#else
|
|
# include <Poco/TemporaryFile.h>
|
|
#endif
|
|
|
|
#include <Poco/File.h>
|
|
#include <Poco/Path.h>
|
|
|
|
#include "Log.hpp"
|
|
#include "Util.hpp"
|
|
#include "Unit.hpp"
|
|
|
|
namespace
|
|
{
|
|
#if HAVE_STD_FILESYSTEM
|
|
/// Class to delete files when the process ends.
|
|
class FileDeleter
|
|
{
|
|
std::vector<std::string> _filesToDelete;
|
|
std::mutex _lock;
|
|
public:
|
|
FileDeleter() {}
|
|
~FileDeleter()
|
|
{
|
|
std::unique_lock<std::mutex> guard(_lock);
|
|
for (const std::string& file: _filesToDelete)
|
|
filesystem::remove(file);
|
|
}
|
|
|
|
void registerForDeletion(const std::string& file)
|
|
{
|
|
std::unique_lock<std::mutex> guard(_lock);
|
|
_filesToDelete.push_back(file);
|
|
}
|
|
};
|
|
#endif
|
|
}
|
|
|
|
namespace FileUtil
|
|
{
|
|
std::string createRandomDir(const std::string& path)
|
|
{
|
|
const std::string name = Util::rng::getFilename(64);
|
|
#if HAVE_STD_FILESYSTEM
|
|
filesystem::create_directory(path + '/' + name);
|
|
#else
|
|
Poco::File(Poco::Path(path, name)).createDirectories();
|
|
#endif
|
|
return name;
|
|
}
|
|
|
|
bool copy(const std::string& fromPath, const std::string& toPath, bool log, bool throw_on_error)
|
|
{
|
|
int from = -1, to = -1;
|
|
try {
|
|
from = open(fromPath.c_str(), O_RDONLY);
|
|
if (from < 0)
|
|
{
|
|
LOG_SYS("Failed to open src " << anonymizeUrl(fromPath));
|
|
throw;
|
|
}
|
|
|
|
struct stat st;
|
|
if (fstat(from, &st) != 0)
|
|
{
|
|
LOG_SYS("Failed to fstat src " << anonymizeUrl(fromPath));
|
|
throw;
|
|
}
|
|
|
|
to = open(toPath.c_str(), O_CREAT | O_TRUNC | O_WRONLY, st.st_mode);
|
|
if (to < 0)
|
|
{
|
|
LOG_SYS("Failed to fstat dest " << anonymizeUrl(toPath));
|
|
throw;
|
|
}
|
|
|
|
// Logging may be redundant and/or noisy.
|
|
if (log)
|
|
LOG_INF("Copying " << st.st_size << " bytes from " << anonymizeUrl(fromPath)
|
|
<< " to " << anonymizeUrl(toPath));
|
|
|
|
char buffer[64 * 1024];
|
|
|
|
int n;
|
|
off_t bytesIn = 0;
|
|
do {
|
|
while ((n = ::read(from, buffer, sizeof(buffer))) < 0 && errno == EINTR)
|
|
LOG_TRC("EINTR reading from " << anonymizeUrl(fromPath));
|
|
if (n < 0)
|
|
{
|
|
LOG_SYS("Failed to read from " << anonymizeUrl(fromPath) << " at " << bytesIn << " bytes in");
|
|
throw;
|
|
}
|
|
bytesIn += n;
|
|
if (n == 0) // EOF
|
|
break;
|
|
assert (off_t(sizeof (buffer)) >= n);
|
|
// Handle short writes and EINTR
|
|
for (int j = 0; j < n;)
|
|
{
|
|
int written;
|
|
while ((written = ::write(to, buffer + j, n - j)) < 0 && errno == EINTR)
|
|
LOG_TRC("EINTR writing to " << anonymizeUrl(toPath));
|
|
if (written < 0)
|
|
{
|
|
LOG_SYS("Failed to write " << n << " bytes to " << anonymizeUrl(toPath) << " at " <<
|
|
bytesIn << " bytes into " << anonymizeUrl(fromPath));
|
|
throw;
|
|
}
|
|
j += written;
|
|
}
|
|
} while(true);
|
|
if (bytesIn != st.st_size)
|
|
{
|
|
LOG_WRN("Unusual: file " << anonymizeUrl(fromPath) << " changed size "
|
|
"during copy from " << st.st_size << " to " << bytesIn);
|
|
}
|
|
close(from);
|
|
close(to);
|
|
return true;
|
|
}
|
|
catch (...)
|
|
{
|
|
std::ostringstream oss;
|
|
oss << "Failed to copy from " << anonymizeUrl(fromPath) << " to "
|
|
<< anonymizeUrl(toPath);
|
|
const std::string err = oss.str();
|
|
|
|
LOG_SYS(err);
|
|
close(from);
|
|
close(to);
|
|
unlink(toPath.c_str());
|
|
if (throw_on_error)
|
|
throw Poco::Exception(err);
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
std::string getTempFilePath(const std::string& srcDir, const std::string& srcFilename, const std::string& dstFilenamePrefix)
|
|
{
|
|
const std::string srcPath = srcDir + '/' + srcFilename;
|
|
const std::string dstFilename = dstFilenamePrefix + Util::encodeId(Util::rng::getNext()) + '_' + srcFilename;
|
|
#if HAVE_STD_FILESYSTEM
|
|
const std::string dstPath = filesystem::temp_directory_path() / dstFilename;
|
|
filesystem::copy(srcPath, dstPath);
|
|
|
|
static FileDeleter fileDeleter;
|
|
fileDeleter.registerForDeletion(dstPath);
|
|
#else
|
|
const std::string dstPath = Poco::Path(Poco::Path::temp(), dstFilename).toString();
|
|
copyFileTo(srcPath, dstPath);
|
|
Poco::TemporaryFile::registerForDeletion(dstPath);
|
|
#endif
|
|
|
|
return dstPath;
|
|
}
|
|
|
|
#if 1 // !HAVE_STD_FILESYSTEM
|
|
static int nftw_cb(const char *fpath, const struct stat*, int type, struct FTW*)
|
|
{
|
|
if (type == FTW_DP)
|
|
{
|
|
rmdir(fpath);
|
|
}
|
|
else if (type == FTW_F || type == FTW_SL)
|
|
{
|
|
unlink(fpath);
|
|
}
|
|
|
|
// Always continue even when things go wrong.
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
void removeFile(const std::string& path, const bool recursive)
|
|
{
|
|
LOG_DBG("Removing [" << path << "] " << (recursive ? "recursively." : "only."));
|
|
|
|
// Amazingly filesystem::remove_all silently fails to work on some
|
|
// systems. No real need to be using experimental API here either.
|
|
#if 0 // HAVE_STD_FILESYSTEM
|
|
std::error_code ec;
|
|
if (recursive)
|
|
filesystem::remove_all(path, ec);
|
|
else
|
|
filesystem::remove(path, ec);
|
|
|
|
// Already removed or we don't care about failures.
|
|
(void) ec;
|
|
#else
|
|
try
|
|
{
|
|
struct stat sb;
|
|
errno = 0;
|
|
if (!recursive || stat(path.c_str(), &sb) == -1 || S_ISREG(sb.st_mode))
|
|
{
|
|
// Non-recursive directories and files that exist.
|
|
if (errno != ENOENT)
|
|
Poco::File(path).remove(recursive);
|
|
}
|
|
else
|
|
{
|
|
// Directories only.
|
|
nftw(path.c_str(), nftw_cb, 128, FTW_DEPTH | FTW_PHYS);
|
|
}
|
|
}
|
|
catch (const std::exception&e)
|
|
{
|
|
// Already removed or we don't care about failures.
|
|
LOG_DBG("Failed to remove [" << path << "] " << (recursive ? "recursively: " : "only: ")
|
|
<< e.what());
|
|
}
|
|
#endif
|
|
}
|
|
|
|
std::string realpath(const char* path)
|
|
{
|
|
char* resolved = ::realpath(path, nullptr);
|
|
if (resolved)
|
|
{
|
|
std::string real = resolved;
|
|
free(resolved);
|
|
return real;
|
|
}
|
|
|
|
LOG_SYS("Failed to get the realpath of [" << path << "]");
|
|
return path;
|
|
}
|
|
|
|
bool isEmptyDirectory(const char* path)
|
|
{
|
|
DIR* dir = opendir(path);
|
|
if (dir == nullptr)
|
|
return errno != EACCES; // Assume it's not empty when EACCES.
|
|
|
|
int count = 0;
|
|
while (readdir(dir) && ++count < 3)
|
|
;
|
|
|
|
closedir(dir);
|
|
return count <= 2; // Discounting . and ..
|
|
}
|
|
|
|
bool updateTimestamps(const std::string& filename, timespec tsAccess, timespec tsModified)
|
|
{
|
|
// The timestamp is in seconds and microseconds.
|
|
timeval timestamps[2]{ { tsAccess.tv_sec, tsAccess.tv_nsec / 1000 },
|
|
{ tsModified.tv_sec, tsModified.tv_nsec / 1000 } };
|
|
if (utimes(filename.c_str(), timestamps) != 0)
|
|
{
|
|
LOG_SYS("Failed to update the timestamp of [" << filename << "]");
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
bool copyAtomic(const std::string& fromPath, const std::string& toPath, bool preserveTimestamps)
|
|
{
|
|
const std::string randFilename = toPath + Util::rng::getFilename(12);
|
|
if (copy(fromPath, randFilename, /*log=*/false, /*throw_on_error=*/false))
|
|
{
|
|
if (preserveTimestamps)
|
|
{
|
|
const Stat st(fromPath);
|
|
updateTimestamps(randFilename, st.sb().st_atim, st.sb().st_mtim);
|
|
}
|
|
|
|
// Now rename atomically, replacing any existing files with the same name.
|
|
if (rename(randFilename.c_str(), toPath.c_str()) == 0)
|
|
return true;
|
|
|
|
LOG_SYS("Failed to copy [" << fromPath << "] -> [" << toPath
|
|
<< "] while atomically renaming:");
|
|
removeFile(randFilename, false); // Cleanup.
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
bool linkOrCopyFile(const char* source, const char* target)
|
|
{
|
|
if (link(source, target) == -1)
|
|
{
|
|
LOG_INF("link(\"" << source << "\", \"" << target << "\") failed: " << strerror(errno)
|
|
<< ". Will copy.");
|
|
return copy(source, target, /*log=*/false, /*throw_on_error=*/false);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
} // namespace FileUtil
|
|
|
|
namespace
|
|
{
|
|
|
|
struct fs
|
|
{
|
|
fs(const std::string& path, dev_t dev)
|
|
: _path(path), _dev(dev)
|
|
{
|
|
}
|
|
|
|
const std::string& getPath() const { return _path; }
|
|
|
|
dev_t getDev() const { return _dev; }
|
|
|
|
private:
|
|
std::string _path;
|
|
dev_t _dev;
|
|
};
|
|
|
|
struct fsComparator
|
|
{
|
|
bool operator() (const fs& lhs, const fs& rhs) const
|
|
{
|
|
return (lhs.getDev() < rhs.getDev());
|
|
}
|
|
};
|
|
|
|
static std::mutex fsmutex;
|
|
static std::set<fs, fsComparator> filesystems;
|
|
|
|
} // anonymous namespace
|
|
|
|
namespace FileUtil
|
|
{
|
|
#if !MOBILEAPP
|
|
void registerFileSystemForDiskSpaceChecks(const std::string& path)
|
|
{
|
|
const std::string::size_type lastSlash = path.rfind('/');
|
|
assert(path.empty() || lastSlash != std::string::npos);
|
|
if (lastSlash != std::string::npos)
|
|
{
|
|
const std::string dirPath = path.substr(0, lastSlash + 1) + '.';
|
|
LOG_INF("Registering filesystem for space checks: [" << dirPath << ']');
|
|
|
|
std::lock_guard<std::mutex> lock(fsmutex);
|
|
|
|
struct stat s;
|
|
if (stat(dirPath.c_str(), &s) == 0)
|
|
{
|
|
filesystems.insert(fs(dirPath, s.st_dev));
|
|
}
|
|
}
|
|
}
|
|
|
|
std::string checkDiskSpaceOnRegisteredFileSystems(const bool cacheLastCheck)
|
|
{
|
|
static std::chrono::steady_clock::time_point lastCheck;
|
|
std::chrono::steady_clock::time_point now(std::chrono::steady_clock::now());
|
|
|
|
std::lock_guard<std::mutex> lock(fsmutex);
|
|
|
|
// Don't check more often than once a minute
|
|
if (std::chrono::duration_cast<std::chrono::seconds>(now - lastCheck).count() < 60)
|
|
return std::string();
|
|
|
|
if (cacheLastCheck)
|
|
lastCheck = now;
|
|
|
|
for (const auto& i: filesystems)
|
|
{
|
|
if (!checkDiskSpace(i.getPath()))
|
|
{
|
|
return i.getPath();
|
|
}
|
|
}
|
|
|
|
return std::string();
|
|
}
|
|
#endif
|
|
|
|
bool checkDiskSpace(const std::string& path)
|
|
{
|
|
assert(!path.empty());
|
|
|
|
#if !MOBILEAPP
|
|
bool hookResult;
|
|
if (UnitBase::get().filterCheckDiskSpace(path, hookResult))
|
|
return hookResult;
|
|
#endif
|
|
|
|
// we should be able to run just OK with 5GB for production or 1GB for development
|
|
#if ENABLE_DEBUG
|
|
const int64_t gb(1);
|
|
#else
|
|
const int64_t gb(5);
|
|
#endif
|
|
constexpr int64_t ENOUGH_SPACE = gb*1024*1024*1024;
|
|
|
|
#ifdef __linux
|
|
struct statfs sfs;
|
|
if (statfs(path.c_str(), &sfs) == -1)
|
|
return true;
|
|
|
|
const int64_t freeBytes = static_cast<int64_t>(sfs.f_bavail) * sfs.f_bsize;
|
|
|
|
LOG_INF("Filesystem [" << path << "] has " << (freeBytes / 1024 / 1024) <<
|
|
" MB free (" << (sfs.f_bavail * 100. / sfs.f_blocks) << "%).");
|
|
|
|
if (freeBytes > ENOUGH_SPACE)
|
|
return true;
|
|
|
|
if (static_cast<double>(sfs.f_bavail) / sfs.f_blocks <= 0.05)
|
|
return false;
|
|
#elif defined IOS
|
|
NSDictionary *atDict = [[NSFileManager defaultManager] attributesOfFileSystemForPath:@"/" error:NULL];
|
|
long long freeSpace = [[atDict objectForKey:NSFileSystemFreeSize] longLongValue];
|
|
long long totalSpace = [[atDict objectForKey:NSFileSystemSize] longLongValue];
|
|
|
|
if (freeSpace > ENOUGH_SPACE)
|
|
return true;
|
|
|
|
if (static_cast<double>(freeSpace) / totalSpace <= 0.05)
|
|
return false;
|
|
#endif
|
|
|
|
return true;
|
|
}
|
|
|
|
namespace {
|
|
bool AnonymizeUserData = false;
|
|
std::uint64_t AnonymizationSalt = 82589933;
|
|
}
|
|
|
|
void setUrlAnonymization(bool anonymize, const std::uint64_t salt)
|
|
{
|
|
AnonymizeUserData = anonymize;
|
|
AnonymizationSalt = salt;
|
|
}
|
|
|
|
/// Anonymize the basename of filenames, preserving the path and extension.
|
|
std::string anonymizeUrl(const std::string& url)
|
|
{
|
|
return AnonymizeUserData ? Util::anonymizeUrl(url, AnonymizationSalt) : url;
|
|
}
|
|
|
|
/// Anonymize user names and IDs.
|
|
/// Will use the Obfuscated User ID if one is provided via WOPI.
|
|
std::string anonymizeUsername(const std::string& username)
|
|
{
|
|
return AnonymizeUserData ? Util::anonymize(username, AnonymizationSalt) : username;
|
|
}
|
|
|
|
} // namespace FileUtil
|
|
|
|
/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
|