libreoffice-online/common/FileUtil.cpp
Ashod Nakashian d6259d6a54 wsd: support parallel systemplate setup
When tests are run in parallel, they will all
compete to update and set up the systemplate
directory, which has a handful of files that
need to be up-to-date. This is a source of errors.

Normally, these files are linked (hard- or soft-
link, whichever succeeds). With linking, we
only need to worry about the initial setup,
as the files will never be out-of-date from
then on. However, when linking fails, we need
to copy the files, and update them (by copying
over fresh versions of the files, if necessary)
every time a new kit is forked. Copying over
is tricky, as it's not atomic. To make it
atomic, we copy the files to the destination
directory under a temporary (random) name,
and then rename to the final name (which is
atomic, including replacing the target file,
if it exists).

No such race exists in production, where there
is (or should be) but one instance of loolwsd
(which does the initial setup) and forkit
(which updates systemplate before forking
new kit instances).
This is an issue with parallel tests only.

Change-Id: I6ba1514d00a84da7397d28efeb6378619711d52f
Reviewed-on: https://gerrit.libreoffice.org/c/online/+/97785
Tested-by: Jenkins
Tested-by: Jenkins CollaboraOffice <jenkinscollaboraoffice@gmail.com>
Reviewed-by: Ashod Nakashian <ashnakash@gmail.com>
2020-07-07 19:05:23 +02:00

487 lines
14 KiB
C++

/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; fill-column: 100 -*- */
/*
* This file is part of the LibreOffice project.
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
#include <config.h>
#include "FileUtil.hpp"
#include <dirent.h>
#include <ftw.h>
#include <sys/time.h>
#ifdef __linux
#include <sys/vfs.h>
#elif defined IOS
#import <Foundation/Foundation.h>
#endif
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <mutex>
#include <string>
#if HAVE_STD_FILESYSTEM
# if HAVE_STD_FILESYSTEM_EXPERIMENTAL
# include <experimental/filesystem>
namespace filesystem = ::std::experimental::filesystem;
# else
# include <filesystem>
namespace filesystem = ::std::filesystem;
# endif
#else
# include <Poco/TemporaryFile.h>
#endif
#include <Poco/File.h>
#include <Poco/Path.h>
#include "Log.hpp"
#include "Util.hpp"
#include "Unit.hpp"
namespace
{
#if HAVE_STD_FILESYSTEM
/// Class to delete files when the process ends.
class FileDeleter
{
std::vector<std::string> _filesToDelete;
std::mutex _lock;
public:
FileDeleter() {}
~FileDeleter()
{
std::unique_lock<std::mutex> guard(_lock);
for (const std::string& file: _filesToDelete)
filesystem::remove(file);
}
void registerForDeletion(const std::string& file)
{
std::unique_lock<std::mutex> guard(_lock);
_filesToDelete.push_back(file);
}
};
#endif
}
namespace FileUtil
{
std::string createRandomDir(const std::string& path)
{
const std::string name = Util::rng::getFilename(64);
#if HAVE_STD_FILESYSTEM
filesystem::create_directory(path + '/' + name);
#else
Poco::File(Poco::Path(path, name)).createDirectories();
#endif
return name;
}
bool copy(const std::string& fromPath, const std::string& toPath, bool log, bool throw_on_error)
{
int from = -1, to = -1;
try {
from = open(fromPath.c_str(), O_RDONLY);
if (from < 0)
{
LOG_SYS("Failed to open src " << anonymizeUrl(fromPath));
throw;
}
struct stat st;
if (fstat(from, &st) != 0)
{
LOG_SYS("Failed to fstat src " << anonymizeUrl(fromPath));
throw;
}
to = open(toPath.c_str(), O_CREAT | O_TRUNC | O_WRONLY, st.st_mode);
if (to < 0)
{
LOG_SYS("Failed to fstat dest " << anonymizeUrl(toPath));
throw;
}
// Logging may be redundant and/or noisy.
if (log)
LOG_INF("Copying " << st.st_size << " bytes from " << anonymizeUrl(fromPath)
<< " to " << anonymizeUrl(toPath));
char buffer[64 * 1024];
int n;
off_t bytesIn = 0;
do {
while ((n = ::read(from, buffer, sizeof(buffer))) < 0 && errno == EINTR)
LOG_TRC("EINTR reading from " << anonymizeUrl(fromPath));
if (n < 0)
{
LOG_SYS("Failed to read from " << anonymizeUrl(fromPath) << " at " << bytesIn << " bytes in");
throw;
}
bytesIn += n;
if (n == 0) // EOF
break;
assert (off_t(sizeof (buffer)) >= n);
// Handle short writes and EINTR
for (int j = 0; j < n;)
{
int written;
while ((written = ::write(to, buffer + j, n - j)) < 0 && errno == EINTR)
LOG_TRC("EINTR writing to " << anonymizeUrl(toPath));
if (written < 0)
{
LOG_SYS("Failed to write " << n << " bytes to " << anonymizeUrl(toPath) << " at " <<
bytesIn << " bytes into " << anonymizeUrl(fromPath));
throw;
}
j += written;
}
} while(true);
if (bytesIn != st.st_size)
{
LOG_WRN("Unusual: file " << anonymizeUrl(fromPath) << " changed size "
"during copy from " << st.st_size << " to " << bytesIn);
}
close(from);
close(to);
return true;
}
catch (...)
{
std::ostringstream oss;
oss << "Failed to copy from " << anonymizeUrl(fromPath) << " to "
<< anonymizeUrl(toPath);
const std::string err = oss.str();
LOG_SYS(err);
close(from);
close(to);
unlink(toPath.c_str());
if (throw_on_error)
throw Poco::Exception(err);
}
return false;
}
std::string getTempFilePath(const std::string& srcDir, const std::string& srcFilename, const std::string& dstFilenamePrefix)
{
const std::string srcPath = srcDir + '/' + srcFilename;
const std::string dstFilename = dstFilenamePrefix + Util::encodeId(Util::rng::getNext()) + '_' + srcFilename;
#if HAVE_STD_FILESYSTEM
const std::string dstPath = filesystem::temp_directory_path() / dstFilename;
filesystem::copy(srcPath, dstPath);
static FileDeleter fileDeleter;
fileDeleter.registerForDeletion(dstPath);
#else
const std::string dstPath = Poco::Path(Poco::Path::temp(), dstFilename).toString();
copyFileTo(srcPath, dstPath);
Poco::TemporaryFile::registerForDeletion(dstPath);
#endif
return dstPath;
}
#if 1 // !HAVE_STD_FILESYSTEM
static int nftw_cb(const char *fpath, const struct stat*, int type, struct FTW*)
{
if (type == FTW_DP)
{
rmdir(fpath);
}
else if (type == FTW_F || type == FTW_SL)
{
unlink(fpath);
}
// Always continue even when things go wrong.
return 0;
}
#endif
void removeFile(const std::string& path, const bool recursive)
{
LOG_DBG("Removing [" << path << "] " << (recursive ? "recursively." : "only."));
// Amazingly filesystem::remove_all silently fails to work on some
// systems. No real need to be using experimental API here either.
#if 0 // HAVE_STD_FILESYSTEM
std::error_code ec;
if (recursive)
filesystem::remove_all(path, ec);
else
filesystem::remove(path, ec);
// Already removed or we don't care about failures.
(void) ec;
#else
try
{
struct stat sb;
errno = 0;
if (!recursive || stat(path.c_str(), &sb) == -1 || S_ISREG(sb.st_mode))
{
// Non-recursive directories and files that exist.
if (errno != ENOENT)
Poco::File(path).remove(recursive);
}
else
{
// Directories only.
nftw(path.c_str(), nftw_cb, 128, FTW_DEPTH | FTW_PHYS);
}
}
catch (const std::exception&e)
{
// Already removed or we don't care about failures.
LOG_DBG("Failed to remove [" << path << "] " << (recursive ? "recursively: " : "only: ")
<< e.what());
}
#endif
}
std::string realpath(const char* path)
{
char* resolved = ::realpath(path, nullptr);
if (resolved)
{
std::string real = resolved;
free(resolved);
return real;
}
LOG_SYS("Failed to get the realpath of [" << path << "]");
return path;
}
bool isEmptyDirectory(const char* path)
{
DIR* dir = opendir(path);
if (dir == nullptr)
return errno != EACCES; // Assume it's not empty when EACCES.
int count = 0;
while (readdir(dir) && ++count < 3)
;
closedir(dir);
return count <= 2; // Discounting . and ..
}
bool updateTimestamps(const std::string& filename, timespec tsAccess, timespec tsModified)
{
// The timestamp is in seconds and microseconds.
timeval timestamps[2]{ { tsAccess.tv_sec, tsAccess.tv_nsec / 1000 },
{ tsModified.tv_sec, tsModified.tv_nsec / 1000 } };
if (utimes(filename.c_str(), timestamps) != 0)
{
LOG_SYS("Failed to update the timestamp of [" << filename << "]");
return false;
}
return true;
}
bool copyAtomic(const std::string& fromPath, const std::string& toPath, bool preserveTimestamps)
{
const std::string randFilename = toPath + Util::rng::getFilename(12);
if (copy(fromPath, randFilename, /*log=*/false, /*throw_on_error=*/false))
{
if (preserveTimestamps)
{
const Stat st(fromPath);
updateTimestamps(randFilename, st.sb().st_atim, st.sb().st_mtim);
}
// Now rename atomically, replacing any existing files with the same name.
if (rename(randFilename.c_str(), toPath.c_str()) == 0)
return true;
LOG_SYS("Failed to copy [" << fromPath << "] -> [" << toPath
<< "] while atomically renaming:");
removeFile(randFilename, false); // Cleanup.
}
return false;
}
bool linkOrCopyFile(const char* source, const char* target)
{
if (link(source, target) == -1)
{
LOG_INF("link(\"" << source << "\", \"" << target << "\") failed: " << strerror(errno)
<< ". Will copy.");
return copy(source, target, /*log=*/false, /*throw_on_error=*/false);
}
return true;
}
} // namespace FileUtil
namespace
{
struct fs
{
fs(const std::string& path, dev_t dev)
: _path(path), _dev(dev)
{
}
const std::string& getPath() const { return _path; }
dev_t getDev() const { return _dev; }
private:
std::string _path;
dev_t _dev;
};
struct fsComparator
{
bool operator() (const fs& lhs, const fs& rhs) const
{
return (lhs.getDev() < rhs.getDev());
}
};
static std::mutex fsmutex;
static std::set<fs, fsComparator> filesystems;
} // anonymous namespace
namespace FileUtil
{
#if !MOBILEAPP
void registerFileSystemForDiskSpaceChecks(const std::string& path)
{
const std::string::size_type lastSlash = path.rfind('/');
assert(path.empty() || lastSlash != std::string::npos);
if (lastSlash != std::string::npos)
{
const std::string dirPath = path.substr(0, lastSlash + 1) + '.';
LOG_INF("Registering filesystem for space checks: [" << dirPath << ']');
std::lock_guard<std::mutex> lock(fsmutex);
struct stat s;
if (stat(dirPath.c_str(), &s) == 0)
{
filesystems.insert(fs(dirPath, s.st_dev));
}
}
}
std::string checkDiskSpaceOnRegisteredFileSystems(const bool cacheLastCheck)
{
static std::chrono::steady_clock::time_point lastCheck;
std::chrono::steady_clock::time_point now(std::chrono::steady_clock::now());
std::lock_guard<std::mutex> lock(fsmutex);
// Don't check more often than once a minute
if (std::chrono::duration_cast<std::chrono::seconds>(now - lastCheck).count() < 60)
return std::string();
if (cacheLastCheck)
lastCheck = now;
for (const auto& i: filesystems)
{
if (!checkDiskSpace(i.getPath()))
{
return i.getPath();
}
}
return std::string();
}
#endif
bool checkDiskSpace(const std::string& path)
{
assert(!path.empty());
#if !MOBILEAPP
bool hookResult;
if (UnitBase::get().filterCheckDiskSpace(path, hookResult))
return hookResult;
#endif
// we should be able to run just OK with 5GB for production or 1GB for development
#if ENABLE_DEBUG
const int64_t gb(1);
#else
const int64_t gb(5);
#endif
constexpr int64_t ENOUGH_SPACE = gb*1024*1024*1024;
#ifdef __linux
struct statfs sfs;
if (statfs(path.c_str(), &sfs) == -1)
return true;
const int64_t freeBytes = static_cast<int64_t>(sfs.f_bavail) * sfs.f_bsize;
LOG_INF("Filesystem [" << path << "] has " << (freeBytes / 1024 / 1024) <<
" MB free (" << (sfs.f_bavail * 100. / sfs.f_blocks) << "%).");
if (freeBytes > ENOUGH_SPACE)
return true;
if (static_cast<double>(sfs.f_bavail) / sfs.f_blocks <= 0.05)
return false;
#elif defined IOS
NSDictionary *atDict = [[NSFileManager defaultManager] attributesOfFileSystemForPath:@"/" error:NULL];
long long freeSpace = [[atDict objectForKey:NSFileSystemFreeSize] longLongValue];
long long totalSpace = [[atDict objectForKey:NSFileSystemSize] longLongValue];
if (freeSpace > ENOUGH_SPACE)
return true;
if (static_cast<double>(freeSpace) / totalSpace <= 0.05)
return false;
#endif
return true;
}
namespace {
bool AnonymizeUserData = false;
std::uint64_t AnonymizationSalt = 82589933;
}
void setUrlAnonymization(bool anonymize, const std::uint64_t salt)
{
AnonymizeUserData = anonymize;
AnonymizationSalt = salt;
}
/// Anonymize the basename of filenames, preserving the path and extension.
std::string anonymizeUrl(const std::string& url)
{
return AnonymizeUserData ? Util::anonymizeUrl(url, AnonymizationSalt) : url;
}
/// Anonymize user names and IDs.
/// Will use the Obfuscated User ID if one is provided via WOPI.
std::string anonymizeUsername(const std::string& username)
{
return AnonymizeUserData ? Util::anonymize(username, AnonymizationSalt) : username;
}
} // namespace FileUtil
/* vim:set shiftwidth=4 softtabstop=4 expandtab: */