diff options
Diffstat (limited to 'nix/libutil')
| -rw-r--r-- | nix/libutil/archive.cc | 171 | ||||
| -rw-r--r-- | nix/libutil/archive.hh | 12 | ||||
| -rw-r--r-- | nix/libutil/serialise.cc | 27 | ||||
| -rw-r--r-- | nix/libutil/serialise.hh | 11 | ||||
| -rw-r--r-- | nix/libutil/types.hh | 17 | ||||
| -rw-r--r-- | nix/libutil/util.cc | 176 | ||||
| -rw-r--r-- | nix/libutil/util.hh | 30 |
7 files changed, 272 insertions, 172 deletions
diff --git a/nix/libutil/archive.cc b/nix/libutil/archive.cc index 70a1c580dd..6856ea0f28 100644 --- a/nix/libutil/archive.cc +++ b/nix/libutil/archive.cc @@ -1,10 +1,14 @@ +#define _XOPEN_SOURCE 600 + #include "config.h" #include <cerrno> #include <algorithm> #include <vector> +#include <map> + +#include <strings.h> // for strcasecmp -#define _XOPEN_SOURCE 600 #include <sys/types.h> #include <sys/stat.h> #include <unistd.h> @@ -18,39 +22,21 @@ namespace nix { +bool useCaseHack = +#if __APPLE__ + true; +#else + false; +#endif + static string archiveVersion1 = "nix-archive-1"; +static string caseHackSuffix = "~nix~case~hack~"; PathFilter defaultPathFilter; -static void dump(const string & path, Sink & sink, PathFilter & filter); - - -static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter) -{ - Strings names = readDirectory(path); - vector<string> names2(names.begin(), names.end()); - sort(names2.begin(), names2.end()); - - for (vector<string>::iterator i = names2.begin(); - i != names2.end(); ++i) - { - Path entry = path + "/" + *i; - if (filter(entry)) { - writeString("entry", sink); - writeString("(", sink); - writeString("name", sink); - writeString(*i, sink); - writeString("node", sink); - dump(entry, sink, filter); - writeString(")", sink); - } - } -} - - -static void dumpContents(const Path & path, size_t size, +static void dumpContents(const Path & path, size_t size, Sink & sink) { writeString("contents", sink); @@ -58,7 +44,7 @@ static void dumpContents(const Path & path, size_t size, AutoCloseFD fd = open(path.c_str(), O_RDONLY); if (fd == -1) throw SysError(format("opening file `%1%'") % path); - + unsigned char buf[65536]; size_t left = size; @@ -89,12 +75,40 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) writeString("", sink); } dumpContents(path, (size_t) st.st_size, sink); - } + } else if (S_ISDIR(st.st_mode)) { writeString("type", sink); writeString("directory", sink); - dumpEntries(path, sink, filter); + + /* If we're on a case-insensitive system like Mac OS X, undo + the case hack applied by restorePath(). */ + std::map<string, string> unhacked; + for (auto & i : readDirectory(path)) + if (useCaseHack) { + string name(i.name); + size_t pos = i.name.find(caseHackSuffix); + if (pos != string::npos) { + printMsg(lvlDebug, format("removing case hack suffix from `%1%'") % (path + "/" + i.name)); + name.erase(pos); + } + if (unhacked.find(name) != unhacked.end()) + throw Error(format("file name collision in between `%1%' and `%2%'") + % (path + "/" + unhacked[name]) % (path + "/" + i.name)); + unhacked[name] = i.name; + } else + unhacked[i.name] = i.name; + + for (auto & i : unhacked) + if (filter(path + "/" + i.first)) { + writeString("entry", sink); + writeString("(", sink); + writeString("name", sink); + writeString(i.first, sink); + writeString("node", sink); + dump(path + "/" + i.second, sink, filter); + writeString(")", sink); + } } else if (S_ISLNK(st.st_mode)) { @@ -123,6 +137,7 @@ static SerialisationError badArchive(string s) } +#if 0 static void skipGeneric(Source & source) { if (readString(source) == "(") { @@ -130,43 +145,13 @@ static void skipGeneric(Source & source) skipGeneric(source); } } - - -static void parse(ParseSink & sink, Source & source, const Path & path); - - - -static void parseEntry(ParseSink & sink, Source & source, const Path & path) -{ - string s, name; - - s = readString(source); - if (s != "(") throw badArchive("expected open tag"); - - while (1) { - checkInterrupt(); - - s = readString(source); - - if (s == ")") { - break; - } else if (s == "name") { - name = readString(source); - } else if (s == "node") { - if (s == "") throw badArchive("entry name missing"); - parse(sink, source, path + "/" + name); - } else { - throw badArchive("unknown field " + s); - skipGeneric(source); - } - } -} +#endif static void parseContents(ParseSink & sink, Source & source, const Path & path) { unsigned long long size = readLongLong(source); - + sink.preallocateContents(size); unsigned long long left = size; @@ -185,6 +170,15 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path) } +struct CaseInsensitiveCompare +{ + bool operator() (const string & a, const string & b) const + { + return strcasecmp(a.c_str(), b.c_str()) < 0; + } +}; + + static void parse(ParseSink & sink, Source & source, const Path & path) { string s; @@ -194,6 +188,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path) enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; + std::map<Path, int, CaseInsensitiveCompare> names; + while (1) { checkInterrupt(); @@ -221,9 +217,9 @@ static void parse(ParseSink & sink, Source & source, const Path & path) else if (t == "symlink") { type = tpSymlink; } - + else throw badArchive("unknown file type " + t); - + } else if (s == "contents" && type == tpRegular) { @@ -236,7 +232,40 @@ static void parse(ParseSink & sink, Source & source, const Path & path) } else if (s == "entry" && type == tpDirectory) { - parseEntry(sink, source, path); + string name, prevName; + + s = readString(source); + if (s != "(") throw badArchive("expected open tag"); + + while (1) { + checkInterrupt(); + + s = readString(source); + + if (s == ")") { + break; + } else if (s == "name") { + name = readString(source); + if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos) + throw Error(format("NAR contains invalid file name `%1%'") % name); + if (name <= prevName) + throw Error("NAR directory is not sorted"); + prevName = name; + if (useCaseHack) { + auto i = names.find(name); + if (i != names.end()) { + printMsg(lvlDebug, format("case collision between `%1%' and `%2%'") % i->first % name); + name += caseHackSuffix; + name += int2String(++i->second); + } else + names[name] = 0; + } + } else if (s == "node") { + if (s.empty()) throw badArchive("entry name missing"); + parse(sink, source, path + "/" + name); + } else + throw badArchive("unknown field " + s); + } } else if (s == "target" && type == tpSymlink) { @@ -244,17 +273,15 @@ static void parse(ParseSink & sink, Source & source, const Path & path) sink.createSymlink(path, target); } - else { + else throw badArchive("unknown field " + s); - skipGeneric(source); - } } } void parseDump(ParseSink & sink, Source & source) { - string version; + string version; try { version = readString(source); } catch (SerialisationError & e) { @@ -323,7 +350,7 @@ struct RestoreSink : ParseSink } }; - + void restorePath(const Path & path, Source & source) { RestoreSink sink; @@ -331,5 +358,5 @@ void restorePath(const Path & path, Source & source) parseDump(sink, source); } - + } diff --git a/nix/libutil/archive.hh b/nix/libutil/archive.hh index ccac92074d..c216e9768f 100644 --- a/nix/libutil/archive.hh +++ b/nix/libutil/archive.hh @@ -28,7 +28,7 @@ namespace nix { where: - attrs(as) = concat(map(attr, as)) + encN(0) + attrs(as) = concat(map(attr, as)) + encN(0) attrs((a, b)) = encS(a) + encS(b) encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) @@ -58,7 +58,7 @@ void dumpPath(const Path & path, Sink & sink, struct ParseSink { virtual void createDirectory(const Path & path) { }; - + virtual void createRegularFile(const Path & path) { }; virtual void isExecutable() { }; virtual void preallocateContents(unsigned long long size) { }; @@ -66,10 +66,14 @@ struct ParseSink virtual void createSymlink(const Path & path, const string & target) { }; }; - + void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); - + +// FIXME: global variables are bad m'kay. +extern bool useCaseHack; + + } diff --git a/nix/libutil/serialise.cc b/nix/libutil/serialise.cc index 6b71f52c15..9241750750 100644 --- a/nix/libutil/serialise.cc +++ b/nix/libutil/serialise.cc @@ -54,8 +54,24 @@ FdSink::~FdSink() } +size_t threshold = 256 * 1024 * 1024; + +static void warnLargeDump() +{ + printMsg(lvlError, "warning: dumping very large path (> 256 MiB); this may run out of memory"); +} + + void FdSink::write(const unsigned char * data, size_t len) { + static bool warned = false; + if (warn && !warned) { + written += len; + if (written > threshold) { + warnLargeDump(); + warned = true; + } + } writeFull(fd, data, len); } @@ -256,4 +272,15 @@ template Paths readStrings(Source & source); template PathSet readStrings(Source & source); +void StringSink::operator () (const unsigned char * data, size_t len) +{ + static bool warned = false; + if (!warned && s.size() > threshold) { + warnLargeDump(); + warned = true; + } + s.append((const char *) data, len); +} + + } diff --git a/nix/libutil/serialise.hh b/nix/libutil/serialise.hh index e5a9df1d05..6a6f028aa6 100644 --- a/nix/libutil/serialise.hh +++ b/nix/libutil/serialise.hh @@ -72,9 +72,11 @@ struct BufferedSource : Source struct FdSink : BufferedSink { int fd; + bool warn; + size_t written; - FdSink() : fd(-1) { } - FdSink(int fd) : fd(fd) { } + FdSink() : fd(-1), warn(false), written(0) { } + FdSink(int fd) : fd(fd), warn(false), written(0) { } ~FdSink(); void write(const unsigned char * data, size_t len); @@ -95,10 +97,7 @@ struct FdSource : BufferedSource struct StringSink : Sink { string s; - void operator () (const unsigned char * data, size_t len) - { - s.append((const char *) data, len); - } + void operator () (const unsigned char * data, size_t len); }; diff --git a/nix/libutil/types.hh b/nix/libutil/types.hh index 4b5ce9a78c..160884ee1a 100644 --- a/nix/libutil/types.hh +++ b/nix/libutil/types.hh @@ -8,6 +8,15 @@ #include <boost/format.hpp> +/* Before 4.7, gcc's std::exception uses empty throw() specifiers for + * its (virtual) destructor and what() in c++11 mode, in violation of spec + */ +#ifdef __GNUC__ +#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) +#define EXCEPTION_NEEDS_THROW_SPEC +#endif +#endif + namespace nix { @@ -39,10 +48,14 @@ protected: public: unsigned int status; // exit status BaseError(const FormatOrString & fs, unsigned int status = 1); +#ifdef EXCEPTION_NEEDS_THROW_SPEC ~BaseError() throw () { }; const char * what() const throw () { return err.c_str(); } - const string & msg() const throw () { return err; } - const string & prefix() const throw () { return prefix_; } +#else + const char * what() const noexcept { return err.c_str(); } +#endif + const string & msg() const { return err; } + const string & prefix() const { return prefix_; } BaseError & addPrefix(const FormatOrString & fs); }; diff --git a/nix/libutil/util.cc b/nix/libutil/util.cc index 846674a29d..a4a1ddb12a 100644 --- a/nix/libutil/util.cc +++ b/nix/libutil/util.cc @@ -1,5 +1,8 @@ #include "config.h" +#include "util.hh" +#include "affinity.hh" + #include <iostream> #include <cerrno> #include <cstdio> @@ -16,8 +19,6 @@ #include <sys/syscall.h> #endif -#include "util.hh" - extern char * * environ; @@ -125,7 +126,6 @@ Path canonPath(const Path & path, bool resolveSymlinks) i = temp.begin(); /* restart */ end = temp.end(); s = ""; - /* !!! potential for infinite loop */ } } } @@ -202,9 +202,10 @@ bool isLink(const Path & path) } -Strings readDirectory(const Path & path) +DirEntries readDirectory(const Path & path) { - Strings names; + DirEntries entries; + entries.reserve(64); AutoCloseDir dir = opendir(path.c_str()); if (!dir) throw SysError(format("opening directory `%1%'") % path); @@ -214,11 +215,21 @@ Strings readDirectory(const Path & path) checkInterrupt(); string name = dirent->d_name; if (name == "." || name == "..") continue; - names.push_back(name); + entries.emplace_back(name, dirent->d_ino, dirent->d_type); } if (errno) throw SysError(format("reading directory `%1%'") % path); - return names; + return entries; +} + + +unsigned char getFileType(const Path & path) +{ + struct stat st = lstat(path); + if (S_ISDIR(st.st_mode)) return DT_DIR; + if (S_ISLNK(st.st_mode)) return DT_LNK; + if (S_ISREG(st.st_mode)) return DT_REG; + return DT_UNKNOWN; } @@ -293,16 +304,14 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed) bytesFreed += st.st_blocks * 512; if (S_ISDIR(st.st_mode)) { - Strings names = readDirectory(path); - /* Make the directory writable. */ if (!(st.st_mode & S_IWUSR)) { if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) throw SysError(format("making `%1%' writable") % path); } - for (Strings::iterator i = names.begin(); i != names.end(); ++i) - _deletePath(path + "/" + *i, bytesFreed); + for (auto & i : readDirectory(path)) + _deletePath(path + "/" + i.name, bytesFreed); } if (remove(path.c_str()) == -1) @@ -380,6 +389,9 @@ Paths createDirs(const Path & path) created.push_back(path); } + if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) + throw SysError(format("statting symlink `%1%'") % path); + if (!S_ISDIR(st.st_mode)) throw Error(format("`%1%' is not a directory") % path); return created; @@ -466,10 +478,18 @@ void warnOnce(bool & haveWarned, const FormatOrString & fs) } +static void defaultWriteToStderr(const unsigned char * buf, size_t count) +{ + writeFull(STDERR_FILENO, buf, count); +} + + void writeToStderr(const string & s) { try { - _writeToStderr((const unsigned char *) s.data(), s.size()); + auto p = _writeToStderr; + if (!p) p = defaultWriteToStderr; + p((const unsigned char *) s.data(), s.size()); } catch (SysError & e) { /* Ignore failing writes to stderr if we're in an exception handler, otherwise throw an exception. We need to ignore @@ -481,12 +501,6 @@ void writeToStderr(const string & s) } -static void defaultWriteToStderr(const unsigned char * buf, size_t count) -{ - writeFull(STDERR_FILENO, buf, count); -} - - void (*_writeToStderr) (const unsigned char * buf, size_t count) = defaultWriteToStderr; @@ -707,10 +721,14 @@ void AutoCloseDir::close() Pid::Pid() + : pid(-1), separatePG(false), killSignal(SIGKILL) +{ +} + + +Pid::Pid(pid_t pid) + : pid(pid), separatePG(false), killSignal(SIGKILL) { - pid = -1; - separatePG = false; - killSignal = SIGKILL; } @@ -734,11 +752,12 @@ Pid::operator pid_t() } -void Pid::kill() +void Pid::kill(bool quiet) { if (pid == -1 || pid == 0) return; - printMsg(lvlError, format("killing process %1%") % pid); + if (!quiet) + printMsg(lvlError, format("killing process %1%") % pid); /* Send the requested signal to the child. If it has its own process group, send the signal to every process in the child @@ -801,43 +820,30 @@ void killUser(uid_t uid) users to which the current process can send signals. So we fork a process, switch to uid, and send a mass kill. */ - Pid pid; - pid = fork(); - switch (pid) { + Pid pid = startProcess([&]() { - case -1: - throw SysError("unable to fork"); + if (setuid(uid) == -1) + throw SysError("setting uid"); - case 0: - try { /* child */ - - if (setuid(uid) == -1) - throw SysError("setting uid"); - - while (true) { + while (true) { #ifdef __APPLE__ - /* OSX's kill syscall takes a third parameter that, among other - things, determines if kill(-1, signo) affects the calling - process. In the OSX libc, it's set to true, which means - "follow POSIX", which we don't want here + /* OSX's kill syscall takes a third parameter that, among + other things, determines if kill(-1, signo) affects the + calling process. In the OSX libc, it's set to true, + which means "follow POSIX", which we don't want here */ - if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; #else - if (kill(-1, SIGKILL) == 0) break; + if (kill(-1, SIGKILL) == 0) break; #endif - if (errno == ESRCH) break; /* no more processes */ - if (errno != EINTR) - throw SysError(format("cannot kill processes for uid `%1%'") % uid); - } - - } catch (std::exception & e) { - writeToStderr((format("killing processes belonging to uid `%1%': %2%\n") % uid % e.what()).str()); - _exit(1); + if (errno == ESRCH) break; /* no more processes */ + if (errno != EINTR) + throw SysError(format("cannot kill processes for uid `%1%'") % uid); } + _exit(0); - } + }); - /* parent */ int status = pid.wait(true); if (status != 0) throw Error(format("cannot kill processes for uid `%1%': %2%") % uid % statusToString(status)); @@ -852,6 +858,32 @@ void killUser(uid_t uid) ////////////////////////////////////////////////////////////////////// +pid_t startProcess(std::function<void()> fun, + bool dieWithParent, const string & errorPrefix, bool runExitHandlers) +{ + pid_t pid = fork(); + if (pid == -1) throw SysError("unable to fork"); + + if (pid == 0) { + _writeToStderr = 0; + try { + restoreAffinity(); + fun(); + } catch (std::exception & e) { + try { + std::cerr << errorPrefix << e.what() << "\n"; + } catch (...) { } + } catch (...) { } + if (runExitHandlers) + exit(1); + else + _exit(1); + } + + return pid; +} + + string runProgram(Path program, bool searchPath, const Strings & args) { checkInterrupt(); @@ -867,32 +899,17 @@ string runProgram(Path program, bool searchPath, const Strings & args) pipe.create(); /* Fork. */ - Pid pid; - pid = maybeVfork(); - - switch (pid) { - - case -1: - throw SysError("unable to fork"); + Pid pid = startProcess([&]() { + if (dup2(pipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); - case 0: /* child */ - try { - if (dup2(pipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - - if (searchPath) - execvp(program.c_str(), (char * *) &cargs[0]); - else - execv(program.c_str(), (char * *) &cargs[0]); - throw SysError(format("executing `%1%'") % program); - - } catch (std::exception & e) { - writeToStderr("error: " + string(e.what()) + "\n"); - } - _exit(1); - } + if (searchPath) + execvp(program.c_str(), (char * *) &cargs[0]); + else + execv(program.c_str(), (char * *) &cargs[0]); - /* Parent. */ + throw SysError(format("executing `%1%'") % program); + }); pipe.writeSide.close(); @@ -901,7 +918,7 @@ string runProgram(Path program, bool searchPath, const Strings & args) /* Wait for the child to finish. */ int status = pid.wait(true); if (!statusOk(status)) - throw Error(format("program `%1%' %2%") + throw ExecError(format("program `%1%' %2%") % program % statusToString(status)); return result; @@ -928,13 +945,6 @@ void closeOnExec(int fd) } -#if HAVE_VFORK -pid_t (*maybeVfork)() = vfork; -#else -pid_t (*maybeVfork)() = fork; -#endif - - ////////////////////////////////////////////////////////////////////// diff --git a/nix/libutil/util.hh b/nix/libutil/util.hh index ce2d77c19a..0ad0026711 100644 --- a/nix/libutil/util.hh +++ b/nix/libutil/util.hh @@ -7,6 +7,7 @@ #include <dirent.h> #include <unistd.h> #include <signal.h> +#include <functional> #include <cstdio> @@ -63,7 +64,20 @@ bool isLink(const Path & path); /* Read the contents of a directory. The entries `.' and `..' are removed. */ -Strings readDirectory(const Path & path); +struct DirEntry +{ + string name; + ino_t ino; + unsigned char type; // one of DT_* + DirEntry(const string & name, ino_t ino, unsigned char type) + : name(name), ino(ino), type(type) { } +}; + +typedef vector<DirEntry> DirEntries; + +DirEntries readDirectory(const Path & path); + +unsigned char getFileType(const Path & path); /* Read the contents of a file into a string. */ string readFile(int fd); @@ -237,10 +251,11 @@ class Pid int killSignal; public: Pid(); + Pid(pid_t pid); ~Pid(); void operator =(pid_t pid); operator pid_t(); - void kill(); + void kill(bool quiet = false); int wait(bool block); void setSeparatePG(bool separatePG); void setKillSignal(int signal); @@ -252,11 +267,19 @@ public: void killUser(uid_t uid); +/* Fork a process that runs the given function, and return the child + pid to the caller. */ +pid_t startProcess(std::function<void()> fun, bool dieWithParent = true, + const string & errorPrefix = "error: ", bool runExitHandlers = false); + + /* Run a program and return its stdout in a string (i.e., like the shell backtick operator). */ string runProgram(Path program, bool searchPath = false, const Strings & args = Strings()); +MakeError(ExecError, Error) + /* Close all file descriptors except stdin, stdout, stderr, and those listed in the given set. Good practice in child processes. */ void closeMostFDs(const set<int> & exceptions); @@ -264,9 +287,6 @@ void closeMostFDs(const set<int> & exceptions); /* Set the close-on-exec flag for the given file descriptor. */ void closeOnExec(int fd); -/* Call vfork() if available, otherwise fork(). */ -extern pid_t (*maybeVfork)(); - /* User interruption. */ |