diff options
Diffstat (limited to 'nix/libutil')
| -rw-r--r-- | nix/libutil/affinity.cc | 55 | ||||
| -rw-r--r-- | nix/libutil/affinity.hh | 9 | ||||
| -rw-r--r-- | nix/libutil/archive.cc | 335 | ||||
| -rw-r--r-- | nix/libutil/archive.hh | 75 | ||||
| -rw-r--r-- | nix/libutil/hash.cc | 382 | ||||
| -rw-r--r-- | nix/libutil/hash.hh | 113 | ||||
| -rw-r--r-- | nix/libutil/serialise.cc | 259 | ||||
| -rw-r--r-- | nix/libutil/serialise.hh | 133 | ||||
| -rw-r--r-- | nix/libutil/types.hh | 86 | ||||
| -rw-r--r-- | nix/libutil/util.cc | 1105 | ||||
| -rw-r--r-- | nix/libutil/util.hh | 349 | ||||
| -rw-r--r-- | nix/libutil/xml-writer.cc | 94 | ||||
| -rw-r--r-- | nix/libutil/xml-writer.hh | 69 |
13 files changed, 3064 insertions, 0 deletions
diff --git a/nix/libutil/affinity.cc b/nix/libutil/affinity.cc new file mode 100644 index 0000000000..3e21f43a2e --- /dev/null +++ b/nix/libutil/affinity.cc @@ -0,0 +1,55 @@ +#include "types.hh" +#include "util.hh" +#include "affinity.hh" + +#if HAVE_SCHED_H +#include <sched.h> +#endif + +namespace nix { + + +#if HAVE_SCHED_SETAFFINITY +static bool didSaveAffinity = false; +static cpu_set_t savedAffinity; +#endif + + +void setAffinityTo(int cpu) +{ +#if HAVE_SCHED_SETAFFINITY + if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; + didSaveAffinity = true; + printMsg(lvlDebug, format("locking this thread to CPU %1%") % cpu); + cpu_set_t newAffinity; + CPU_ZERO(&newAffinity); + CPU_SET(cpu, &newAffinity); + if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) + printMsg(lvlError, format("failed to lock thread to CPU %1%") % cpu); +#endif +} + + +int lockToCurrentCPU() +{ +#if HAVE_SCHED_SETAFFINITY + int cpu = sched_getcpu(); + if (cpu != -1) setAffinityTo(cpu); + return cpu; +#else + return -1; +#endif +} + + +void restoreAffinity() +{ +#if HAVE_SCHED_SETAFFINITY + if (!didSaveAffinity) return; + if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) + printMsg(lvlError, "failed to restore affinity %1%"); +#endif +} + + +} diff --git a/nix/libutil/affinity.hh b/nix/libutil/affinity.hh new file mode 100644 index 0000000000..c1bd28e136 --- /dev/null +++ b/nix/libutil/affinity.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix { + +void setAffinityTo(int cpu); +int lockToCurrentCPU(); +void restoreAffinity(); + +} diff --git a/nix/libutil/archive.cc b/nix/libutil/archive.cc new file mode 100644 index 0000000000..ab4cd47351 --- /dev/null +++ b/nix/libutil/archive.cc @@ -0,0 +1,335 @@ +#include "config.h" + +#include <cerrno> +#include <algorithm> +#include <vector> + +#define _XOPEN_SOURCE 600 +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <dirent.h> +#include <fcntl.h> + +#include "archive.hh" +#include "util.hh" + + +namespace nix { + + +static string archiveVersion1 = "nix-archive-1"; + + +PathFilter defaultPathFilter; + + +static void dump(const string & path, Sink & sink, PathFilter & filter); + + +static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter) +{ + Strings names = readDirectory(path); + vector<string> names2(names.begin(), names.end()); + sort(names2.begin(), names2.end()); + + for (vector<string>::iterator i = names2.begin(); + i != names2.end(); ++i) + { + Path entry = path + "/" + *i; + if (filter(entry)) { + writeString("entry", sink); + writeString("(", sink); + writeString("name", sink); + writeString(*i, sink); + writeString("node", sink); + dump(entry, sink, filter); + writeString(")", sink); + } + } +} + + +static void dumpContents(const Path & path, size_t size, + Sink & sink) +{ + writeString("contents", sink); + writeLongLong(size, sink); + + AutoCloseFD fd = open(path.c_str(), O_RDONLY); + if (fd == -1) throw SysError(format("opening file `%1%'") % path); + + unsigned char buf[65536]; + size_t left = size; + + while (left > 0) { + size_t n = left > sizeof(buf) ? sizeof(buf) : left; + readFull(fd, buf, n); + left -= n; + sink(buf, n); + } + + writePadding(size, sink); +} + + +static void dump(const Path & path, Sink & sink, PathFilter & filter) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % path); + + writeString("(", sink); + + if (S_ISREG(st.st_mode)) { + writeString("type", sink); + writeString("regular", sink); + if (st.st_mode & S_IXUSR) { + writeString("executable", sink); + writeString("", sink); + } + dumpContents(path, (size_t) st.st_size, sink); + } + + else if (S_ISDIR(st.st_mode)) { + writeString("type", sink); + writeString("directory", sink); + dumpEntries(path, sink, filter); + } + + else if (S_ISLNK(st.st_mode)) { + writeString("type", sink); + writeString("symlink", sink); + writeString("target", sink); + writeString(readLink(path), sink); + } + + else throw Error(format("file `%1%' has an unknown type") % path); + + writeString(")", sink); +} + + +void dumpPath(const Path & path, Sink & sink, PathFilter & filter) +{ + writeString(archiveVersion1, sink); + dump(path, sink, filter); +} + + +static SerialisationError badArchive(string s) +{ + return SerialisationError("bad archive: " + s); +} + + +static void skipGeneric(Source & source) +{ + if (readString(source) == "(") { + while (readString(source) != ")") + skipGeneric(source); + } +} + + +static void parse(ParseSink & sink, Source & source, const Path & path); + + + +static void parseEntry(ParseSink & sink, Source & source, const Path & path) +{ + string s, name; + + s = readString(source); + if (s != "(") throw badArchive("expected open tag"); + + while (1) { + checkInterrupt(); + + s = readString(source); + + if (s == ")") { + break; + } else if (s == "name") { + name = readString(source); + } else if (s == "node") { + if (s == "") throw badArchive("entry name missing"); + parse(sink, source, path + "/" + name); + } else { + throw badArchive("unknown field " + s); + skipGeneric(source); + } + } +} + + +static void parseContents(ParseSink & sink, Source & source, const Path & path) +{ + unsigned long long size = readLongLong(source); + + sink.preallocateContents(size); + + unsigned long long left = size; + unsigned char buf[65536]; + + while (left) { + checkInterrupt(); + unsigned int n = sizeof(buf); + if ((unsigned long long) n > left) n = left; + source(buf, n); + sink.receiveContents(buf, n); + left -= n; + } + + readPadding(size, source); +} + + +static void parse(ParseSink & sink, Source & source, const Path & path) +{ + string s; + + s = readString(source); + if (s != "(") throw badArchive("expected open tag"); + + enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; + + while (1) { + checkInterrupt(); + + s = readString(source); + + if (s == ")") { + break; + } + + else if (s == "type") { + if (type != tpUnknown) + throw badArchive("multiple type fields"); + string t = readString(source); + + if (t == "regular") { + type = tpRegular; + sink.createRegularFile(path); + } + + else if (t == "directory") { + sink.createDirectory(path); + type = tpDirectory; + } + + else if (t == "symlink") { + type = tpSymlink; + } + + else throw badArchive("unknown file type " + t); + + } + + else if (s == "contents" && type == tpRegular) { + parseContents(sink, source, path); + } + + else if (s == "executable" && type == tpRegular) { + readString(source); + sink.isExecutable(); + } + + else if (s == "entry" && type == tpDirectory) { + parseEntry(sink, source, path); + } + + else if (s == "target" && type == tpSymlink) { + string target = readString(source); + sink.createSymlink(path, target); + } + + else { + throw badArchive("unknown field " + s); + skipGeneric(source); + } + } +} + + +void parseDump(ParseSink & sink, Source & source) +{ + string version; + try { + version = readString(source); + } catch (SerialisationError & e) { + /* This generally means the integer at the start couldn't be + decoded. Ignore and throw the exception below. */ + } + if (version != archiveVersion1) + throw badArchive("input doesn't look like a Nix archive"); + parse(sink, source, ""); +} + + +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + void createDirectory(const Path & path) + { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError(format("creating directory `%1%'") % p); + }; + + void createRegularFile(const Path & path) + { + Path p = dstPath + path; + fd.close(); + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY, 0666); + if (fd == -1) throw SysError(format("creating file `%1%'") % p); + } + + void isExecutable() + { + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("fstat"); + if (fchmod(fd, st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } + + void preallocateContents(unsigned long long len) + { +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd, 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL) + throw SysError(format("preallocating file of %1% bytes") % len); + } +#endif + } + + void receiveContents(unsigned char * data, unsigned int len) + { + writeFull(fd, data, len); + } + + void createSymlink(const Path & path, const string & target) + { + Path p = dstPath + path; + nix::createSymlink(target, p); + } +}; + + +void restorePath(const Path & path, Source & source) +{ + RestoreSink sink; + sink.dstPath = path; + parseDump(sink, source); +} + + +} diff --git a/nix/libutil/archive.hh b/nix/libutil/archive.hh new file mode 100644 index 0000000000..ccac92074d --- /dev/null +++ b/nix/libutil/archive.hh @@ -0,0 +1,75 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" + + +namespace nix { + + +/* dumpPath creates a Nix archive of the specified path. The format + is as follows: + + IF path points to a REGULAR FILE: + dump(path) = attrs( + [ ("type", "regular") + , ("contents", contents(path)) + ]) + + IF path points to a DIRECTORY: + dump(path) = attrs( + [ ("type", "directory") + , ("entries", concat(map(f, sort(entries(path))))) + ]) + where f(fn) = attrs( + [ ("name", fn) + , ("file", dump(path + "/" + fn)) + ]) + + where: + + attrs(as) = concat(map(attr, as)) + encN(0) + attrs((a, b)) = encS(a) + encS(b) + + encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) + + encN(n) = 64-bit little-endian encoding of n. + + contents(path) = the contents of a regular file. + + sort(strings) = lexicographic sort by 8-bit value (strcmp). + + entries(path) = the entries of a directory, without `.' and + `..'. + + `+' denotes string concatenation. */ + +struct PathFilter +{ + virtual ~PathFilter() { } + virtual bool operator () (const Path & path) { return true; } +}; + +extern PathFilter defaultPathFilter; + +void dumpPath(const Path & path, Sink & sink, + PathFilter & filter = defaultPathFilter); + +struct ParseSink +{ + virtual void createDirectory(const Path & path) { }; + + virtual void createRegularFile(const Path & path) { }; + virtual void isExecutable() { }; + virtual void preallocateContents(unsigned long long size) { }; + virtual void receiveContents(unsigned char * data, unsigned int len) { }; + + virtual void createSymlink(const Path & path, const string & target) { }; +}; + +void parseDump(ParseSink & sink, Source & source); + +void restorePath(const Path & path, Source & source); + + +} diff --git a/nix/libutil/hash.cc b/nix/libutil/hash.cc new file mode 100644 index 0000000000..050446610f --- /dev/null +++ b/nix/libutil/hash.cc @@ -0,0 +1,382 @@ +#include "config.h" + +#include <iostream> +#include <cstring> + +#ifdef HAVE_OPENSSL +#include <openssl/md5.h> +#include <openssl/sha.h> +#else +extern "C" { +#include "md5.h" +#include "sha1.h" +#include "sha256.h" +} +#endif + +#include "hash.hh" +#include "archive.hh" +#include "util.hh" + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + + +namespace nix { + + +Hash::Hash() +{ + type = htUnknown; + hashSize = 0; + memset(hash, 0, maxHashSize); +} + + +Hash::Hash(HashType type) +{ + this->type = type; + if (type == htMD5) hashSize = md5HashSize; + else if (type == htSHA1) hashSize = sha1HashSize; + else if (type == htSHA256) hashSize = sha256HashSize; + else throw Error("unknown hash type"); + assert(hashSize <= maxHashSize); + memset(hash, 0, maxHashSize); +} + + +bool Hash::operator == (const Hash & h2) const +{ + if (hashSize != h2.hashSize) return false; + for (unsigned int i = 0; i < hashSize; i++) + if (hash[i] != h2.hash[i]) return false; + return true; +} + + +bool Hash::operator != (const Hash & h2) const +{ + return !(*this == h2); +} + + +bool Hash::operator < (const Hash & h) const +{ + for (unsigned int i = 0; i < hashSize; i++) { + if (hash[i] < h.hash[i]) return true; + if (hash[i] > h.hash[i]) return false; + } + return false; +} + + +const string base16Chars = "0123456789abcdef"; + + +string printHash(const Hash & hash) +{ + char buf[hash.hashSize * 2]; + for (unsigned int i = 0; i < hash.hashSize; i++) { + buf[i * 2] = base16Chars[hash.hash[i] >> 4]; + buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f]; + } + return string(buf, hash.hashSize * 2); +} + + +Hash parseHash(HashType ht, const string & s) +{ + Hash hash(ht); + if (s.length() != hash.hashSize * 2) + throw Error(format("invalid hash `%1%'") % s); + for (unsigned int i = 0; i < hash.hashSize; i++) { + string s2(s, i * 2, 2); + if (!isxdigit(s2[0]) || !isxdigit(s2[1])) + throw Error(format("invalid hash `%1%'") % s); + std::istringstream str(s2); + int n; + str >> std::hex >> n; + hash.hash[i] = n; + } + return hash; +} + + +static unsigned char divMod(unsigned char * bytes, unsigned char y) +{ + unsigned int borrow = 0; + + int pos = Hash::maxHashSize - 1; + while (pos >= 0 && !bytes[pos]) --pos; + + for ( ; pos >= 0; --pos) { + unsigned int s = bytes[pos] + (borrow << 8); + unsigned int d = s / y; + borrow = s % y; + bytes[pos] = d; + } + + return borrow; +} + + +unsigned int hashLength32(const Hash & hash) +{ + return (hash.hashSize * 8 - 1) / 5 + 1; +} + + +// omitted: E O U T +const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; + + +string printHash32(const Hash & hash) +{ + Hash hash2(hash); + unsigned int len = hashLength32(hash); + + const char * chars = base32Chars.data(); + + string s(len, '0'); + + int pos = len - 1; + while (pos >= 0) { + unsigned char digit = divMod(hash2.hash, 32); + s[pos--] = chars[digit]; + } + + for (unsigned int i = 0; i < hash2.maxHashSize; ++i) + assert(hash2.hash[i] == 0); + + return s; +} + + +string printHash16or32(const Hash & hash) +{ + return hash.type == htMD5 ? printHash(hash) : printHash32(hash); +} + + +static bool mul(unsigned char * bytes, unsigned char y, int maxSize) +{ + unsigned char carry = 0; + + for (int pos = 0; pos < maxSize; ++pos) { + unsigned int m = bytes[pos] * y + carry; + bytes[pos] = m & 0xff; + carry = m >> 8; + } + + return carry; +} + + +static bool add(unsigned char * bytes, unsigned char y, int maxSize) +{ + unsigned char carry = y; + + for (int pos = 0; pos < maxSize; ++pos) { + unsigned int m = bytes[pos] + carry; + bytes[pos] = m & 0xff; + carry = m >> 8; + if (carry == 0) break; + } + + return carry; +} + + +Hash parseHash32(HashType ht, const string & s) +{ + Hash hash(ht); + + const char * chars = base32Chars.data(); + + for (unsigned int i = 0; i < s.length(); ++i) { + char c = s[i]; + unsigned char digit; + for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ + if (chars[digit] == c) break; + if (digit >= 32) + throw Error(format("invalid base-32 hash `%1%'") % s); + if (mul(hash.hash, 32, hash.hashSize) || + add(hash.hash, digit, hash.hashSize)) + throw Error(format("base-32 hash `%1%' is too large") % s); + } + + return hash; +} + + +Hash parseHash16or32(HashType ht, const string & s) +{ + Hash hash(ht); + if (s.size() == hash.hashSize * 2) + /* hexadecimal representation */ + hash = parseHash(ht, s); + else if (s.size() == hashLength32(hash)) + /* base-32 representation */ + hash = parseHash32(ht, s); + else + throw Error(format("hash `%1%' has wrong length for hash type `%2%'") + % s % printHashType(ht)); + return hash; +} + + +bool isHash(const string & s) +{ + if (s.length() != 32) return false; + for (int i = 0; i < 32; i++) { + char c = s[i]; + if (!((c >= '0' && c <= '9') || + (c >= 'a' && c <= 'f'))) + return false; + } + return true; +} + + +struct Ctx +{ + MD5_CTX md5; + SHA_CTX sha1; + SHA256_CTX sha256; +}; + + +static void start(HashType ht, Ctx & ctx) +{ + if (ht == htMD5) MD5_Init(&ctx.md5); + else if (ht == htSHA1) SHA1_Init(&ctx.sha1); + else if (ht == htSHA256) SHA256_Init(&ctx.sha256); +} + + +static void update(HashType ht, Ctx & ctx, + const unsigned char * bytes, unsigned int len) +{ + if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); + else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); + else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len); +} + + +static void finish(HashType ht, Ctx & ctx, unsigned char * hash) +{ + if (ht == htMD5) MD5_Final(hash, &ctx.md5); + else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1); + else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256); +} + + +Hash hashString(HashType ht, const string & s) +{ + Ctx ctx; + Hash hash(ht); + start(ht, ctx); + update(ht, ctx, (const unsigned char *) s.data(), s.length()); + finish(ht, ctx, hash.hash); + return hash; +} + + +Hash hashFile(HashType ht, const Path & path) +{ + Ctx ctx; + Hash hash(ht); + start(ht, ctx); + + AutoCloseFD fd = open(path.c_str(), O_RDONLY); + if (fd == -1) throw SysError(format("opening file `%1%'") % path); + + unsigned char buf[8192]; + ssize_t n; + while ((n = read(fd, buf, sizeof(buf)))) { + checkInterrupt(); + if (n == -1) throw SysError(format("reading file `%1%'") % path); + update(ht, ctx, buf, n); + } + + finish(ht, ctx, hash.hash); + return hash; +} + + +HashSink::HashSink(HashType ht) : ht(ht) +{ + ctx = new Ctx; + bytes = 0; + start(ht, *ctx); +} + +HashSink::~HashSink() +{ + bufPos = 0; + delete ctx; +} + +void HashSink::write(const unsigned char * data, size_t len) +{ + bytes += len; + update(ht, *ctx, data, len); +} + +HashResult HashSink::finish() +{ + flush(); + Hash hash(ht); + nix::finish(ht, *ctx, hash.hash); + return HashResult(hash, bytes); +} + +HashResult HashSink::currentHash() +{ + flush(); + Ctx ctx2 = *ctx; + Hash hash(ht); + nix::finish(ht, ctx2, hash.hash); + return HashResult(hash, bytes); +} + + +HashResult hashPath( + HashType ht, const Path & path, PathFilter & filter) +{ + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish(); +} + + +Hash compressHash(const Hash & hash, unsigned int newSize) +{ + Hash h; + h.hashSize = newSize; + for (unsigned int i = 0; i < hash.hashSize; ++i) + h.hash[i % newSize] ^= hash.hash[i]; + return h; +} + + +HashType parseHashType(const string & s) +{ + if (s == "md5") return htMD5; + else if (s == "sha1") return htSHA1; + else if (s == "sha256") return htSHA256; + else return htUnknown; +} + + +string printHashType(HashType ht) +{ + if (ht == htMD5) return "md5"; + else if (ht == htSHA1) return "sha1"; + else if (ht == htSHA256) return "sha256"; + else throw Error("cannot print unknown hash type"); +} + + +} diff --git a/nix/libutil/hash.hh b/nix/libutil/hash.hh new file mode 100644 index 0000000000..8f099c4f07 --- /dev/null +++ b/nix/libutil/hash.hh @@ -0,0 +1,113 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" + + +namespace nix { + + +typedef enum { htUnknown, htMD5, htSHA1, htSHA256 } HashType; + + +const int md5HashSize = 16; +const int sha1HashSize = 20; +const int sha256HashSize = 32; + +extern const string base32Chars; + + +struct Hash +{ + static const unsigned int maxHashSize = 32; + unsigned int hashSize; + unsigned char hash[maxHashSize]; + + HashType type; + + /* Create an unusable hash object. */ + Hash(); + + /* Create a zero-filled hash object. */ + Hash(HashType type); + + /* Check whether two hash are equal. */ + bool operator == (const Hash & h2) const; + + /* Check whether two hash are not equal. */ + bool operator != (const Hash & h2) const; + + /* For sorting. */ + bool operator < (const Hash & h) const; +}; + + +/* Convert a hash to a hexadecimal representation. */ +string printHash(const Hash & hash); + +/* Parse a hexadecimal representation of a hash code. */ +Hash parseHash(HashType ht, const string & s); + +/* Returns the length of a base-32 hash representation. */ +unsigned int hashLength32(const Hash & hash); + +/* Convert a hash to a base-32 representation. */ +string printHash32(const Hash & hash); + +/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ +string printHash16or32(const Hash & hash); + +/* Parse a base-32 representation of a hash code. */ +Hash parseHash32(HashType ht, const string & s); + +/* Parse a base-16 or base-32 representation of a hash code. */ +Hash parseHash16or32(HashType ht, const string & s); + +/* Verify that the given string is a valid hash code. */ +bool isHash(const string & s); + +/* Compute the hash of the given string. */ +Hash hashString(HashType ht, const string & s); + +/* Compute the hash of the given file. */ +Hash hashFile(HashType ht, const Path & path); + +/* Compute the hash of the given path. The hash is defined as + (essentially) hashString(ht, dumpPath(path)). */ +struct PathFilter; +extern PathFilter defaultPathFilter; +typedef std::pair<Hash, unsigned long long> HashResult; +HashResult hashPath(HashType ht, const Path & path, + PathFilter & filter = defaultPathFilter); + +/* Compress a hash to the specified number of bytes by cyclically + XORing bytes together. */ +Hash compressHash(const Hash & hash, unsigned int newSize); + +/* Parse a string representing a hash type. */ +HashType parseHashType(const string & s); + +/* And the reverse. */ +string printHashType(HashType ht); + + +struct Ctx; + +class HashSink : public BufferedSink +{ +private: + HashType ht; + Ctx * ctx; + unsigned long long bytes; + +public: + HashSink(HashType ht); + HashSink(const HashSink & h); + ~HashSink(); + void write(const unsigned char * data, size_t len); + HashResult finish(); + HashResult currentHash(); +}; + + +} diff --git a/nix/libutil/serialise.cc b/nix/libutil/serialise.cc new file mode 100644 index 0000000000..6b71f52c15 --- /dev/null +++ b/nix/libutil/serialise.cc @@ -0,0 +1,259 @@ +#include "serialise.hh" +#include "util.hh" + +#include <cstring> +#include <cerrno> + + +namespace nix { + + +BufferedSink::~BufferedSink() +{ + /* We can't call flush() here, because C++ for some insane reason + doesn't allow you to call virtual methods from a destructor. */ + assert(!bufPos); + delete[] buffer; +} + + +void BufferedSink::operator () (const unsigned char * data, size_t len) +{ + if (!buffer) buffer = new unsigned char[bufSize]; + + while (len) { + /* Optimisation: bypass the buffer if the data exceeds the + buffer size. */ + if (bufPos + len >= bufSize) { + flush(); + write(data, len); + break; + } + /* Otherwise, copy the bytes to the buffer. Flush the buffer + when it's full. */ + size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; + memcpy(buffer + bufPos, data, n); + data += n; bufPos += n; len -= n; + if (bufPos == bufSize) flush(); + } +} + + +void BufferedSink::flush() +{ + if (bufPos == 0) return; + size_t n = bufPos; + bufPos = 0; // don't trigger the assert() in ~BufferedSink() + write(buffer, n); +} + + +FdSink::~FdSink() +{ + try { flush(); } catch (...) { ignoreException(); } +} + + +void FdSink::write(const unsigned char * data, size_t len) +{ + writeFull(fd, data, len); +} + + +void Source::operator () (unsigned char * data, size_t len) +{ + while (len) { + size_t n = read(data, len); + data += n; len -= n; + } +} + + +BufferedSource::~BufferedSource() +{ + delete[] buffer; +} + + +size_t BufferedSource::read(unsigned char * data, size_t len) +{ + if (!buffer) buffer = new unsigned char[bufSize]; + + if (!bufPosIn) bufPosIn = readUnbuffered(buffer, bufSize); + + /* Copy out the data in the buffer. */ + size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; + memcpy(data, buffer + bufPosOut, n); + bufPosOut += n; + if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; + return n; +} + + +bool BufferedSource::hasData() +{ + return bufPosOut < bufPosIn; +} + + +size_t FdSource::readUnbuffered(unsigned char * data, size_t len) +{ + ssize_t n; + do { + checkInterrupt(); + n = ::read(fd, (char *) data, bufSize); + } while (n == -1 && errno == EINTR); + if (n == -1) throw SysError("reading from file"); + if (n == 0) throw EndOfFile("unexpected end-of-file"); + return n; +} + + +size_t StringSource::read(unsigned char * data, size_t len) +{ + if (pos == s.size()) throw EndOfFile("end of string reached"); + size_t n = s.copy((char *) data, len, pos); + pos += n; + return n; +} + + +void writePadding(size_t len, Sink & sink) +{ + if (len % 8) { + unsigned char zero[8]; + memset(zero, 0, sizeof(zero)); + sink(zero, 8 - (len % 8)); + } +} + + +void writeInt(unsigned int n, Sink & sink) +{ + unsigned char buf[8]; + memset(buf, 0, sizeof(buf)); + buf[0] = n & 0xff; + buf[1] = (n >> 8) & 0xff; + buf[2] = (n >> 16) & 0xff; + buf[3] = (n >> 24) & 0xff; + sink(buf, sizeof(buf)); +} + + +void writeLongLong(unsigned long long n, Sink & sink) +{ + unsigned char buf[8]; + buf[0] = n & 0xff; + buf[1] = (n >> 8) & 0xff; + buf[2] = (n >> 16) & 0xff; + buf[3] = (n >> 24) & 0xff; + buf[4] = (n >> 32) & 0xff; + buf[5] = (n >> 40) & 0xff; + buf[6] = (n >> 48) & 0xff; + buf[7] = (n >> 56) & 0xff; + sink(buf, sizeof(buf)); +} + + +void writeString(const unsigned char * buf, size_t len, Sink & sink) +{ + writeInt(len, sink); + sink(buf, len); + writePadding(len, sink); +} + + +void writeString(const string & s, Sink & sink) +{ + writeString((const unsigned char *) s.data(), s.size(), sink); +} + + +template<class T> void writeStrings(const T & ss, Sink & sink) +{ + writeInt(ss.size(), sink); + foreach (typename T::const_iterator, i, ss) + writeString(*i, sink); +} + +template void writeStrings(const Paths & ss, Sink & sink); +template void writeStrings(const PathSet & ss, Sink & sink); + + +void readPadding(size_t len, Source & source) +{ + if (len % 8) { + unsigned char zero[8]; + size_t n = 8 - (len % 8); + source(zero, n); + for (unsigned int i = 0; i < n; i++) + if (zero[i]) throw SerialisationError("non-zero padding"); + } +} + + +unsigned int readInt(Source & source) +{ + unsigned char buf[8]; + source(buf, sizeof(buf)); + if (buf[4] || buf[5] || buf[6] || buf[7]) + throw SerialisationError("implementation cannot deal with > 32-bit integers"); + return + buf[0] | + (buf[1] << 8) | + (buf[2] << 16) | + (buf[3] << 24); +} + + +unsigned long long readLongLong(Source & source) +{ + unsigned char buf[8]; + source(buf, sizeof(buf)); + return + ((unsigned long long) buf[0]) | + ((unsigned long long) buf[1] << 8) | + ((unsigned long long) buf[2] << 16) | + ((unsigned long long) buf[3] << 24) | + ((unsigned long long) buf[4] << 32) | + ((unsigned long long) buf[5] << 40) | + ((unsigned long long) buf[6] << 48) | + ((unsigned long long) buf[7] << 56); +} + + +size_t readString(unsigned char * buf, size_t max, Source & source) +{ + size_t len = readInt(source); + if (len > max) throw Error("string is too long"); + source(buf, len); + readPadding(len, source); + return len; +} + + +string readString(Source & source) +{ + size_t len = readInt(source); + unsigned char * buf = new unsigned char[len]; + AutoDeleteArray<unsigned char> d(buf); + source(buf, len); + readPadding(len, source); + return string((char *) buf, len); +} + + +template<class T> T readStrings(Source & source) +{ + unsigned int count = readInt(source); + T ss; + while (count--) + ss.insert(ss.end(), readString(source)); + return ss; +} + +template Paths readStrings(Source & source); +template PathSet readStrings(Source & source); + + +} diff --git a/nix/libutil/serialise.hh b/nix/libutil/serialise.hh new file mode 100644 index 0000000000..e5a9df1d05 --- /dev/null +++ b/nix/libutil/serialise.hh @@ -0,0 +1,133 @@ +#pragma once + +#include "types.hh" + + +namespace nix { + + +/* Abstract destination of binary data. */ +struct Sink +{ + virtual ~Sink() { } + virtual void operator () (const unsigned char * data, size_t len) = 0; +}; + + +/* A buffered abstract sink. */ +struct BufferedSink : Sink +{ + size_t bufSize, bufPos; + unsigned char * buffer; + + BufferedSink(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPos(0), buffer(0) { } + ~BufferedSink(); + + void operator () (const unsigned char * data, size_t len); + + void flush(); + + virtual void write(const unsigned char * data, size_t len) = 0; +}; + + +/* Abstract source of binary data. */ +struct Source +{ + virtual ~Source() { } + + /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. + It blocks until all the requested data is available, or throws + an error if it is not going to be available. */ + void operator () (unsigned char * data, size_t len); + + /* Store up to ‘len’ in the buffer pointed to by ‘data’, and + return the number of bytes stored. If blocks until at least + one byte is available. */ + virtual size_t read(unsigned char * data, size_t len) = 0; +}; + + +/* A buffered abstract source. */ +struct BufferedSource : Source +{ + size_t bufSize, bufPosIn, bufPosOut; + unsigned char * buffer; + + BufferedSource(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(0) { } + ~BufferedSource(); + + size_t read(unsigned char * data, size_t len); + + /* Underlying read call, to be overridden. */ + virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; + + bool hasData(); +}; + + +/* A sink that writes data to a file descriptor. */ +struct FdSink : BufferedSink +{ + int fd; + + FdSink() : fd(-1) { } + FdSink(int fd) : fd(fd) { } + ~FdSink(); + + void write(const unsigned char * data, size_t len); +}; + + +/* A source that reads data from a file descriptor. */ +struct FdSource : BufferedSource +{ + int fd; + FdSource() : fd(-1) { } + FdSource(int fd) : fd(fd) { } + size_t readUnbuffered(unsigned char * data, size_t len); +}; + + +/* A sink that writes data to a string. */ +struct StringSink : Sink +{ + string s; + void operator () (const unsigned char * data, size_t len) + { + s.append((const char *) data, len); + } +}; + + +/* A source that reads data from a string. */ +struct StringSource : Source +{ + const string & s; + size_t pos; + StringSource(const string & _s) : s(_s), pos(0) { } + size_t read(unsigned char * data, size_t len); +}; + + +void writePadding(size_t len, Sink & sink); +void writeInt(unsigned int n, Sink & sink); +void writeLongLong(unsigned long long n, Sink & sink); +void writeString(const unsigned char * buf, size_t len, Sink & sink); +void writeString(const string & s, Sink & sink); +template<class T> void writeStrings(const T & ss, Sink & sink); + +void readPadding(size_t len, Source & source); +unsigned int readInt(Source & source); +unsigned long long readLongLong(Source & source); +size_t readString(unsigned char * buf, size_t max, Source & source); +string readString(Source & source); +template<class T> T readStrings(Source & source); + + +MakeError(SerialisationError, Error) + + +} diff --git a/nix/libutil/types.hh b/nix/libutil/types.hh new file mode 100644 index 0000000000..4b5ce9a78c --- /dev/null +++ b/nix/libutil/types.hh @@ -0,0 +1,86 @@ +#pragma once + +#include "config.h" + +#include <string> +#include <list> +#include <set> + +#include <boost/format.hpp> + + +namespace nix { + + +/* Inherit some names from other namespaces for convenience. */ +using std::string; +using std::list; +using std::set; +using std::vector; +using boost::format; + + +struct FormatOrString +{ + string s; + FormatOrString(const string & s) : s(s) { }; + FormatOrString(const format & f) : s(f.str()) { }; + FormatOrString(const char * s) : s(s) { }; +}; + + +/* BaseError should generally not be caught, as it has Interrupted as + a subclass. Catch Error instead. */ +class BaseError : public std::exception +{ +protected: + string prefix_; // used for location traces etc. + string err; +public: + unsigned int status; // exit status + BaseError(const FormatOrString & fs, unsigned int status = 1); + ~BaseError() throw () { }; + const char * what() const throw () { return err.c_str(); } + const string & msg() const throw () { return err; } + const string & prefix() const throw () { return prefix_; } + BaseError & addPrefix(const FormatOrString & fs); +}; + +#define MakeError(newClass, superClass) \ + class newClass : public superClass \ + { \ + public: \ + newClass(const FormatOrString & fs, unsigned int status = 1) : superClass(fs, status) { }; \ + }; + +MakeError(Error, BaseError) + +class SysError : public Error +{ +public: + int errNo; + SysError(const FormatOrString & fs); +}; + + +typedef list<string> Strings; +typedef set<string> StringSet; + + +/* Paths are just strings. */ +typedef string Path; +typedef list<Path> Paths; +typedef set<Path> PathSet; + + +typedef enum { + lvlError = 0, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit +} Verbosity; + + +} diff --git a/nix/libutil/util.cc b/nix/libutil/util.cc new file mode 100644 index 0000000000..15c462ce4e --- /dev/null +++ b/nix/libutil/util.cc @@ -0,0 +1,1105 @@ +#include "config.h" + +#include <iostream> +#include <cerrno> +#include <cstdio> +#include <cstdlib> +#include <sstream> +#include <cstring> + +#include <sys/wait.h> +#include <unistd.h> +#include <fcntl.h> +#include <limits.h> + +#ifdef __APPLE__ +#include <sys/syscall.h> +#endif + +#include "util.hh" + + +extern char * * environ; + + +namespace nix { + + +BaseError::BaseError(const FormatOrString & fs, unsigned int status) + : status(status) +{ + err = fs.s; +} + + +BaseError & BaseError::addPrefix(const FormatOrString & fs) +{ + prefix_ = fs.s + prefix_; + return *this; +} + + +SysError::SysError(const FormatOrString & fs) + : Error(format("%1%: %2%") % fs.s % strerror(errno)) + , errNo(errno) +{ +} + + +string getEnv(const string & key, const string & def) +{ + char * value = getenv(key.c_str()); + return value ? string(value) : def; +} + + +Path absPath(Path path, Path dir) +{ + if (path[0] != '/') { + if (dir == "") { +#ifdef __GNU__ + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char *buf = getcwd(NULL, 0); + if (buf == NULL) +#else + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) +#endif + throw SysError("cannot get cwd"); + dir = buf; +#ifdef __GNU__ + free(buf); +#endif + } + path = dir + "/" + path; + } + return canonPath(path); +} + + +Path canonPath(const Path & path, bool resolveSymlinks) +{ + string s; + + if (path[0] != '/') + throw Error(format("not an absolute path: `%1%'") % path); + + string::const_iterator i = path.begin(), end = path.end(); + string temp; + + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0, maxFollow = 1024; + + while (1) { + + /* Skip slashes. */ + while (i != end && *i == '/') i++; + if (i == end) break; + + /* Ignore `.'. */ + if (*i == '.' && (i + 1 == end || i[1] == '/')) + i++; + + /* If `..', delete the last component. */ + else if (*i == '.' && i + 1 < end && i[1] == '.' && + (i + 2 == end || i[2] == '/')) + { + if (!s.empty()) s.erase(s.rfind('/')); + i += 2; + } + + /* Normal component; copy it. */ + else { + s += '/'; + while (i != end && *i != '/') s += *i++; + + /* If s points to a symlink, resolve it and restart (since + the symlink target might contain new symlinks). */ + if (resolveSymlinks && isLink(s)) { + if (++followCount >= maxFollow) + throw Error(format("infinite symlink recursion in path `%1%'") % path); + temp = absPath(readLink(s), dirOf(s)) + + string(i, end); + i = temp.begin(); /* restart */ + end = temp.end(); + s = ""; + /* !!! potential for infinite loop */ + } + } + } + + return s.empty() ? "/" : s; +} + + +Path dirOf(const Path & path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == string::npos) + throw Error(format("invalid file name `%1%'") % path); + return pos == 0 ? "/" : Path(path, 0, pos); +} + + +string baseNameOf(const Path & path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == string::npos) + throw Error(format("invalid file name `%1%'") % path); + return string(path, pos + 1); +} + + +bool isInDir(const Path & path, const Path & dir) +{ + return path[0] == '/' + && string(path, 0, dir.size()) == dir + && path.size() >= dir.size() + 2 + && path[dir.size()] == '/'; +} + + +struct stat lstat(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of `%1%'") % path); + return st; +} + + +bool pathExists(const Path & path) +{ + int res; + struct stat st; + res = lstat(path.c_str(), &st); + if (!res) return true; + if (errno != ENOENT && errno != ENOTDIR) + throw SysError(format("getting status of %1%") % path); + return false; +} + + +Path readLink(const Path & path) +{ + checkInterrupt(); + struct stat st = lstat(path); + if (!S_ISLNK(st.st_mode)) + throw Error(format("`%1%' is not a symlink") % path); + char buf[st.st_size]; + if (readlink(path.c_str(), buf, st.st_size) != st.st_size) + throw SysError(format("reading symbolic link `%1%'") % path); + return string(buf, st.st_size); +} + + +bool isLink(const Path & path) +{ + struct stat st = lstat(path); + return S_ISLNK(st.st_mode); +} + + +Strings readDirectory(const Path & path) +{ + Strings names; + + AutoCloseDir dir = opendir(path.c_str()); + if (!dir) throw SysError(format("opening directory `%1%'") % path); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { /* sic */ + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + names.push_back(name); + } + if (errno) throw SysError(format("reading directory `%1%'") % path); + + return names; +} + + +string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + unsigned char * buf = new unsigned char[st.st_size]; + AutoDeleteArray<unsigned char> d(buf); + readFull(fd, buf, st.st_size); + + return string((char *) buf, st.st_size); +} + + +string readFile(const Path & path, bool drain) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY); + if (fd == -1) + throw SysError(format("opening file `%1%'") % path); + return drain ? drainFD(fd) : readFile(fd); +} + + +void writeFile(const Path & path, const string & s) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666); + if (fd == -1) + throw SysError(format("opening file `%1%'") % path); + writeFull(fd, (unsigned char *) s.data(), s.size()); +} + + +string readLine(int fd) +{ + string s; + while (1) { + checkInterrupt(); + char ch; + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void writeLine(int fd, string s) +{ + s += '\n'; + writeFull(fd, (const unsigned char *) s.data(), s.size()); +} + + +static void _deletePath(const Path & path, unsigned long long & bytesFreed) +{ + checkInterrupt(); + + printMsg(lvlVomit, format("%1%") % path); + + struct stat st = lstat(path); + + if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) + bytesFreed += st.st_blocks * 512; + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + + /* Make the directory writable. */ + if (!(st.st_mode & S_IWUSR)) { + if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) + throw SysError(format("making `%1%' writable") % path); + } + + for (Strings::iterator i = names.begin(); i != names.end(); ++i) + _deletePath(path + "/" + *i, bytesFreed); + } + + if (remove(path.c_str()) == -1) + throw SysError(format("cannot unlink `%1%'") % path); +} + + +void deletePath(const Path & path) +{ + unsigned long long dummy; + deletePath(path, dummy); +} + + +void deletePath(const Path & path, unsigned long long & bytesFreed) +{ + startNest(nest, lvlDebug, + format("recursively deleting path `%1%'") % path); + bytesFreed = 0; + _deletePath(path, bytesFreed); +} + + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + int & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR", "/tmp") : tmpRoot, true); + if (includePid) + return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); + else + return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); +} + + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static int globalCounter = 0; + int localCounter = 0; + int & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError(format("setting group of directory `%1%'") % tmpDir); + return tmpDir; + } + if (errno != EEXIST) + throw SysError(format("creating directory `%1%'") % tmpDir); + } +} + + +Paths createDirs(const Path & path) +{ + Paths created; + if (path == "/") return created; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + created = createDirs(dirOf(path)); + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) + throw SysError(format("creating directory `%1%'") % path); + st = lstat(path); + created.push_back(path); + } + + if (!S_ISDIR(st.st_mode)) throw Error(format("`%1%' is not a directory") % path); + + return created; +} + + +void createSymlink(const Path & target, const Path & link) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError(format("creating symlink from `%1%' to `%2%'") % link % target); +} + + +LogType logType = ltPretty; +Verbosity verbosity = lvlInfo; + +static int nestingLevel = 0; + + +Nest::Nest() +{ + nest = false; +} + + +Nest::~Nest() +{ + close(); +} + + +static string escVerbosity(Verbosity level) +{ + return int2String((int) level); +} + + +void Nest::open(Verbosity level, const FormatOrString & fs) +{ + if (level <= verbosity) { + if (logType == ltEscapes) + std::cerr << "\033[" << escVerbosity(level) << "p" + << fs.s << "\n"; + else + printMsg_(level, fs); + nest = true; + nestingLevel++; + } +} + + +void Nest::close() +{ + if (nest) { + nestingLevel--; + if (logType == ltEscapes) + std::cerr << "\033[q"; + nest = false; + } +} + + +void printMsg_(Verbosity level, const FormatOrString & fs) +{ + checkInterrupt(); + if (level > verbosity) return; + string prefix; + if (logType == ltPretty) + for (int i = 0; i < nestingLevel; i++) + prefix += "| "; + else if (logType == ltEscapes && level != lvlInfo) + prefix = "\033[" + escVerbosity(level) + "s"; + string s = (format("%1%%2%\n") % prefix % fs.s).str(); + writeToStderr(s); +} + + +void warnOnce(bool & haveWarned, const FormatOrString & fs) +{ + if (!haveWarned) { + printMsg(lvlError, format("warning: %1%") % fs.s); + haveWarned = true; + } +} + + +void writeToStderr(const string & s) +{ + try { + _writeToStderr((const unsigned char *) s.data(), s.size()); + } catch (SysError & e) { + /* Ignore failing writes to stderr if we're in an exception + handler, otherwise throw an exception. We need to ignore + write errors in exception handlers to ensure that cleanup + code runs to completion if the other side of stderr has + been closed unexpectedly. */ + if (!std::uncaught_exception()) throw; + } +} + + +static void defaultWriteToStderr(const unsigned char * buf, size_t count) +{ + writeFull(STDERR_FILENO, buf, count); +} + + +void (*_writeToStderr) (const unsigned char * buf, size_t count) = defaultWriteToStderr; + + +void readFull(int fd, unsigned char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, (char *) buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, const unsigned char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = write(fd, (char *) buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("writing to file"); + } + count -= res; + buf += res; + } +} + + +string drainFD(int fd) +{ + string result; + unsigned char buffer[4096]; + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buffer, sizeof buffer); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else result.append((char *) buffer, rd); + } + return result; +} + + + +////////////////////////////////////////////////////////////////////// + + +AutoDelete::AutoDelete(const string & p, bool recursive) : path(p) +{ + del = true; + this->recursive = recursive; +} + +AutoDelete::~AutoDelete() +{ + try { + if (del) { + if (recursive) + deletePath(path); + else { + if (remove(path.c_str()) == -1) + throw SysError(format("cannot unlink `%1%'") % path); + } + } + } catch (...) { + ignoreException(); + } +} + +void AutoDelete::cancel() +{ + del = false; +} + + + +////////////////////////////////////////////////////////////////////// + + +AutoCloseFD::AutoCloseFD() +{ + fd = -1; +} + + +AutoCloseFD::AutoCloseFD(int fd) +{ + this->fd = fd; +} + + +AutoCloseFD::AutoCloseFD(const AutoCloseFD & fd) +{ + /* Copying an AutoCloseFD isn't allowed (who should get to close + it?). But as an edge case, allow copying of closed + AutoCloseFDs. This is necessary due to tiresome reasons + involving copy constructor use on default object values in STL + containers (like when you do `map[value]' where value isn't in + the map yet). */ + this->fd = fd.fd; + if (this->fd != -1) abort(); +} + + +AutoCloseFD::~AutoCloseFD() +{ + try { + close(); + } catch (...) { + ignoreException(); + } +} + + +void AutoCloseFD::operator =(int fd) +{ + if (this->fd != fd) close(); + this->fd = fd; +} + + +AutoCloseFD::operator int() const +{ + return fd; +} + + +void AutoCloseFD::close() +{ + if (fd != -1) { + if (::close(fd) == -1) + /* This should never happen. */ + throw SysError(format("closing file descriptor %1%") % fd); + fd = -1; + } +} + + +bool AutoCloseFD::isOpen() +{ + return fd != -1; +} + + +/* Pass responsibility for closing this fd to the caller. */ +int AutoCloseFD::borrow() +{ + int oldFD = fd; + fd = -1; + return oldFD; +} + + +void Pipe::create() +{ + int fds[2]; + if (pipe(fds) != 0) throw SysError("creating pipe"); + readSide = fds[0]; + writeSide = fds[1]; + closeOnExec(readSide); + closeOnExec(writeSide); +} + + + +////////////////////////////////////////////////////////////////////// + + +AutoCloseDir::AutoCloseDir() +{ + dir = 0; +} + + +AutoCloseDir::AutoCloseDir(DIR * dir) +{ + this->dir = dir; +} + + +AutoCloseDir::~AutoCloseDir() +{ + close(); +} + + +void AutoCloseDir::operator =(DIR * dir) +{ + this->dir = dir; +} + + +AutoCloseDir::operator DIR *() +{ + return dir; +} + + +void AutoCloseDir::close() +{ + if (dir) { + closedir(dir); + dir = 0; + } +} + + +////////////////////////////////////////////////////////////////////// + + +Pid::Pid() +{ + pid = -1; + separatePG = false; + killSignal = SIGKILL; +} + + +Pid::~Pid() +{ + kill(); +} + + +void Pid::operator =(pid_t pid) +{ + if (this->pid != pid) kill(); + this->pid = pid; + killSignal = SIGKILL; // reset signal to default +} + + +Pid::operator pid_t() +{ + return pid; +} + + +void Pid::kill() +{ + if (pid == -1 || pid == 0) return; + + printMsg(lvlError, format("killing process %1%") % pid); + + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) + printMsg(lvlError, (SysError(format("killing process %1%") % pid).msg())); + + /* Wait until the child dies, disregarding the exit status. */ + int status; + while (waitpid(pid, &status, 0) == -1) { + checkInterrupt(); + if (errno != EINTR) { + printMsg(lvlError, + (SysError(format("waiting for process %1%") % pid).msg())); + break; + } + } + + pid = -1; +} + + +int Pid::wait(bool block) +{ + assert(pid != -1); + while (1) { + int status; + int res = waitpid(pid, &status, block ? 0 : WNOHANG); + if (res == pid) { + pid = -1; + return status; + } + if (res == 0 && !block) return -1; + if (errno != EINTR) + throw SysError("cannot get child exit status"); + checkInterrupt(); + } +} + + +void Pid::setSeparatePG(bool separatePG) +{ + this->separatePG = separatePG; +} + + +void Pid::setKillSignal(int signal) +{ + this->killSignal = signal; +} + + +void killUser(uid_t uid) +{ + debug(format("killing all processes running under uid `%1%'") % uid); + + assert(uid != 0); /* just to be safe... */ + + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ + + Pid pid; + pid = fork(); + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: + try { /* child */ + + if (setuid(uid) == -1) + throw SysError("setting uid"); + + while (true) { +#ifdef __APPLE__ + /* OSX's kill syscall takes a third parameter that, among other + things, determines if kill(-1, signo) affects the calling + process. In the OSX libc, it's set to true, which means + "follow POSIX", which we don't want here + */ + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; +#else + if (kill(-1, SIGKILL) == 0) break; +#endif + if (errno == ESRCH) break; /* no more processes */ + if (errno != EINTR) + throw SysError(format("cannot kill processes for uid `%1%'") % uid); + } + + } catch (std::exception & e) { + writeToStderr((format("killing processes belonging to uid `%1%': %2%\n") % uid % e.what()).str()); + _exit(1); + } + _exit(0); + } + + /* parent */ + int status = pid.wait(true); + if (status != 0) + throw Error(format("cannot kill processes for uid `%1%': %2%") % uid % statusToString(status)); + + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ +} + + +////////////////////////////////////////////////////////////////////// + + +string runProgram(Path program, bool searchPath, const Strings & args) +{ + checkInterrupt(); + + std::vector<const char *> cargs; /* careful with c_str()! */ + cargs.push_back(program.c_str()); + for (Strings::const_iterator i = args.begin(); i != args.end(); ++i) + cargs.push_back(i->c_str()); + cargs.push_back(0); + + /* Create a pipe. */ + Pipe pipe; + pipe.create(); + + /* Fork. */ + Pid pid; + pid = maybeVfork(); + + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: /* child */ + try { + if (dup2(pipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + + if (searchPath) + execvp(program.c_str(), (char * *) &cargs[0]); + else + execv(program.c_str(), (char * *) &cargs[0]); + throw SysError(format("executing `%1%'") % program); + + } catch (std::exception & e) { + writeToStderr("error: " + string(e.what()) + "\n"); + } + _exit(1); + } + + /* Parent. */ + + pipe.writeSide.close(); + + string result = drainFD(pipe.readSide); + + /* Wait for the child to finish. */ + int status = pid.wait(true); + if (!statusOk(status)) + throw Error(format("program `%1%' %2%") + % program % statusToString(status)); + + return result; +} + + +void closeMostFDs(const set<int> & exceptions) +{ + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO + && exceptions.find(fd) == exceptions.end()) + close(fd); /* ignore result */ +} + + +void closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + + +#if HAVE_VFORK +pid_t (*maybeVfork)() = vfork; +#else +pid_t (*maybeVfork)() = fork; +#endif + + +////////////////////////////////////////////////////////////////////// + + +volatile sig_atomic_t _isInterrupted = 0; + +void _interrupted() +{ + /* Block user interrupts while an exception is being handled. + Throwing an exception while another exception is being handled + kills the program! */ + if (!std::uncaught_exception()) { + _isInterrupted = 0; + throw Interrupted("interrupted by the user"); + } +} + + + +////////////////////////////////////////////////////////////////////// + + +template<class C> C tokenizeString(const string & s, const string & separators) +{ + C result; + string::size_type pos = s.find_first_not_of(separators, 0); + while (pos != string::npos) { + string::size_type end = s.find_first_of(separators, pos + 1); + if (end == string::npos) end = s.size(); + string token(s, pos, end - pos); + result.insert(result.end(), token); + pos = s.find_first_not_of(separators, end); + } + return result; +} + +template Strings tokenizeString(const string & s, const string & separators); +template StringSet tokenizeString(const string & s, const string & separators); +template vector<string> tokenizeString(const string & s, const string & separators); + + +string concatStringsSep(const string & sep, const Strings & ss) +{ + string s; + foreach (Strings::const_iterator, i, ss) { + if (s.size() != 0) s += sep; + s += *i; + } + return s; +} + + +string concatStringsSep(const string & sep, const StringSet & ss) +{ + string s; + foreach (StringSet::const_iterator, i, ss) { + if (s.size() != 0) s += sep; + s += *i; + } + return s; +} + + +string chomp(const string & s) +{ + size_t i = s.find_last_not_of(" \n\r\t"); + return i == string::npos ? "" : string(s, 0, i + 1); +} + + +string statusToString(int status) +{ + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) + return (format("failed with exit code %1%") % WEXITSTATUS(status)).str(); + else if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); +#if HAVE_STRSIGNAL + const char * description = strsignal(sig); + return (format("failed due to signal %1% (%2%)") % sig % description).str(); +#else + return (format("failed due to signal %1%") % sig).str(); +#endif + } + else + return "died abnormally"; + } else return "succeeded"; +} + + +bool statusOk(int status) +{ + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + + +bool hasSuffix(const string & s, const string & suffix) +{ + return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix; +} + + +void expect(std::istream & str, const string & s) +{ + char s2[s.size()]; + str.read(s2, s.size()); + if (string(s2, s.size()) != s) + throw Error(format("expected string `%1%'") % s); +} + + +string parseString(std::istream & str) +{ + string res; + expect(str, "\""); + int c; + while ((c = str.get()) != '"') + if (c == '\\') { + c = str.get(); + if (c == 'n') res += '\n'; + else if (c == 'r') res += '\r'; + else if (c == 't') res += '\t'; + else res += c; + } + else res += c; + return res; +} + + +bool endOfList(std::istream & str) +{ + if (str.peek() == ',') { + str.get(); + return false; + } + if (str.peek() == ']') { + str.get(); + return true; + } + return false; +} + + +string decodeOctalEscaped(const string & s) +{ + string r; + for (string::const_iterator i = s.begin(); i != s.end(); ) { + if (*i != '\\') { r += *i++; continue; } + unsigned char c = 0; + ++i; + while (i != s.end() && *i >= '0' && *i < '8') + c = c * 8 + (*i++ - '0'); + r += c; + } + return r; +} + + +void ignoreException() +{ + try { + throw; + } catch (std::exception & e) { + printMsg(lvlError, format("error (ignored): %1%") % e.what()); + } +} + + +} diff --git a/nix/libutil/util.hh b/nix/libutil/util.hh new file mode 100644 index 0000000000..8bedfea9a0 --- /dev/null +++ b/nix/libutil/util.hh @@ -0,0 +1,349 @@ +#pragma once + +#include "types.hh" + +#include <sys/types.h> +#include <sys/stat.h> +#include <dirent.h> +#include <unistd.h> +#include <signal.h> + +#include <cstdio> + + +namespace nix { + + +#define foreach(it_type, it, collection) \ + for (it_type it = (collection).begin(); it != (collection).end(); ++it) + +#define foreach_reverse(it_type, it, collection) \ + for (it_type it = (collection).rbegin(); it != (collection).rend(); ++it) + + +/* Return an environment variable. */ +string getEnv(const string & key, const string & def = ""); + +/* Return an absolutized path, resolving paths relative to the + specified directory, or the current directory otherwise. The path + is also canonicalised. */ +Path absPath(Path path, Path dir = ""); + +/* Canonicalise a path by removing all `.' or `..' components and + double or trailing slashes. Optionally resolves all symlink + components such that each component of the resulting path is *not* + a symbolic link. */ +Path canonPath(const Path & path, bool resolveSymlinks = false); + +/* Return the directory part of the given canonical path, i.e., + everything before the final `/'. If the path is the root or an + immediate child thereof (e.g., `/foo'), this means an empty string + is returned. */ +Path dirOf(const Path & path); + +/* Return the base name of the given canonical path, i.e., everything + following the final `/'. */ +string baseNameOf(const Path & path); + +/* Check whether a given path is a descendant of the given + directory. */ +bool isInDir(const Path & path, const Path & dir); + +/* Get status of `path'. */ +struct stat lstat(const Path & path); + +/* Return true iff the given path exists. */ +bool pathExists(const Path & path); + +/* Read the contents (target) of a symbolic link. The result is not + in any way canonicalised. */ +Path readLink(const Path & path); + +bool isLink(const Path & path); + +/* Read the contents of a directory. The entries `.' and `..' are + removed. */ +Strings readDirectory(const Path & path); + +/* Read the contents of a file into a string. */ +string readFile(int fd); +string readFile(const Path & path, bool drain = false); + +/* Write a string to a file. */ +void writeFile(const Path & path, const string & s); + +/* Read a line from a file descriptor. */ +string readLine(int fd); + +/* Write a line to a file descriptor. */ +void writeLine(int fd, string s); + +/* Delete a path; i.e., in the case of a directory, it is deleted + recursively. Don't use this at home, kids. The second variant + returns the number of bytes and blocks freed. */ +void deletePath(const Path & path); + +void deletePath(const Path & path, unsigned long long & bytesFreed); + +/* Create a temporary directory. */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/* Create a directory and all its parents, if necessary. Returns the + list of created directories, in order of creation. */ +Paths createDirs(const Path & path); + +/* Create a symlink. */ +void createSymlink(const Path & target, const Path & link); + + +template<class T, class A> +T singleton(const A & a) +{ + T t; + t.insert(a); + return t; +} + + +/* Messages. */ + + +typedef enum { + ltPretty, /* nice, nested output */ + ltEscapes, /* nesting indicated using escape codes (for log2xml) */ + ltFlat /* no nesting */ +} LogType; + +extern LogType logType; +extern Verbosity verbosity; /* suppress msgs > this */ + +class Nest +{ +private: + bool nest; +public: + Nest(); + ~Nest(); + void open(Verbosity level, const FormatOrString & fs); + void close(); +}; + +void printMsg_(Verbosity level, const FormatOrString & fs); + +#define startNest(varName, level, f) \ + Nest varName; \ + if (level <= verbosity) { \ + varName.open(level, (f)); \ + } + +#define printMsg(level, f) \ + do { \ + if (level <= verbosity) { \ + printMsg_(level, (f)); \ + } \ + } while (0) + +#define debug(f) printMsg(lvlDebug, f) + +void warnOnce(bool & haveWarned, const FormatOrString & fs); + +void writeToStderr(const string & s); + +extern void (*_writeToStderr) (const unsigned char * buf, size_t count); + + +/* Wrappers arount read()/write() that read/write exactly the + requested number of bytes. */ +void readFull(int fd, unsigned char * buf, size_t count); +void writeFull(int fd, const unsigned char * buf, size_t count); + +MakeError(EndOfFile, Error) + + +/* Read a file descriptor until EOF occurs. */ +string drainFD(int fd); + + + +/* Automatic cleanup of resources. */ + + +template <class T> +struct AutoDeleteArray +{ + T * p; + AutoDeleteArray(T * p) : p(p) { } + ~AutoDeleteArray() + { + delete [] p; + } +}; + + +class AutoDelete +{ + Path path; + bool del; + bool recursive; +public: + AutoDelete(const Path & p, bool recursive = true); + ~AutoDelete(); + void cancel(); +}; + + +class AutoCloseFD +{ + int fd; +public: + AutoCloseFD(); + AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD & fd); + ~AutoCloseFD(); + void operator =(int fd); + operator int() const; + void close(); + bool isOpen(); + int borrow(); +}; + + +class Pipe +{ +public: + AutoCloseFD readSide, writeSide; + void create(); +}; + + +class AutoCloseDir +{ + DIR * dir; +public: + AutoCloseDir(); + AutoCloseDir(DIR * dir); + ~AutoCloseDir(); + void operator =(DIR * dir); + operator DIR *(); + void close(); +}; + + +class Pid +{ + pid_t pid; + bool separatePG; + int killSignal; +public: + Pid(); + ~Pid(); + void operator =(pid_t pid); + operator pid_t(); + void kill(); + int wait(bool block); + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); +}; + + +/* Kill all processes running under the specified uid by sending them + a SIGKILL. */ +void killUser(uid_t uid); + + +/* Run a program and return its stdout in a string (i.e., like the + shell backtick operator). */ +string runProgram(Path program, bool searchPath = false, + const Strings & args = Strings()); + +/* Close all file descriptors except stdin, stdout, stderr, and those + listed in the given set. Good practice in child processes. */ +void closeMostFDs(const set<int> & exceptions); + +/* Set the close-on-exec flag for the given file descriptor. */ +void closeOnExec(int fd); + +/* Call vfork() if available, otherwise fork(). */ +extern pid_t (*maybeVfork)(); + + +/* User interruption. */ + +extern volatile sig_atomic_t _isInterrupted; + +void _interrupted(); + +void inline checkInterrupt() +{ + if (_isInterrupted) _interrupted(); +} + +MakeError(Interrupted, BaseError) + + +/* String tokenizer. */ +template<class C> C tokenizeString(const string & s, const string & separators = " \t\n\r"); + + +/* Concatenate the given strings with a separator between the + elements. */ +string concatStringsSep(const string & sep, const Strings & ss); +string concatStringsSep(const string & sep, const StringSet & ss); + + +/* Remove trailing whitespace from a string. */ +string chomp(const string & s); + + +/* Convert the exit status of a child as returned by wait() into an + error string. */ +string statusToString(int status); + +bool statusOk(int status); + + +/* Parse a string into an integer. */ +template<class N> bool string2Int(const string & s, N & n) +{ + std::istringstream str(s); + str >> n; + return str && str.get() == EOF; +} + +template<class N> string int2String(N n) +{ + std::ostringstream str; + str << n; + return str.str(); +} + + +/* Return true iff `s' ends in `suffix'. */ +bool hasSuffix(const string & s, const string & suffix); + + +/* Read string `s' from stream `str'. */ +void expect(std::istream & str, const string & s); + + +/* Read a C-style string from stream `str'. */ +string parseString(std::istream & str); + + +/* Utility function used to parse legacy ATerms. */ +bool endOfList(std::istream & str); + + +/* Escape a string that contains octal-encoded escape codes such as + used in /etc/fstab and /proc/mounts (e.g. "foo\040bar" decodes to + "foo bar"). */ +string decodeOctalEscaped(const string & s); + + +/* Exception handling in destructors: print an error message, then + ignore the exception. */ +void ignoreException(); + + +} diff --git a/nix/libutil/xml-writer.cc b/nix/libutil/xml-writer.cc new file mode 100644 index 0000000000..01794001b2 --- /dev/null +++ b/nix/libutil/xml-writer.cc @@ -0,0 +1,94 @@ +#include <assert.h> + +#include "xml-writer.hh" + + +namespace nix { + + +XMLWriter::XMLWriter(bool indent, std::ostream & output) + : output(output), indent(indent) +{ + output << "<?xml version='1.0' encoding='utf-8'?>" << std::endl; + closed = false; +} + + +XMLWriter::~XMLWriter() +{ + close(); +} + + +void XMLWriter::close() +{ + if (closed) return; + while (!pendingElems.empty()) closeElement(); + closed = true; +} + + +void XMLWriter::indent_(unsigned int depth) +{ + if (!indent) return; + output << string(depth * 2, ' '); +} + + +void XMLWriter::openElement(const string & name, + const XMLAttrs & attrs) +{ + assert(!closed); + indent_(pendingElems.size()); + output << "<" << name; + writeAttrs(attrs); + output << ">"; + if (indent) output << std::endl; + pendingElems.push_back(name); +} + + +void XMLWriter::closeElement() +{ + assert(!pendingElems.empty()); + indent_(pendingElems.size() - 1); + output << "</" << pendingElems.back() << ">"; + if (indent) output << std::endl; + pendingElems.pop_back(); + if (pendingElems.empty()) closed = true; +} + + +void XMLWriter::writeEmptyElement(const string & name, + const XMLAttrs & attrs) +{ + assert(!closed); + indent_(pendingElems.size()); + output << "<" << name; + writeAttrs(attrs); + output << " />"; + if (indent) output << std::endl; +} + + +void XMLWriter::writeAttrs(const XMLAttrs & attrs) +{ + for (XMLAttrs::const_iterator i = attrs.begin(); i != attrs.end(); ++i) { + output << " " << i->first << "=\""; + for (unsigned int j = 0; j < i->second.size(); ++j) { + char c = i->second[j]; + if (c == '"') output << """; + else if (c == '<') output << "<"; + else if (c == '>') output << ">"; + else if (c == '&') output << "&"; + /* Escape newlines to prevent attribute normalisation (see + XML spec, section 3.3.3. */ + else if (c == '\n') output << "
"; + else output << c; + } + output << "\""; + } +} + + +} diff --git a/nix/libutil/xml-writer.hh b/nix/libutil/xml-writer.hh new file mode 100644 index 0000000000..3cefe3712c --- /dev/null +++ b/nix/libutil/xml-writer.hh @@ -0,0 +1,69 @@ +#pragma once + +#include <iostream> +#include <string> +#include <list> +#include <map> + + +namespace nix { + +using std::string; +using std::map; +using std::list; + + +typedef map<string, string> XMLAttrs; + + +class XMLWriter +{ +private: + + std::ostream & output; + + bool indent; + bool closed; + + list<string> pendingElems; + +public: + + XMLWriter(bool indent, std::ostream & output); + ~XMLWriter(); + + void close(); + + void openElement(const string & name, + const XMLAttrs & attrs = XMLAttrs()); + void closeElement(); + + void writeEmptyElement(const string & name, + const XMLAttrs & attrs = XMLAttrs()); + +private: + void writeAttrs(const XMLAttrs & attrs); + + void indent_(unsigned int depth); +}; + + +class XMLOpenElement +{ +private: + XMLWriter & writer; +public: + XMLOpenElement(XMLWriter & writer, const string & name, + const XMLAttrs & attrs = XMLAttrs()) + : writer(writer) + { + writer.openElement(name, attrs); + } + ~XMLOpenElement() + { + writer.closeElement(); + } +}; + + +} |