diff options
Diffstat (limited to 'nix/libstore')
| -rw-r--r-- | nix/libstore/local-store.cc | 468 | ||||
| -rw-r--r-- | nix/libstore/local-store.hh | 42 | ||||
| -rw-r--r-- | nix/libstore/optimise-store.cc | 23 | ||||
| -rw-r--r-- | nix/libstore/sqlite.cc | 166 | ||||
| -rw-r--r-- | nix/libstore/sqlite.hh | 102 | ||||
| -rw-r--r-- | nix/libstore/store-api.hh | 4 |
6 files changed, 392 insertions, 413 deletions
diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc index 347e8a703f..882bce1f40 100644 --- a/nix/libstore/local-store.cc +++ b/nix/libstore/local-store.cc @@ -40,171 +40,6 @@ namespace nix { -MakeError(SQLiteError, Error); -MakeError(SQLiteBusy, SQLiteError); - - -static void throwSQLiteError(sqlite3 * db, const format & f) - __attribute__ ((noreturn)); - -static void throwSQLiteError(sqlite3 * db, const format & f) -{ - int err = sqlite3_errcode(db); - if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - if (err == SQLITE_PROTOCOL) - printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); - else { - static bool warned = false; - if (!warned) { - printMsg(lvlError, "warning: SQLite database is busy"); - warned = true; - } - } - /* Sleep for a while since retrying the transaction right away - is likely to fail again. */ -#if HAVE_NANOSLEEP - struct timespec t; - t.tv_sec = 0; - t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ - nanosleep(&t, 0); -#else - sleep(1); -#endif - throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); - } - else - throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); -} - - -/* Convenience macros for retrying a SQLite transaction. */ -#define retry_sqlite while (1) { try { -#define end_retry_sqlite break; } catch (SQLiteBusy & e) { } } - - -SQLite::~SQLite() -{ - try { - if (db && sqlite3_close(db) != SQLITE_OK) - throwSQLiteError(db, "closing database"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::create(sqlite3 * db, const string & s) -{ - checkInterrupt(); - assert(!stmt); - if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) - throwSQLiteError(db, "creating statement"); - this->db = db; -} - - -void SQLiteStmt::reset() -{ - assert(stmt); - /* Note: sqlite3_reset() returns the error code for the most - recent call to sqlite3_step(). So ignore it. */ - sqlite3_reset(stmt); - curArg = 1; -} - - -SQLiteStmt::~SQLiteStmt() -{ - try { - if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throwSQLiteError(db, "finalizing statement"); - } catch (...) { - ignoreException(); - } -} - - -void SQLiteStmt::bind(const string & value) -{ - if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind(int value) -{ - if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind64(long long value) -{ - if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -void SQLiteStmt::bind() -{ - if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(db, "binding argument"); -} - - -/* Helper class to ensure that prepared statements are reset when - leaving the scope that uses them. Unfinished prepared statements - prevent transactions from being aborted, and can cause locks to be - kept when they should be released. */ -struct SQLiteStmtUse -{ - SQLiteStmt & stmt; - SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt) - { - stmt.reset(); - } - ~SQLiteStmtUse() - { - try { - stmt.reset(); - } catch (...) { - ignoreException(); - } - } -}; - - -struct SQLiteTxn -{ - bool active; - sqlite3 * db; - - SQLiteTxn(sqlite3 * db) : active(false) { - this->db = db; - if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "starting transaction"); - active = true; - } - - void commit() - { - if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "committing transaction"); - active = false; - } - - ~SQLiteTxn() - { - try { - if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "aborting transaction"); - } catch (...) { - ignoreException(); - } - } -}; - - void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; @@ -494,6 +329,7 @@ void LocalStore::openDB(bool create) // ensure efficient lookup. stmtQueryPathFromHashPart.create(db, "select path from ValidPaths where path >= ? limit 1;"); + stmtQueryValidPaths.create(db, "select path from ValidPaths"); } @@ -688,23 +524,16 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & } -unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) +uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs) { - SQLiteStmtUse use(stmtRegisterValidPath); - stmtRegisterValidPath.bind(info.path); - stmtRegisterValidPath.bind("sha256:" + printHash(info.hash)); - stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); - if (info.deriver != "") - stmtRegisterValidPath.bind(info.deriver); - else - stmtRegisterValidPath.bind(); // null - if (info.narSize != 0) - stmtRegisterValidPath.bind64(info.narSize); - else - stmtRegisterValidPath.bind(); // null - if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering valid path `%1%' in database") % info.path); - unsigned long long id = sqlite3_last_insert_rowid(db); + stmtRegisterValidPath.use() + (info.path) + ("sha256:" + printHash(info.hash)) + (info.registrationTime == 0 ? time(0) : info.registrationTime) + (info.deriver, info.deriver != "") + (info.narSize, info.narSize != 0) + .exec(); + uint64_t id = sqlite3_last_insert_rowid(db); /* If this is a derivation, then store the derivation outputs in the database. This is useful for the garbage collector: it can @@ -720,13 +549,12 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che registration above is undone. */ if (checkOutputs) checkDerivationOutputs(info.path, drv); - foreach (DerivationOutputs::iterator, i, drv.outputs) { - SQLiteStmtUse use(stmtAddDerivationOutput); - stmtAddDerivationOutput.bind(id); - stmtAddDerivationOutput.bind(i->first); - stmtAddDerivationOutput.bind(i->second.path); - if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) - throwSQLiteError(db, format("adding derivation output for `%1%' in database") % info.path); + for (auto & i : drv.outputs) { + stmtAddDerivationOutput.use() + (id) + (i.first) + (i.second.path) + .exec(); } } @@ -734,76 +562,52 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che } -void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) +void LocalStore::addReference(uint64_t referrer, uint64_t reference) { - SQLiteStmtUse use(stmtAddReference); - stmtAddReference.bind(referrer); - stmtAddReference.bind(reference); - if (sqlite3_step(stmtAddReference) != SQLITE_DONE) - throwSQLiteError(db, "adding reference to database"); + stmtAddReference.use()(referrer)(reference).exec(); } void LocalStore::registerFailedPath(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtRegisterFailedPath); - stmtRegisterFailedPath.bind(path); - stmtRegisterFailedPath.bind(time(0)); - if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("registering failed path `%1%'") % path); - } end_retry_sqlite; + retrySQLite<void>([&]() { + stmtRegisterFailedPath.use()(path)(time(0)).step(); + }); } bool LocalStore::hasPathFailed(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtHasPathFailed); - stmtHasPathFailed.bind(path); - int res = sqlite3_step(stmtHasPathFailed); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying whether path failed"); - return res == SQLITE_ROW; - } end_retry_sqlite; + return retrySQLite<bool>([&]() { + return stmtHasPathFailed.use()(path).next(); + }); } PathSet LocalStore::queryFailedPaths() { - retry_sqlite { - SQLiteStmtUse use(stmtQueryFailedPaths); + return retrySQLite<PathSet>([&]() { + auto useQueryFailedPaths(stmtQueryFailedPaths.use()); PathSet res; - int r; - while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error querying failed paths"); + while (useQueryFailedPaths.next()) + res.insert(useQueryFailedPaths.getStr(0)); return res; - } end_retry_sqlite; + }); } void LocalStore::clearFailedPaths(const PathSet & paths) { - retry_sqlite { + retrySQLite<void>([&]() { SQLiteTxn txn(db); - foreach (PathSet::const_iterator, i, paths) { - SQLiteStmtUse use(stmtClearFailedPath); - stmtClearFailedPath.bind(*i); - if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) - throwSQLiteError(db, format("clearing failed path `%1%' in database") % *i); - } + for (auto & path : paths) + stmtClearFailedPath.use()(path).exec(); txn.commit(); - } end_retry_sqlite; + }); } @@ -828,47 +632,34 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) assertStorePath(path); - retry_sqlite { + return retrySQLite<ValidPathInfo>([&]() { /* Get the path info. */ - SQLiteStmtUse use1(stmtQueryPathInfo); + auto useQueryPathInfo(stmtQueryPathInfo.use()(path)); - stmtQueryPathInfo.bind(path); + if (!useQueryPathInfo.next()) + throw Error(format("path `%1%' is not valid") % path); - int r = sqlite3_step(stmtQueryPathInfo); - if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); - if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); + info.id = useQueryPathInfo.getInt(0); - info.id = sqlite3_column_int(stmtQueryPathInfo, 0); + info.hash = parseHashField(path, useQueryPathInfo.getStr(1)); - const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); - assert(s); - info.hash = parseHashField(path, s); + info.registrationTime = useQueryPathInfo.getInt(2); - info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); - - s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); + auto s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); if (s) info.deriver = s; /* Note that narSize = NULL yields 0. */ - info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4); + info.narSize = useQueryPathInfo.getInt(4); /* Get the references. */ - SQLiteStmtUse use2(stmtQueryReferences); - - stmtQueryReferences.bind(info.id); - - while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { - s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); - assert(s); - info.references.insert(s); - } + auto useQueryReferences(stmtQueryReferences.use()(info.id)); - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of `%1%'") % path); + while (useQueryReferences.next()) + info.references.insert(useQueryReferences.getStr(0)); return info; - } end_retry_sqlite; + }); } @@ -876,78 +667,56 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) narSize field. */ void LocalStore::updatePathInfo(const ValidPathInfo & info) { - SQLiteStmtUse use(stmtUpdatePathInfo); - if (info.narSize != 0) - stmtUpdatePathInfo.bind64(info.narSize); - else - stmtUpdatePathInfo.bind(); // null - stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash)); - stmtUpdatePathInfo.bind(info.path); - if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) - throwSQLiteError(db, format("updating info of path `%1%' in database") % info.path); + stmtUpdatePathInfo.use() + (info.narSize, info.narSize != 0) + ("sha256:" + printHash(info.hash)) + (info.path) + .exec(); } -unsigned long long LocalStore::queryValidPathId(const Path & path) +uint64_t LocalStore::queryValidPathId(const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); - if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); - throwSQLiteError(db, "querying path in database"); + auto use(stmtQueryPathInfo.use()(path)); + if (!use.next()) + throw Error(format("path ‘%1%’ is not valid") % path); + return use.getInt(0); } bool LocalStore::isValidPath_(const Path & path) { - SQLiteStmtUse use(stmtQueryPathInfo); - stmtQueryPathInfo.bind(path); - int res = sqlite3_step(stmtQueryPathInfo); - if (res != SQLITE_DONE && res != SQLITE_ROW) - throwSQLiteError(db, "querying path in database"); - return res == SQLITE_ROW; + return stmtQueryPathInfo.use()(path).next(); } bool LocalStore::isValidPath(const Path & path) { - retry_sqlite { + return retrySQLite<bool>([&]() { return isValidPath_(path); - } end_retry_sqlite; + }); } PathSet LocalStore::queryValidPaths(const PathSet & paths) { - retry_sqlite { + return retrySQLite<PathSet>([&]() { PathSet res; foreach (PathSet::const_iterator, i, paths) if (isValidPath_(*i)) res.insert(*i); return res; - } end_retry_sqlite; + }); } PathSet LocalStore::queryAllValidPaths() { - retry_sqlite { - SQLiteStmt stmt; - stmt.create(db, "select path from ValidPaths"); - + return retrySQLite<PathSet>([&]() { + auto use(stmtQueryValidPaths.use()); PathSet res; - int r; - while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmt, 0); - assert(s); - res.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, "error getting valid paths"); - + while (use.next()) res.insert(use.getStr(0)); return res; - } end_retry_sqlite; + }); } @@ -961,28 +730,19 @@ void LocalStore::queryReferences(const Path & path, void LocalStore::queryReferrers_(const Path & path, PathSet & referrers) { - SQLiteStmtUse use(stmtQueryReferrers); - - stmtQueryReferrers.bind(path); + auto useQueryReferrers(stmtQueryReferrers.use()(path)); - int r; - while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0); - assert(s); - referrers.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting references of `%1%'") % path); + while (useQueryReferrers.next()) + referrers.insert(useQueryReferrers.getStr(0)); } void LocalStore::queryReferrers(const Path & path, PathSet & referrers) { assertStorePath(path); - retry_sqlite { + return retrySQLite<void>([&]() { queryReferrers_(path, referrers); - } end_retry_sqlite; + }); } @@ -996,67 +756,43 @@ PathSet LocalStore::queryValidDerivers(const Path & path) { assertStorePath(path); - retry_sqlite { - SQLiteStmtUse use(stmtQueryValidDerivers); - stmtQueryValidDerivers.bind(path); + return retrySQLite<PathSet>([&]() { + auto useQueryValidDerivers(stmtQueryValidDerivers.use()(path)); PathSet derivers; - int r; - while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1); - assert(s); - derivers.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting valid derivers of `%1%'") % path); + while (useQueryValidDerivers.next()) + derivers.insert(useQueryValidDerivers.getStr(1)); return derivers; - } end_retry_sqlite; + }); } PathSet LocalStore::queryDerivationOutputs(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + return retrySQLite<PathSet>([&]() { + auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path))); PathSet outputs; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1); - assert(s); - outputs.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting outputs of `%1%'") % path); + while (useQueryDerivationOutputs.next()) + outputs.insert(useQueryDerivationOutputs.getStr(1)); return outputs; - } end_retry_sqlite; + }); } StringSet LocalStore::queryDerivationOutputNames(const Path & path) { - retry_sqlite { - SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryValidPathId(path)); + return retrySQLite<StringSet>([&]() { + auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path))); StringSet outputNames; - int r; - while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { - const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0); - assert(s); - outputNames.insert(s); - } - - if (r != SQLITE_DONE) - throwSQLiteError(db, format("error getting output names of `%1%'") % path); + while (useQueryDerivationOutputs.next()) + outputNames.insert(useQueryDerivationOutputs.getStr(0)); return outputNames; - } end_retry_sqlite; + }); } @@ -1066,17 +802,14 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart) Path prefix = settings.nixStore + "/" + hashPart; - retry_sqlite { - SQLiteStmtUse use(stmtQueryPathFromHashPart); - stmtQueryPathFromHashPart.bind(prefix); + return retrySQLite<Path>([&]() -> Path { + auto useQueryPathFromHashPart(stmtQueryPathFromHashPart.use()(prefix)); - int res = sqlite3_step(stmtQueryPathFromHashPart); - if (res == SQLITE_DONE) return ""; - if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database"); + if (!useQueryPathFromHashPart.next()) return ""; const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0); return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; - } end_retry_sqlite; + }); } @@ -1306,7 +1039,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) * expense of some speed of the path registering operation. */ if (settings.syncBeforeRegistering) sync(); - retry_sqlite { + return retrySQLite<void>([&]() { SQLiteTxn txn(db); PathSet paths; @@ -1319,10 +1052,10 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) paths.insert(i->path); } - foreach (ValidPathInfos::const_iterator, i, infos) { - unsigned long long referrer = queryValidPathId(i->path); - foreach (PathSet::iterator, j, i->references) - addReference(referrer, queryValidPathId(*j)); + for (auto & i : infos) { + auto referrer = queryValidPathId(i.path); + for (auto & j : i.references) + addReference(referrer, queryValidPathId(j)); } /* Check that the derivation outputs are correct. We can't do @@ -1343,7 +1076,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) topoSortPaths(*this, paths); txn.commit(); - } end_retry_sqlite; + }); } @@ -1355,12 +1088,7 @@ void LocalStore::invalidatePath(const Path & path) drvHashes.erase(path); - SQLiteStmtUse use(stmtInvalidatePath); - - stmtInvalidatePath.bind(path); - - if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) - throwSQLiteError(db, format("invalidating path `%1%' in database") % path); + stmtInvalidatePath.use()(path).exec(); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ @@ -1733,7 +1461,7 @@ void LocalStore::invalidatePathChecked(const Path & path) { assertStorePath(path); - retry_sqlite { + retrySQLite<void>([&]() { SQLiteTxn txn(db); if (isValidPath_(path)) { @@ -1746,7 +1474,7 @@ void LocalStore::invalidatePathChecked(const Path & path) } txn.commit(); - } end_retry_sqlite; + }); } diff --git a/nix/libstore/local-store.hh b/nix/libstore/local-store.hh index 819f59327a..6110468498 100644 --- a/nix/libstore/local-store.hh +++ b/nix/libstore/local-store.hh @@ -1,15 +1,12 @@ #pragma once +#include "sqlite.hh" #include <string> #include <unordered_set> +#include "pathlocks.hh" #include "store-api.hh" #include "util.hh" -#include "pathlocks.hh" - - -class sqlite3; -class sqlite3_stmt; namespace nix { @@ -52,34 +49,6 @@ struct RunningSubstituter }; -/* Wrapper object to close the SQLite database automatically. */ -struct SQLite -{ - sqlite3 * db; - SQLite() { db = 0; } - ~SQLite(); - operator sqlite3 * () { return db; } -}; - - -/* Wrapper object to create and destroy SQLite prepared statements. */ -struct SQLiteStmt -{ - sqlite3 * db; - sqlite3_stmt * stmt; - unsigned int curArg; - SQLiteStmt() { stmt = 0; } - void create(sqlite3 * db, const string & s); - void reset(); - ~SQLiteStmt(); - operator sqlite3_stmt * () { return stmt; } - void bind(const string & value); - void bind(int value); - void bind64(long long value); - void bind(); -}; - - class LocalStore : public StoreAPI { private: @@ -238,6 +207,7 @@ private: SQLiteStmt stmtQueryValidDerivers; SQLiteStmt stmtQueryDerivationOutputs; SQLiteStmt stmtQueryPathFromHashPart; + SQLiteStmt stmtQueryValidPaths; /* Cache for pathContentsGood(). */ std::map<Path, bool> pathContentsGoodCache; @@ -254,11 +224,11 @@ private: void makeStoreWritable(); - unsigned long long queryValidPathId(const Path & path); + uint64_t queryValidPathId(const Path & path); - unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true); + uint64_t addValidPath(const ValidPathInfo & info, bool checkOutputs = true); - void addReference(unsigned long long referrer, unsigned long long reference); + void addReference(uint64_t referrer, uint64_t reference); void appendReferrer(const Path & from, const Path & to, bool lock); diff --git a/nix/libstore/optimise-store.cc b/nix/libstore/optimise-store.cc index d7508b025e..565c62ca83 100644 --- a/nix/libstore/optimise-store.cc +++ b/nix/libstore/optimise-store.cc @@ -148,10 +148,23 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa inodeHash.insert(st.st_ino); return; } - if (errno != EEXIST) + + switch (errno) { + case EEXIST: + /* Fall through if another process created ‘linkPath’ before + we did. */ + break; + + case ENOSPC: + /* On ext4, that probably means the directory index is full. When + that happens, it's fine to ignore it: we just effectively + disable deduplication of this file. */ + printMsg(lvlInfo, format("cannot link `%1%' to `%2%': %m") % linkPath % path); + return; + + default: throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path); - /* Fall through if another process created ‘linkPath’ before - we did. */ + } } /* Yes! We've seen a file with the same contents. Replace the @@ -195,8 +208,8 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHa printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); return; } - throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath); - } + throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath); + } /* Atomically replace the old file with the new hard link. */ if (rename(tempLink.c_str(), path.c_str()) == -1) { diff --git a/nix/libstore/sqlite.cc b/nix/libstore/sqlite.cc new file mode 100644 index 0000000000..e08c67f40e --- /dev/null +++ b/nix/libstore/sqlite.cc @@ -0,0 +1,166 @@ +#include "sqlite.hh" +#include "util.hh" + +#include <sqlite3.h> + +namespace nix { + +[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f) +{ + int err = sqlite3_errcode(db); + if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { + if (err == SQLITE_PROTOCOL) + printMsg(lvlError, "warning: SQLite database is busy (SQLITE_PROTOCOL)"); + else { + static bool warned = false; + if (!warned) { + printMsg(lvlError, "warning: SQLite database is busy"); + warned = true; + } + } + /* Sleep for a while since retrying the transaction right away + is likely to fail again. */ +#if HAVE_NANOSLEEP + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ + nanosleep(&t, 0); +#else + sleep(1); +#endif + throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); + } + else + throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); +} + +SQLite::~SQLite() +{ + try { + if (db && sqlite3_close(db) != SQLITE_OK) + throwSQLiteError(db, "closing database"); + } catch (...) { + ignoreException(); + } +} + +void SQLiteStmt::create(sqlite3 * db, const string & s) +{ + checkInterrupt(); + assert(!stmt); + if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) + throwSQLiteError(db, "creating statement"); + this->db = db; +} + +SQLiteStmt::~SQLiteStmt() +{ + try { + if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) + throwSQLiteError(db, "finalizing statement"); + } catch (...) { + ignoreException(); + } +} + +SQLiteStmt::Use::Use(SQLiteStmt & stmt) + : stmt(stmt) +{ + assert(stmt.stmt); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use::~Use() +{ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + +SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + +SQLiteStmt::Use & SQLiteStmt::Use::bind() +{ + if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + return *this; +} + +int SQLiteStmt::Use::step() +{ + return sqlite3_step(stmt); +} + +void SQLiteStmt::Use::exec() +{ + int r = step(); + assert(r != SQLITE_ROW); + if (r != SQLITE_DONE) + throwSQLiteError(stmt.db, "executing SQLite statement"); +} + +bool SQLiteStmt::Use::next() +{ + int r = step(); + if (r != SQLITE_DONE && r != SQLITE_ROW) + throwSQLiteError(stmt.db, "executing SQLite query"); + return r == SQLITE_ROW; +} + +std::string SQLiteStmt::Use::getStr(int col) +{ + auto s = (const char *) sqlite3_column_text(stmt, col); + assert(s); + return s; +} + +int64_t SQLiteStmt::Use::getInt(int col) +{ + // FIXME: detect nulls? + return sqlite3_column_int64(stmt, col); +} + +SQLiteTxn::SQLiteTxn(sqlite3 * db) +{ + this->db = db; + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "starting transaction"); + active = true; +} + +void SQLiteTxn::commit() +{ + if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "committing transaction"); + active = false; +} + +SQLiteTxn::~SQLiteTxn() +{ + try { + if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "aborting transaction"); + } catch (...) { + ignoreException(); + } +} + +} diff --git a/nix/libstore/sqlite.hh b/nix/libstore/sqlite.hh new file mode 100644 index 0000000000..326e4a4855 --- /dev/null +++ b/nix/libstore/sqlite.hh @@ -0,0 +1,102 @@ +#pragma once + +#include <functional> +#include <string> + +#include "types.hh" + +class sqlite3; +class sqlite3_stmt; + +namespace nix { + +/* RAII wrapper to close a SQLite database automatically. */ +struct SQLite +{ + sqlite3 * db; + SQLite() { db = 0; } + ~SQLite(); + operator sqlite3 * () { return db; } +}; + +/* RAII wrapper to create and destroy SQLite prepared statements. */ +struct SQLiteStmt +{ + sqlite3 * db = 0; + sqlite3_stmt * stmt = 0; + SQLiteStmt() { } + void create(sqlite3 * db, const std::string & s); + ~SQLiteStmt(); + operator sqlite3_stmt * () { return stmt; } + + /* Helper for binding / executing statements. */ + class Use + { + friend struct SQLiteStmt; + private: + SQLiteStmt & stmt; + unsigned int curArg = 1; + Use(SQLiteStmt & stmt); + + public: + + ~Use(); + + /* Bind the next parameter. */ + Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (int64_t value, bool notNull = true); + Use & bind(); // null + + int step(); + + /* Execute a statement that does not return rows. */ + void exec(); + + /* For statements that return 0 or more rows. Returns true iff + a row is available. */ + bool next(); + + std::string getStr(int col); + int64_t getInt(int col); + }; + + Use use() + { + return Use(*this); + } +}; + +/* RAII helper that ensures transactions are aborted unless explicitly + committed. */ +struct SQLiteTxn +{ + bool active = false; + sqlite3 * db; + + SQLiteTxn(sqlite3 * db); + + void commit(); + + ~SQLiteTxn(); +}; + + +MakeError(SQLiteError, Error); +MakeError(SQLiteBusy, SQLiteError); + +[[noreturn]] void throwSQLiteError(sqlite3 * db, const format & f); + +/* Convenience function for retrying a SQLite transaction when the + database is busy. */ +template<typename T> +T retrySQLite(std::function<T()> fun) +{ + while (true) { + try { + return fun(); + } catch (SQLiteBusy & e) { + } + } +} + +} diff --git a/nix/libstore/store-api.hh b/nix/libstore/store-api.hh index 3e982f6dd3..fa78d595f2 100644 --- a/nix/libstore/store-api.hh +++ b/nix/libstore/store-api.hh @@ -89,8 +89,8 @@ struct ValidPathInfo Hash hash; PathSet references; time_t registrationTime = 0; - unsigned long long narSize = 0; // 0 = unknown - unsigned long long id; // internal use only + uint64_t narSize = 0; // 0 = unknown + uint64_t id; // internal use only bool operator == (const ValidPathInfo & i) const { |